You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/06/06 18:35:28 UTC
[01/36] hive git commit: HIVE-16757 Remove calls to deprecated
AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/hive-14535 d9e4ae09e -> 78f28c885
HIVE-16757 Remove calls to deprecated AbstractRelNode.getRows (Remus Rusanu reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aee8d4f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aee8d4f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aee8d4f
Branch: refs/heads/hive-14535
Commit: 8aee8d4f2b124fcfa093724b4de0a54287a8084f
Parents: 86b1877
Author: Remus Rusanu <re...@apache.org>
Authored: Tue May 30 22:37:54 2017 -0700
Committer: Remus Rusanu <re...@apache.org>
Committed: Tue May 30 22:37:54 2017 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java | 4 ++--
.../hive/ql/optimizer/calcite/cost/HiveCostModel.java | 7 ++++---
.../ql/optimizer/calcite/cost/HiveDefaultCostModel.java | 2 +-
.../hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java | 4 ++--
.../hive/ql/optimizer/calcite/cost/HiveRelMdCost.java | 2 +-
.../optimizer/calcite/stats/FilterSelectivityEstimator.java | 9 +++++----
.../optimizer/calcite/stats/HiveRelMdDistinctRowCount.java | 2 +-
.../hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java | 3 ++-
.../ql/optimizer/calcite/stats/HiveRelMdSelectivity.java | 4 ++--
.../ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java | 2 +-
10 files changed, 21 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
index 4ba10e2..28c7362 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveAlgorithmsUtil.java
@@ -60,8 +60,8 @@ public class HiveAlgorithmsUtil {
* Double.parseDouble(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_CBO_COST_MODEL_HDFS_READ));
}
- public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr) {
- return new HiveCost(hr.getRows(), 0, 0);
+ public static RelOptCost computeCardinalityBasedCost(HiveRelNode hr, RelMetadataQuery mq) {
+ return new HiveCost(mq.getRowCount(hr), 0, 0);
}
public HiveCost computeScanCost(double cardinality, double avgTupleSize) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
index 4af1f8d..3783d5f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveCostModel.java
@@ -23,13 +23,14 @@ import org.apache.calcite.plan.RelOptCost;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelDistribution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Cost model interface.
@@ -49,7 +50,7 @@ public abstract class HiveCostModel {
public abstract RelOptCost getAggregateCost(HiveAggregate aggregate);
- public abstract RelOptCost getScanCost(HiveTableScan ts);
+ public abstract RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq);
public RelOptCost getJoinCost(HiveJoin join) {
// Select algorithm with min cost
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
index 40f2cef..11b6aa3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveDefaultCostModel.java
@@ -55,7 +55,7 @@ public class HiveDefaultCostModel extends HiveCostModel {
}
@Override
- public RelOptCost getScanCost(HiveTableScan ts) {
+ public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) {
return HiveCost.FACTORY.makeZeroCost();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
index af5fc5e..2dbfd8f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveOnTezCostModel.java
@@ -78,8 +78,8 @@ public class HiveOnTezCostModel extends HiveCostModel {
}
@Override
- public RelOptCost getScanCost(HiveTableScan ts) {
- return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts));
+ public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) {
+ return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts));
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
index cbea307..c6b8ce3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/cost/HiveRelMdCost.java
@@ -67,7 +67,7 @@ public class HiveRelMdCost implements MetadataHandler<BuiltInMetadata.NonCumulat
}
public RelOptCost getNonCumulativeCost(HiveTableScan ts, RelMetadataQuery mq) {
- return hiveCostModel.getScanCost(ts);
+ return hiveCostModel.getScanCost(ts, mq);
}
// Default case
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index a25b58b..9b8e6a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -44,11 +44,13 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics;
public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
private final RelNode childRel;
private final double childCardinality;
+ private final RelMetadataQuery mq;
- protected FilterSelectivityEstimator(RelNode childRel) {
+ protected FilterSelectivityEstimator(RelNode childRel, RelMetadataQuery mq) {
super(true);
+ this.mq = mq;
this.childRel = childRel;
- this.childCardinality = RelMetadataQuery.instance().getRowCount(childRel);
+ this.childCardinality = mq.getRowCount(childRel);
}
public Double estimateSelectivity(RexNode predicate) {
@@ -91,7 +93,7 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
case IS_NOT_NULL: {
if (childRel instanceof HiveTableScan) {
double noOfNulls = getMaxNulls(call, (HiveTableScan) childRel);
- double totalNoOfTuples = childRel.getRows();
+ double totalNoOfTuples = mq.getRowCount(childRel);
if (totalNoOfTuples >= noOfNulls) {
selectivity = (totalNoOfTuples - noOfNulls) / Math.max(totalNoOfTuples, 1);
} else {
@@ -252,7 +254,6 @@ public class FilterSelectivityEstimator extends RexVisitorImpl<Double> {
double tmpNDV;
double maxNDV = 1.0;
InputReferencedVisitor irv;
- RelMetadataQuery mq = RelMetadataQuery.instance();
for (RexNode op : call.getOperands()) {
if (op instanceof RexInputRef) {
tmpNDV = HiveRelMdDistinctRowCount.getDistinctRowCount(this.childRel, mq,
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
index 77f7aa8..be2d5b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdDistinctRowCount.java
@@ -81,7 +81,7 @@ public class HiveRelMdDistinctRowCount extends RelMdDistinctRowCount {
noDistinctRows *= cStat.getCountDistint();
}
- return Math.min(noDistinctRows, htRel.getRows());
+ return Math.min(noDistinctRows, mq.getRowCount(htRel));
}
public static Double getDistinctRowCount(RelNode r, RelMetadataQuery mq, int indx) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
index 7bba80b..9199d52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdRowCount.java
@@ -72,7 +72,8 @@ public class HiveRelMdRowCount extends RelMdRowCount {
}
return pkfk.fkInfo.rowCount * selectivity;
}
- return join.getRows();
+ // Do not call mq.getRowCount(join), will trigger CyclicMetadataException
+ return join.estimateRowCount(mq);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
index 7192684..046f51b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSelectivity.java
@@ -56,7 +56,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
public Double getSelectivity(HiveTableScan t, RelMetadataQuery mq, RexNode predicate) {
if (predicate != null) {
- FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t);
+ FilterSelectivityEstimator filterSelEstmator = new FilterSelectivityEstimator(t, mq);
return filterSelEstmator.estimateSelectivity(predicate);
}
@@ -85,7 +85,7 @@ public class HiveRelMdSelectivity extends RelMdSelectivity {
getCombinedPredicateForJoin(j, predicate);
if (!predInfo.getKey()) {
return
- new FilterSelectivityEstimator(j).
+ new FilterSelectivityEstimator(j, mq).
estimateSelectivity(predInfo.getValue());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8aee8d4f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
index 9a5a2ba..aa4d8a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdUniqueKeys.java
@@ -138,7 +138,7 @@ public class HiveRelMdUniqueKeys implements MetadataHandler<BuiltInMetadata.Uniq
projectPos++;
}
- double numRows = tScan.getRows();
+ double numRows = mq.getRowCount(tScan);
List<ColStatistics> colStats = tScan.getColStat(BitSets
.toList(projectedCols));
Set<ImmutableBitSet> keys = new HashSet<ImmutableBitSet>();
[03/36] hive git commit: HIVE-16776: Strange cast behavior for table
backed by druid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16776: Strange cast behavior for table backed by druid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/727a3dfc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/727a3dfc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/727a3dfc
Branch: refs/heads/hive-14535
Commit: 727a3dfcda43972b5947b8351223506e2bdfb4c0
Parents: 509308f
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 31 15:54:34 2017 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 31 15:54:34 2017 +0100
----------------------------------------------------------------------
.../hadoop/hive/druid/serde/DruidSerDe.java | 95 ++++++++---
.../hadoop/hive/druid/TestDruidSerDe.java | 169 ++++++++++++++++++-
.../test/queries/clientpositive/druid_basic2.q | 4 +
.../results/clientpositive/druid_basic2.q.out | 40 +++++
4 files changed, 278 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/727a3dfc/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
index 656c0f1..2d11e4b 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
@@ -40,7 +40,10 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.SerDeStats;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -58,12 +61,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspe
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.StringUtils;
@@ -73,6 +73,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.metamx.common.lifecycle.Lifecycle;
import com.metamx.http.client.HttpClient;
@@ -203,12 +204,29 @@ public class DruidSerDe extends AbstractSerDe {
try {
query = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, Query.class);
+ // Extract column names and types (if present)
+ ImmutableMap.Builder<String, PrimitiveTypeInfo> mapColumnNamesTypes = ImmutableMap.builder();
+ if (!org.apache.commons.lang3.StringUtils
+ .isEmpty(properties.getProperty(serdeConstants.LIST_COLUMNS))
+ && !org.apache.commons.lang3.StringUtils
+ .isEmpty(properties.getProperty(serdeConstants.LIST_COLUMN_TYPES))) {
+ List<String> propColumnNames = Utilities.getColumnNames(properties);
+ List<String> propColumnTypes = Utilities.getColumnTypes(properties);
+ for (int i = 0; i < propColumnNames.size(); i++) {
+ mapColumnNamesTypes.put(
+ propColumnNames.get(i),
+ TypeInfoFactory.getPrimitiveTypeInfo(propColumnTypes.get(i)));
+ }
+ }
+
switch (query.getType()) {
case Query.TIMESERIES:
- inferSchema((TimeseriesQuery) query, columnNames, columnTypes);
+ inferSchema((TimeseriesQuery) query, columnNames, columnTypes,
+ mapColumnNamesTypes.build());
break;
case Query.TOPN:
- inferSchema((TopNQuery) query, columnNames, columnTypes);
+ inferSchema((TopNQuery) query, columnNames, columnTypes,
+ mapColumnNamesTypes.build());
break;
case Query.SELECT:
String address = HiveConf.getVar(configuration,
@@ -216,10 +234,12 @@ public class DruidSerDe extends AbstractSerDe {
if (org.apache.commons.lang3.StringUtils.isEmpty(address)) {
throw new SerDeException("Druid broker address not specified in configuration");
}
- inferSchema((SelectQuery) query, columnNames, columnTypes, address);
+ inferSchema((SelectQuery) query, columnNames, columnTypes, address,
+ mapColumnNamesTypes.build());
break;
case Query.GROUP_BY:
- inferSchema((GroupByQuery) query, columnNames, columnTypes);
+ inferSchema((GroupByQuery) query, columnNames, columnTypes,
+ mapColumnNamesTypes.build());
break;
default:
throw new SerDeException("Not supported Druid query");
@@ -233,8 +253,7 @@ public class DruidSerDe extends AbstractSerDe {
for (int i = 0; i < columnTypes.size(); ++i) {
columns[i] = columnNames.get(i);
types[i] = columnTypes.get(i);
- inspectors
- .add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(types[i]));
+ inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(types[i]));
}
inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
}
@@ -280,16 +299,22 @@ public class DruidSerDe extends AbstractSerDe {
}
/* Timeseries query */
- private void inferSchema(TimeseriesQuery query, List<String> columnNames,
- List<PrimitiveTypeInfo> columnTypes
- ) {
+ private void inferSchema(TimeseriesQuery query,
+ List<String> columnNames, List<PrimitiveTypeInfo> columnTypes,
+ Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidTable.DEFAULT_TIMESTAMP_COLUMN);
columnTypes.add(TypeInfoFactory.timestampTypeInfo);
// Aggregator columns
for (AggregatorFactory af : query.getAggregatorSpecs()) {
columnNames.add(af.getName());
- columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ PrimitiveTypeInfo typeInfo = mapColumnNamesTypes.get(af.getName());
+ if (typeInfo != null) {
+ // If datasource was created by Hive, we consider Hive type
+ columnTypes.add(typeInfo);
+ } else {
+ columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ }
}
// Post-aggregator columns
// TODO: Currently Calcite only infers avg for post-aggregate,
@@ -302,9 +327,9 @@ public class DruidSerDe extends AbstractSerDe {
}
/* TopN query */
- private void inferSchema(TopNQuery query, List<String> columnNames,
- List<PrimitiveTypeInfo> columnTypes
- ) {
+ private void inferSchema(TopNQuery query,
+ List<String> columnNames, List<PrimitiveTypeInfo> columnTypes,
+ Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidTable.DEFAULT_TIMESTAMP_COLUMN);
columnTypes.add(TypeInfoFactory.timestampTypeInfo);
@@ -314,7 +339,13 @@ public class DruidSerDe extends AbstractSerDe {
// Aggregator columns
for (AggregatorFactory af : query.getAggregatorSpecs()) {
columnNames.add(af.getName());
- columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ PrimitiveTypeInfo typeInfo = mapColumnNamesTypes.get(af.getName());
+ if (typeInfo != null) {
+ // If datasource was created by Hive, we consider Hive type
+ columnTypes.add(typeInfo);
+ } else {
+ columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ }
}
// Post-aggregator columns
// TODO: Currently Calcite only infers avg for post-aggregate,
@@ -327,8 +358,10 @@ public class DruidSerDe extends AbstractSerDe {
}
/* Select query */
- private void inferSchema(SelectQuery query, List<String> columnNames,
- List<PrimitiveTypeInfo> columnTypes, String address) throws SerDeException {
+ private void inferSchema(SelectQuery query,
+ List<String> columnNames, List<PrimitiveTypeInfo> columnTypes,
+ String address, Map<String, PrimitiveTypeInfo> mapColumnNamesTypes)
+ throws SerDeException {
// Timestamp column
columnNames.add(DruidTable.DEFAULT_TIMESTAMP_COLUMN);
columnTypes.add(TypeInfoFactory.timestampTypeInfo);
@@ -356,15 +389,21 @@ public class DruidSerDe extends AbstractSerDe {
}
for (String metric : query.getMetrics()) {
columnNames.add(metric);
- columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(
- schemaInfo.getColumns().get(metric).getType()));
+ PrimitiveTypeInfo typeInfo = mapColumnNamesTypes.get(metric);
+ if (typeInfo != null) {
+ // If datasource was created by Hive, we consider Hive type
+ columnTypes.add(typeInfo);
+ } else {
+ columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(
+ schemaInfo.getColumns().get(metric).getType()));
+ }
}
}
/* GroupBy query */
- private void inferSchema(GroupByQuery query, List<String> columnNames,
- List<PrimitiveTypeInfo> columnTypes
- ) {
+ private void inferSchema(GroupByQuery query,
+ List<String> columnNames, List<PrimitiveTypeInfo> columnTypes,
+ Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidTable.DEFAULT_TIMESTAMP_COLUMN);
columnTypes.add(TypeInfoFactory.timestampTypeInfo);
@@ -376,7 +415,13 @@ public class DruidSerDe extends AbstractSerDe {
// Aggregator columns
for (AggregatorFactory af : query.getAggregatorSpecs()) {
columnNames.add(af.getName());
- columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ PrimitiveTypeInfo typeInfo = mapColumnNamesTypes.get(af.getName());
+ if (typeInfo != null) {
+ // If datasource was created by Hive, we consider Hive type
+ columnTypes.add(typeInfo);
+ } else {
+ columnTypes.add(DruidSerDeUtils.convertDruidToHiveType(af.getTypeName()));
+ }
}
// Post-aggregator columns
// TODO: Currently Calcite only infers avg for post-aggregate,
http://git-wip-us.apache.org/repos/asf/hive/blob/727a3dfc/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidSerDe.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidSerDe.java
index 1bd5d84..695e0dd 100644
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidSerDe.java
+++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidSerDe.java
@@ -143,6 +143,16 @@ public class TestDruidSerDe {
new FloatWritable(3.32F), new FloatWritable(4F) }
};
+ // Timeseries query results as records (types defined by metastore)
+ private static final String TIMESERIES_COLUMN_NAMES = "__time,sample_name1,sample_name2,sample_divide";
+ private static final String TIMESERIES_COLUMN_TYPES = "timestamp,smallint,double,float";
+ private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS_2 = new Object[][] {
+ new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new ShortWritable((short) 0),
+ new DoubleWritable(1.0d), new FloatWritable(2.2222F) },
+ new Object[] { new TimestampWritable(new Timestamp(1325462400000L)), new ShortWritable((short) 2),
+ new DoubleWritable(3.32d), new FloatWritable(4F) }
+ };
+
// TopN query
private static final String TOPN_QUERY =
"{ \"queryType\": \"topN\", "
@@ -259,6 +269,27 @@ public class TestDruidSerDe {
new FloatWritable(46.45F) }
};
+ // TopN query results as records (types defined by metastore)
+ private static final String TOPN_COLUMN_NAMES = "__time,sample_dim,count,some_metric,sample_divide";
+ private static final String TOPN_COLUMN_TYPES = "timestamp,string,bigint,double,float";
+ private static final Object[][] TOPN_QUERY_RESULTS_RECORDS_2 = new Object[][] {
+ new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("dim1_val"), new LongWritable(111), new DoubleWritable(10669d),
+ new FloatWritable(96.11711711711712F) },
+ new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("another_dim1_val"), new LongWritable(88), new DoubleWritable(28344d),
+ new FloatWritable(322.09090909090907F) },
+ new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("dim1_val3"), new LongWritable(70), new DoubleWritable(871d),
+ new FloatWritable(12.442857142857143F) },
+ new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("dim1_val4"), new LongWritable(62), new DoubleWritable(815d),
+ new FloatWritable(13.14516129032258F) },
+ new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("dim1_val5"), new LongWritable(60), new DoubleWritable(2787d),
+ new FloatWritable(46.45F) }
+ };
+
// GroupBy query
private static final String GROUP_BY_QUERY =
"{ "
@@ -339,6 +370,18 @@ public class TestDruidSerDe {
new FloatWritable(6.333333F) }
};
+ // GroupBy query results as records (types defined by metastore)
+ private static final String GROUP_BY_COLUMN_NAMES = "__time,country,device,total_usage,data_transfer,avg_usage";
+ private static final String GROUP_BY_COLUMN_TYPES = "timestamp,string,string,int,double,float";
+ private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS_2 = new Object[][] {
+ new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new Text("India"),
+ new Text("phone"), new IntWritable(88), new DoubleWritable(29.91233453F),
+ new FloatWritable(60.32F) },
+ new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new Text("Spain"),
+ new Text("pc"), new IntWritable(16), new DoubleWritable(172.93494959F),
+ new FloatWritable(6.333333F) }
+ };
+
// Select query
private static final String SELECT_QUERY =
"{ \"queryType\": \"select\", "
@@ -483,6 +526,38 @@ public class TestDruidSerDe {
new FloatWritable(68.0F), new FloatWritable(0.0F) }
};
+ // Select query results as records (types defined by metastore)
+ private static final String SELECT_COLUMN_NAMES = "__time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted";
+ private static final String SELECT_COLUMN_TYPES = "timestamp,string,string,string,string,string,string,string,string,double,double,float,float,float";
+ private static final Object[][] SELECT_QUERY_RESULTS_RECORDS_2 = new Object[][] {
+ new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("1"),
+ new Text("article"), new Text("0"), new Text("0"),
+ new Text("11._korpus_(NOVJ)"), new Text("sl"), new Text("0"),
+ new Text("EmausBot"),
+ new DoubleWritable(1.0d), new DoubleWritable(39.0d), new FloatWritable(39.0F),
+ new FloatWritable(39.0F), new FloatWritable(0.0F) },
+ new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("0"),
+ new Text("article"), new Text("0"), new Text("0"),
+ new Text("112_U.S._580"), new Text("en"), new Text("1"), new Text("MZMcBride"),
+ new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F),
+ new FloatWritable(70.0F), new FloatWritable(0.0F) },
+ new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Text("article"), new Text("0"), new Text("0"),
+ new Text("113_U.S._243"), new Text("en"), new Text("1"), new Text("MZMcBride"),
+ new DoubleWritable(1.0d), new DoubleWritable(77.0d), new FloatWritable(77.0F),
+ new FloatWritable(77.0F), new FloatWritable(0.0F) },
+ new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Text("article"), new Text("0"), new Text("0"),
+ new Text("113_U.S._73"), new Text("en"), new Text("1"), new Text("MZMcBride"),
+ new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F),
+ new FloatWritable(70.0F), new FloatWritable(0.0F) },
+ new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Text("article"), new Text("0"), new Text("0"),
+ new Text("113_U.S._756"), new Text("en"), new Text("1"), new Text("MZMcBride"),
+ new DoubleWritable(1.0d), new DoubleWritable(68.0d), new FloatWritable(68.0F),
+ new FloatWritable(68.0F), new FloatWritable(0.0F) }
+ };
+
/**
* Test the default behavior of the objects and object inspectors.
* @throws IOException
@@ -511,24 +586,52 @@ public class TestDruidSerDe {
deserializeQueryResults(serDe, Query.TIMESERIES, TIMESERIES_QUERY,
TIMESERIES_QUERY_RESULTS, TIMESERIES_QUERY_RESULTS_RECORDS
);
+ // Timeseries query (simulating column types from metastore)
+ tbl.setProperty(serdeConstants.LIST_COLUMNS, TIMESERIES_COLUMN_NAMES);
+ tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, TIMESERIES_COLUMN_TYPES);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ deserializeQueryResults(serDe, Query.TIMESERIES, TIMESERIES_QUERY,
+ TIMESERIES_QUERY_RESULTS, TIMESERIES_QUERY_RESULTS_RECORDS_2
+ );
// TopN query
tbl = createPropertiesQuery("sample_data", Query.TOPN, TOPN_QUERY);
SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
deserializeQueryResults(serDe, Query.TOPN, TOPN_QUERY,
TOPN_QUERY_RESULTS, TOPN_QUERY_RESULTS_RECORDS
);
+ // TopN query (simulating column types from metastore)
+ tbl.setProperty(serdeConstants.LIST_COLUMNS, TOPN_COLUMN_NAMES);
+ tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, TOPN_COLUMN_TYPES);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ deserializeQueryResults(serDe, Query.TOPN, TOPN_QUERY,
+ TOPN_QUERY_RESULTS, TOPN_QUERY_RESULTS_RECORDS_2
+ );
// GroupBy query
tbl = createPropertiesQuery("sample_datasource", Query.GROUP_BY, GROUP_BY_QUERY);
SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
deserializeQueryResults(serDe, Query.GROUP_BY, GROUP_BY_QUERY,
GROUP_BY_QUERY_RESULTS, GROUP_BY_QUERY_RESULTS_RECORDS
);
+ // GroupBy query (simulating column types from metastore)
+ tbl.setProperty(serdeConstants.LIST_COLUMNS, GROUP_BY_COLUMN_NAMES);
+ tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, GROUP_BY_COLUMN_TYPES);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ deserializeQueryResults(serDe, Query.GROUP_BY, GROUP_BY_QUERY,
+ GROUP_BY_QUERY_RESULTS, GROUP_BY_QUERY_RESULTS_RECORDS_2
+ );
// Select query
tbl = createPropertiesQuery("wikipedia", Query.SELECT, SELECT_QUERY);
SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
deserializeQueryResults(serDe, Query.SELECT, SELECT_QUERY,
SELECT_QUERY_RESULTS, SELECT_QUERY_RESULTS_RECORDS
);
+ // Select query (simulating column types from metastore)
+ tbl.setProperty(serdeConstants.LIST_COLUMNS, SELECT_COLUMN_NAMES);
+ tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, SELECT_COLUMN_TYPES);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ deserializeQueryResults(serDe, Query.SELECT, SELECT_QUERY,
+ SELECT_QUERY_RESULTS, SELECT_QUERY_RESULTS_RECORDS_2
+ );
}
private static Properties createPropertiesQuery(String dataSource, String queryType,
@@ -543,6 +646,7 @@ public class TestDruidSerDe {
return tbl;
}
+ @SuppressWarnings("unchecked")
private static void deserializeQueryResults(DruidSerDe serDe, String queryType, String jsonQuery,
String resultString, Object[][] records
) throws SerDeException, JsonParseException,
@@ -612,10 +716,11 @@ public class TestDruidSerDe {
DruidWritable writable = new DruidWritable();
int pos = 0;
while (reader.next(NullWritable.get(), writable)) {
- Object row = serDe.deserialize(writable);
+ List<Object> row = (List<Object>) serDe.deserialize(writable);
Object[] expectedFieldsData = records[pos];
assertEquals(expectedFieldsData.length, fieldRefs.size());
for (int i = 0; i < fieldRefs.size(); i++) {
+ assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass());
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
assertEquals("Field " + i, expectedFieldsData[i], fieldData);
}
@@ -628,10 +733,11 @@ public class TestDruidSerDe {
field2.set(reader, results);
pos = 0;
while (reader.nextKeyValue()) {
- Object row = serDe.deserialize(reader.getCurrentValue());
+ List<Object> row = (List<Object>) serDe.deserialize(reader.getCurrentValue());
Object[] expectedFieldsData = records[pos];
assertEquals(expectedFieldsData.length, fieldRefs.size());
for (int i = 0; i < fieldRefs.size(); i++) {
+ assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass());
Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
assertEquals("Field " + i, expectedFieldsData[i], fieldData);
}
@@ -682,7 +788,7 @@ public class TestDruidSerDe {
* @throws NoSuchMethodException
*/
@Test
- public void testDruidSerializer()
+ public void testDruidObjectSerializer()
throws SerDeException, JsonParseException, JsonMappingException,
NoSuchFieldException, SecurityException, IllegalArgumentException,
IllegalAccessException, IOException, InterruptedException,
@@ -737,9 +843,62 @@ public class TestDruidSerDe {
// Serialize
DruidWritable writable = (DruidWritable) serDe.serialize(rowObject, inspector);
// Check result
- assertEquals(DRUID_WRITABLE.getValue().size(), writable.getValue().size());
- for (Entry<String, Object> e: DRUID_WRITABLE.getValue().entrySet()) {
+ assertEquals(druidWritable.getValue().size(), writable.getValue().size());
+ for (Entry<String, Object> e: druidWritable.getValue().entrySet()) {
assertEquals(e.getValue(), writable.getValue().get(e.getKey()));
}
}
+
+ private static final Object[] ROW_OBJECT_2 = new Object[] {
+ new TimestampWritable(new Timestamp(1377907200000L)),
+ new Text("dim1_val"),
+ new DoubleWritable(10669.3D),
+ new FloatWritable(10669.45F),
+ new HiveDecimalWritable(HiveDecimal.create(1064.34D)),
+ new LongWritable(1113939),
+ new IntWritable(1112123),
+ new ShortWritable((short) 12),
+ new ByteWritable((byte) 0)
+ };
+ private static final DruidWritable DRUID_WRITABLE_2 = new DruidWritable(
+ ImmutableMap.<String, Object>builder()
+ .put("__time", 1377907200000L)
+ .put("c0", "dim1_val")
+ .put("c1", 10669.3D)
+ .put("c2", 10669.45F)
+ .put("c3", 1064.34D)
+ .put("c4", 1113939L)
+ .put("c5", 1112123)
+ .put("c6", (short) 12)
+ .put("c7", (byte) 0)
+ .build());
+
+ @Test
+ public void testDruidObjectDeserializer()
+ throws SerDeException, JsonParseException, JsonMappingException,
+ NoSuchFieldException, SecurityException, IllegalArgumentException,
+ IllegalAccessException, IOException, InterruptedException,
+ NoSuchMethodException, InvocationTargetException {
+ // Create, initialize, and test the SerDe
+ DruidSerDe serDe = new DruidSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl;
+ // Mixed source (all types)
+ tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ deserializeObject(tbl, serDe, ROW_OBJECT_2, DRUID_WRITABLE_2);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void deserializeObject(Properties properties, DruidSerDe serDe,
+ Object[] rowObject, DruidWritable druidWritable) throws SerDeException {
+ // Deserialize
+ List<Object> object = (List<Object>) serDe.deserialize(druidWritable);
+ // Check result
+ assertEquals(rowObject.length, object.size());
+ for (int i = 0; i < rowObject.length; i++) {
+ assertEquals(rowObject[i].getClass(), object.get(i).getClass());
+ assertEquals(rowObject[i], object.get(i));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/727a3dfc/ql/src/test/queries/clientpositive/druid_basic2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druid_basic2.q b/ql/src/test/queries/clientpositive/druid_basic2.q
index 530e53a..3c17bc5 100644
--- a/ql/src/test/queries/clientpositive/druid_basic2.q
+++ b/ql/src/test/queries/clientpositive/druid_basic2.q
@@ -59,6 +59,10 @@ ORDER BY CAST(robot AS INTEGER) ASC, m DESC
LIMIT 10;
EXPLAIN
+SELECT substring(namespace, CAST(deleted AS INT), 4)
+FROM druid_table_1;
+
+EXPLAIN
SELECT robot, floor_day(`__time`)
FROM druid_table_1
WHERE floor_day(`__time`) BETWEEN '1999-11-01 00:00:00' AND '1999-11-10 00:00:00'
http://git-wip-us.apache.org/repos/asf/hive/blob/727a3dfc/ql/src/test/results/clientpositive/druid_basic2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_basic2.q.out b/ql/src/test/results/clientpositive/druid_basic2.q.out
index a3f05be..0d9e683 100644
--- a/ql/src/test/results/clientpositive/druid_basic2.q.out
+++ b/ql/src/test/results/clientpositive/druid_basic2.q.out
@@ -672,6 +672,46 @@ STAGE PLANS:
ListSink
PREHOOK: query: EXPLAIN
+SELECT substring(namespace, CAST(deleted AS INT), 4)
+FROM druid_table_1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT substring(namespace, CAST(deleted AS INT), 4)
+FROM druid_table_1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: druid_table_1
+ properties:
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["namespace"],"metrics":["deleted"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.type select
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: substring(namespace, UDFToInteger(deleted), 4) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
SELECT robot, floor_day(`__time`)
FROM druid_table_1
WHERE floor_day(`__time`) BETWEEN '1999-11-01 00:00:00' AND '1999-11-10 00:00:00'
[30/36] hive git commit: HIVE-16775 : Fix
HiveFilterAggregateTransposeRule when filter is always false (Pengcheng Xiong
via Jesus Camacho Rodriguez)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/results/clientpositive/perf/query4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query4.q.out b/ql/src/test/results/clientpositive/perf/query4.q.out
new file mode 100644
index 0000000..1b20486
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/query4.q.out
@@ -0,0 +1,1128 @@
+PREHOOK: query: explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total
+ ,'c' sale_type
+ from customer
+ ,catalog_sales
+ ,date_dim
+ where c_customer_sk = cs_bill_customer_sk
+ and cs_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ )
+ select t_s_secyear.customer_preferred_cust_flag
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_c_firstyear
+ ,year_total t_c_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_c_secyear.customer_id
+ and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_c_firstyear.sale_type = 'c'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_c_secyear.sale_type = 'c'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.dyear = 2001
+ and t_s_secyear.dyear = 2001+1
+ and t_c_firstyear.dyear = 2001
+ and t_c_secyear.dyear = 2001+1
+ and t_w_firstyear.dyear = 2001
+ and t_w_secyear.dyear = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_c_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ order by t_s_secyear.customer_preferred_cust_flag
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total
+ ,'c' sale_type
+ from customer
+ ,catalog_sales
+ ,date_dim
+ where c_customer_sk = cs_bill_customer_sk
+ and cs_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ )
+ select t_s_secyear.customer_preferred_cust_flag
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_c_firstyear
+ ,year_total t_c_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_c_secyear.customer_id
+ and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_c_firstyear.sale_type = 'c'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_c_secyear.sale_type = 'c'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.dyear = 2001
+ and t_s_secyear.dyear = 2001+1
+ and t_c_firstyear.dyear = 2001
+ and t_c_secyear.dyear = 2001+1
+ and t_w_firstyear.dyear = 2001
+ and t_w_secyear.dyear = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_c_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ order by t_s_secyear.customer_preferred_cust_flag
+limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 12 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 13 <- Map 65 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Union 15 (CONTAINS)
+Reducer 16 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 17 <- Map 65 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 19 (CONTAINS)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 20 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 21 <- Map 65 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 22 <- Reducer 21 (SIMPLE_EDGE), Union 23 (CONTAINS)
+Reducer 24 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 25 <- Map 65 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
+Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Union 27 (CONTAINS)
+Reducer 29 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 3 <- Map 65 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 30 <- Map 65 (SIMPLE_EDGE), Reducer 29 (SIMPLE_EDGE)
+Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 32 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 33 <- Map 65 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE)
+Reducer 34 <- Reducer 33 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 35 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 36 <- Map 65 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE)
+Reducer 37 <- Reducer 36 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 38 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 39 <- Map 65 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 40 <- Reducer 39 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 41 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 42 <- Map 65 (SIMPLE_EDGE), Reducer 41 (SIMPLE_EDGE)
+Reducer 43 <- Reducer 42 (SIMPLE_EDGE), Union 15 (CONTAINS)
+Reducer 44 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 45 <- Map 65 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE)
+Reducer 46 <- Reducer 45 (SIMPLE_EDGE), Union 15 (CONTAINS)
+Reducer 47 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 48 <- Map 65 (SIMPLE_EDGE), Reducer 47 (SIMPLE_EDGE)
+Reducer 49 <- Reducer 48 (SIMPLE_EDGE), Union 19 (CONTAINS)
+Reducer 50 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 51 <- Map 65 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE)
+Reducer 52 <- Reducer 51 (SIMPLE_EDGE), Union 19 (CONTAINS)
+Reducer 53 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 54 <- Map 65 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE)
+Reducer 55 <- Reducer 54 (SIMPLE_EDGE), Union 23 (CONTAINS)
+Reducer 56 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 57 <- Map 65 (SIMPLE_EDGE), Reducer 56 (SIMPLE_EDGE)
+Reducer 58 <- Reducer 57 (SIMPLE_EDGE), Union 23 (CONTAINS)
+Reducer 59 <- Map 28 (SIMPLE_EDGE), Map 66 (SIMPLE_EDGE)
+Reducer 6 <- Union 11 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE), Union 19 (SIMPLE_EDGE), Union 23 (SIMPLE_EDGE), Union 27 (SIMPLE_EDGE), Union 5 (SIMPLE_EDGE)
+Reducer 60 <- Map 65 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE)
+Reducer 61 <- Reducer 60 (SIMPLE_EDGE), Union 27 (CONTAINS)
+Reducer 62 <- Map 28 (SIMPLE_EDGE), Map 67 (SIMPLE_EDGE)
+Reducer 63 <- Map 65 (SIMPLE_EDGE), Reducer 62 (SIMPLE_EDGE)
+Reducer 64 <- Reducer 63 (SIMPLE_EDGE), Union 27 (CONTAINS)
+Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
+Reducer 9 <- Map 65 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:100
+ Stage-1
+ Reducer 7
+ File Output Operator [FS_412]
+ Limit [LIM_411] (rows=100 width=88)
+ Number of rows:100
+ Select Operator [SEL_410] (rows=479156402 width=88)
+ Output:["_col0"]
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_409]
+ Select Operator [SEL_408] (rows=479156402 width=88)
+ Output:["_col0"]
+ Filter Operator [FIL_406] (rows=479156402 width=88)
+ predicate:(CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col3 > 0)) THEN (((_col9 / _col3) > (_col12 / _col1))) ELSE ((null > (_col12 / _col1))) END) ELSE (CASE WHEN ((_col3 > 0)) THEN (((_col9 / _col3) > null)) ELSE (null) END) END and CASE WHEN ((_col5 > 0)) THEN (CASE WHEN ((_col3 > 0)) THEN (((_col9 / _col3) > (_col7 / _col5))) ELSE ((null > (_col7 / _col5))) END) ELSE (CASE WHEN ((_col3 > 0)) THEN (((_col9 / _col3) > null)) ELSE (null) END) END)
+ Merge Join Operator [MERGEJOIN_866] (rows=1916625609 width=88)
+ Conds:Union 5._col0=Union 11._col0(Inner),Union 5._col0=Union 15._col0(Inner),Union 5._col0=Union 19._col0(Inner),Union 5._col0=Union 23._col0(Inner),Union 5._col0=Union 27._col0(Inner),Output:["_col1","_col3","_col5","_col7","_col9","_col11","_col12"]
+ <-Union 11 [SIMPLE_EDGE]
+ <-Reducer 10 [CONTAINS]
+ Reduce Output Operator [RS_400]
+ PartitionCols:_col0
+ Select Operator [SEL_87] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_86] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_85]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_84] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_82] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_837] (rows=1 width=105)
+ Conds:RS_79._col1=RS_80._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_80]
+ PartitionCols:_col0
+ Select Operator [SEL_75] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_784] (rows=1 width=860)
+ predicate:false
+ TableScan [TS_6] (rows=80000000 width=860)
+ default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_country","c_login","c_email_address"]
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_79]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_836] (rows=1 width=96)
+ Conds:RS_76._col0=RS_77._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_77]
+ PartitionCols:_col0
+ Select Operator [SEL_72] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_783] (rows=1 width=1119)
+ predicate:false
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_76]
+ PartitionCols:_col0
+ Select Operator [SEL_69] (rows=1 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_782] (rows=1 width=88)
+ predicate:false
+ TableScan [TS_0] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price"]
+ <-Reducer 37 [CONTAINS]
+ Reduce Output Operator [RS_400]
+ PartitionCols:_col0
+ Select Operator [SEL_109] (rows=58077952 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_108] (rows=58077952 width=135)
+ predicate:(_col7 > 0)
+ Select Operator [SEL_827] (rows=174233858 width=135)
+ Output:["_col0","_col7"]
+ Group By Operator [GBY_107] (rows=174233858 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 36 [SIMPLE_EDGE]
+ SHUFFLE [RS_106]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_105] (rows=348467716 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_103] (rows=348467716 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_839] (rows=348467716 width=135)
+ Conds:RS_100._col1=RS_101._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_101]
+ PartitionCols:_col0
+ Select Operator [SEL_96] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_787] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 35 [SIMPLE_EDGE]
+ SHUFFLE [RS_100]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_838] (rows=316788826 width=135)
+ Conds:RS_97._col0=RS_98._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_98]
+ PartitionCols:_col0
+ Select Operator [SEL_93] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_786] (rows=36524 width=1119)
+ predicate:((d_year = 2001) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_97]
+ PartitionCols:_col0
+ Select Operator [SEL_90] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_785] (rows=287989836 width=135)
+ predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null)
+ TableScan [TS_22] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_ext_discount_amt","cs_ext_sales_price","cs_ext_wholesale_cost","cs_ext_list_price"]
+ <-Reducer 40 [CONTAINS]
+ Reduce Output Operator [RS_400]
+ PartitionCols:_col0
+ Select Operator [SEL_132] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_131] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 39 [SIMPLE_EDGE]
+ SHUFFLE [RS_130]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_129] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_127] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_841] (rows=1 width=162)
+ Conds:RS_124._col1=RS_125._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_125]
+ PartitionCols:_col0
+ Select Operator [SEL_120] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_790] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 38 [SIMPLE_EDGE]
+ SHUFFLE [RS_124]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_840] (rows=1 width=148)
+ Conds:RS_121._col0=RS_122._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_122]
+ PartitionCols:_col0
+ Select Operator [SEL_117] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_789] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_121]
+ PartitionCols:_col0
+ Select Operator [SEL_114] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_788] (rows=1 width=135)
+ predicate:false
+ TableScan [TS_45] (rows=144002668 width=135)
+ default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk","ws_ext_discount_amt","ws_ext_sales_price","ws_ext_wholesale_cost","ws_ext_list_price"]
+ <-Union 15 [SIMPLE_EDGE]
+ <-Reducer 14 [CONTAINS]
+ Reduce Output Operator [RS_401]
+ PartitionCols:_col0
+ Select Operator [SEL_154] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_153] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_152]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_151] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_149] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_843] (rows=1 width=105)
+ Conds:RS_146._col1=RS_147._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_147]
+ PartitionCols:_col0
+ Select Operator [SEL_142] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_793] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_146]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_842] (rows=1 width=96)
+ Conds:RS_143._col0=RS_144._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_144]
+ PartitionCols:_col0
+ Select Operator [SEL_139] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_792] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_143]
+ PartitionCols:_col0
+ Select Operator [SEL_136] (rows=1 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_791] (rows=1 width=88)
+ predicate:false
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 43 [CONTAINS]
+ Reduce Output Operator [RS_401]
+ PartitionCols:_col0
+ Select Operator [SEL_175] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_174] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 42 [SIMPLE_EDGE]
+ SHUFFLE [RS_173]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_172] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_170] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_845] (rows=1 width=162)
+ Conds:RS_167._col1=RS_168._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_168]
+ PartitionCols:_col0
+ Select Operator [SEL_163] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_796] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 41 [SIMPLE_EDGE]
+ SHUFFLE [RS_167]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_844] (rows=1 width=148)
+ Conds:RS_164._col0=RS_165._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_165]
+ PartitionCols:_col0
+ Select Operator [SEL_160] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_795] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_164]
+ PartitionCols:_col0
+ Select Operator [SEL_157] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_794] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_22]
+ <-Reducer 46 [CONTAINS]
+ Reduce Output Operator [RS_401]
+ PartitionCols:_col0
+ Select Operator [SEL_199] (rows=29040539 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_198] (rows=29040539 width=135)
+ predicate:(_col7 > 0)
+ Select Operator [SEL_828] (rows=87121617 width=135)
+ Output:["_col0","_col7"]
+ Group By Operator [GBY_197] (rows=87121617 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 45 [SIMPLE_EDGE]
+ SHUFFLE [RS_196]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_195] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_193] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_847] (rows=174243235 width=135)
+ Conds:RS_190._col1=RS_191._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_191]
+ PartitionCols:_col0
+ Select Operator [SEL_186] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_799] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 44 [SIMPLE_EDGE]
+ SHUFFLE [RS_190]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_846] (rows=158402938 width=135)
+ Conds:RS_187._col0=RS_188._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_188]
+ PartitionCols:_col0
+ Select Operator [SEL_183] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_798] (rows=36524 width=1119)
+ predicate:((d_year = 2001) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_187]
+ PartitionCols:_col0
+ Select Operator [SEL_180] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_797] (rows=144002668 width=135)
+ predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_45]
+ <-Union 19 [SIMPLE_EDGE]
+ <-Reducer 18 [CONTAINS]
+ Reduce Output Operator [RS_402]
+ PartitionCols:_col0
+ Select Operator [SEL_221] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_220] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_219]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_218] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_216] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_849] (rows=1 width=105)
+ Conds:RS_213._col1=RS_214._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_214]
+ PartitionCols:_col0
+ Select Operator [SEL_209] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_802] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_213]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_848] (rows=1 width=96)
+ Conds:RS_210._col0=RS_211._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_211]
+ PartitionCols:_col0
+ Select Operator [SEL_206] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_801] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_210]
+ PartitionCols:_col0
+ Select Operator [SEL_203] (rows=1 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_800] (rows=1 width=88)
+ predicate:false
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 49 [CONTAINS]
+ Reduce Output Operator [RS_402]
+ PartitionCols:_col0
+ Select Operator [SEL_242] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_241] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 48 [SIMPLE_EDGE]
+ SHUFFLE [RS_240]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_239] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_237] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_851] (rows=1 width=162)
+ Conds:RS_234._col1=RS_235._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_235]
+ PartitionCols:_col0
+ Select Operator [SEL_230] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_805] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 47 [SIMPLE_EDGE]
+ SHUFFLE [RS_234]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_850] (rows=1 width=148)
+ Conds:RS_231._col0=RS_232._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_232]
+ PartitionCols:_col0
+ Select Operator [SEL_227] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_804] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_231]
+ PartitionCols:_col0
+ Select Operator [SEL_224] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_803] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_22]
+ <-Reducer 52 [CONTAINS]
+ Reduce Output Operator [RS_402]
+ PartitionCols:_col0
+ Select Operator [SEL_265] (rows=87121617 width=135)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_264] (rows=87121617 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 51 [SIMPLE_EDGE]
+ SHUFFLE [RS_263]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_262] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_260] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_853] (rows=174243235 width=135)
+ Conds:RS_257._col1=RS_258._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_258]
+ PartitionCols:_col0
+ Select Operator [SEL_253] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_808] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 50 [SIMPLE_EDGE]
+ SHUFFLE [RS_257]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_852] (rows=158402938 width=135)
+ Conds:RS_254._col0=RS_255._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_255]
+ PartitionCols:_col0
+ Select Operator [SEL_250] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_807] (rows=36524 width=1119)
+ predicate:((d_year = 2002) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_254]
+ PartitionCols:_col0
+ Select Operator [SEL_247] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_806] (rows=144002668 width=135)
+ predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_45]
+ <-Union 23 [SIMPLE_EDGE]
+ <-Reducer 22 [CONTAINS]
+ Reduce Output Operator [RS_403]
+ PartitionCols:_col0
+ Select Operator [SEL_287] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_286] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_285]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_284] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_282] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_855] (rows=1 width=105)
+ Conds:RS_279._col1=RS_280._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_280]
+ PartitionCols:_col0
+ Select Operator [SEL_275] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_811] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_279]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_854] (rows=1 width=96)
+ Conds:RS_276._col0=RS_277._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_277]
+ PartitionCols:_col0
+ Select Operator [SEL_272] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_810] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_276]
+ PartitionCols:_col0
+ Select Operator [SEL_269] (rows=1 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_809] (rows=1 width=88)
+ predicate:false
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 55 [CONTAINS]
+ Reduce Output Operator [RS_403]
+ PartitionCols:_col0
+ Select Operator [SEL_308] (rows=174233858 width=135)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_307] (rows=174233858 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 54 [SIMPLE_EDGE]
+ SHUFFLE [RS_306]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_305] (rows=348467716 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_303] (rows=348467716 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_857] (rows=348467716 width=135)
+ Conds:RS_300._col1=RS_301._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_301]
+ PartitionCols:_col0
+ Select Operator [SEL_296] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_814] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 53 [SIMPLE_EDGE]
+ SHUFFLE [RS_300]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_856] (rows=316788826 width=135)
+ Conds:RS_297._col0=RS_298._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_298]
+ PartitionCols:_col0
+ Select Operator [SEL_293] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_813] (rows=36524 width=1119)
+ predicate:((d_year = 2002) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_297]
+ PartitionCols:_col0
+ Select Operator [SEL_290] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_812] (rows=287989836 width=135)
+ predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_22]
+ <-Reducer 58 [CONTAINS]
+ Reduce Output Operator [RS_403]
+ PartitionCols:_col0
+ Select Operator [SEL_331] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_330] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 57 [SIMPLE_EDGE]
+ SHUFFLE [RS_329]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_328] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_326] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_859] (rows=1 width=162)
+ Conds:RS_323._col1=RS_324._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_324]
+ PartitionCols:_col0
+ Select Operator [SEL_319] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_817] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 56 [SIMPLE_EDGE]
+ SHUFFLE [RS_323]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_858] (rows=1 width=148)
+ Conds:RS_320._col0=RS_321._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_321]
+ PartitionCols:_col0
+ Select Operator [SEL_316] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_816] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_320]
+ PartitionCols:_col0
+ Select Operator [SEL_313] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_815] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_45]
+ <-Union 27 [SIMPLE_EDGE]
+ <-Reducer 26 [CONTAINS]
+ Reduce Output Operator [RS_404]
+ PartitionCols:_col0
+ Select Operator [SEL_353] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2"]
+ Group By Operator [GBY_352] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 25 [SIMPLE_EDGE]
+ SHUFFLE [RS_351]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_350] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_348] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_861] (rows=696954748 width=88)
+ Conds:RS_345._col1=RS_346._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_346]
+ PartitionCols:_col0
+ Select Operator [SEL_341] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_820] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 24 [SIMPLE_EDGE]
+ SHUFFLE [RS_345]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_860] (rows=633595212 width=88)
+ Conds:RS_342._col0=RS_343._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_343]
+ PartitionCols:_col0
+ Select Operator [SEL_338] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_819] (rows=36524 width=1119)
+ predicate:((d_year = 2002) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_342]
+ PartitionCols:_col0
+ Select Operator [SEL_335] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_818] (rows=575995635 width=88)
+ predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 61 [CONTAINS]
+ Reduce Output Operator [RS_404]
+ PartitionCols:_col0
+ Select Operator [SEL_374] (rows=1 width=162)
+ Output:["_col0","_col1","_col2"]
+ Group By Operator [GBY_373] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 60 [SIMPLE_EDGE]
+ SHUFFLE [RS_372]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_371] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_369] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_863] (rows=1 width=162)
+ Conds:RS_366._col1=RS_367._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_367]
+ PartitionCols:_col0
+ Select Operator [SEL_362] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_823] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 59 [SIMPLE_EDGE]
+ SHUFFLE [RS_366]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_862] (rows=1 width=148)
+ Conds:RS_363._col0=RS_364._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_364]
+ PartitionCols:_col0
+ Select Operator [SEL_359] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_822] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_363]
+ PartitionCols:_col0
+ Select Operator [SEL_356] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_821] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_22]
+ <-Reducer 64 [CONTAINS]
+ Reduce Output Operator [RS_404]
+ PartitionCols:_col0
+ Select Operator [SEL_397] (rows=1 width=162)
+ Output:["_col0","_col1","_col2"]
+ Group By Operator [GBY_396] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 63 [SIMPLE_EDGE]
+ SHUFFLE [RS_395]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_394] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_392] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_865] (rows=1 width=162)
+ Conds:RS_389._col1=RS_390._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_390]
+ PartitionCols:_col0
+ Select Operator [SEL_385] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_826] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 62 [SIMPLE_EDGE]
+ SHUFFLE [RS_389]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_864] (rows=1 width=148)
+ Conds:RS_386._col0=RS_387._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_387]
+ PartitionCols:_col0
+ Select Operator [SEL_382] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_825] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_386]
+ PartitionCols:_col0
+ Select Operator [SEL_379] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_824] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_45]
+ <-Union 5 [SIMPLE_EDGE]
+ <-Reducer 31 [CONTAINS]
+ Reduce Output Operator [RS_399]
+ PartitionCols:_col0
+ Select Operator [SEL_42] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_41] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 30 [SIMPLE_EDGE]
+ SHUFFLE [RS_40]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_39] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_37] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_833] (rows=1 width=162)
+ Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_35]
+ PartitionCols:_col0
+ Select Operator [SEL_30] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_778] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 29 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_832] (rows=1 width=148)
+ Conds:RS_31._col0=RS_32._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_32]
+ PartitionCols:_col0
+ Select Operator [SEL_27] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_777] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 66 [SIMPLE_EDGE]
+ SHUFFLE [RS_31]
+ PartitionCols:_col0
+ Select Operator [SEL_24] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_776] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_22]
+ <-Reducer 34 [CONTAINS]
+ Reduce Output Operator [RS_399]
+ PartitionCols:_col0
+ Select Operator [SEL_65] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_64] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7
+ <-Reducer 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_63]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Group By Operator [GBY_62] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col8)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Select Operator [SEL_60] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_835] (rows=1 width=162)
+ Conds:RS_57._col1=RS_58._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
+ PartitionCols:_col0
+ Select Operator [SEL_53] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_781] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 32 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_834] (rows=1 width=148)
+ Conds:RS_54._col0=RS_55._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col7"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:_col0
+ Select Operator [SEL_50] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_780] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 67 [SIMPLE_EDGE]
+ SHUFFLE [RS_54]
+ PartitionCols:_col0
+ Select Operator [SEL_47] (rows=1 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_779] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_45]
+ <-Reducer 4 [CONTAINS]
+ Reduce Output Operator [RS_399]
+ PartitionCols:_col0
+ Select Operator [SEL_21] (rows=116159124 width=88)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_20] (rows=116159124 width=88)
+ predicate:(_col7 > 0)
+ Select Operator [SEL_829] (rows=348477374 width=88)
+ Output:["_col0","_col7"]
+ Group By Operator [GBY_19] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Group By Operator [GBY_17] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col7)"],keys:_col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Select Operator [SEL_15] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_831] (rows=696954748 width=88)
+ Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
+ <-Map 65 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_775] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_830] (rows=633595212 width=88)
+ Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_10]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_774] (rows=36524 width=1119)
+ predicate:((d_year = 2001) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_9]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_773] (rows=575995635 width=88)
+ predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+
[09/36] hive git commit: HIVE-16654: Optimize a combination of avg(),
sum(), count(distinct) etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/perf/query94.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query94.q.out b/ql/src/test/results/clientpositive/perf/query94.q.out
index 6e24345..c5fc9e7 100644
--- a/ql/src/test/results/clientpositive/perf/query94.q.out
+++ b/ql/src/test/results/clientpositive/perf/query94.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join MERGEJOIN[107][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 17' is a cross product
+Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: explain
select
count(distinct ws_order_number) as `order count`
@@ -58,174 +58,182 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 13 <- Map 12 (SIMPLE_EDGE)
-Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
-Reducer 16 <- Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Map 14 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE)
-Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
-Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 16 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+Reducer 14 <- Map 13 (SIMPLE_EDGE)
+Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
+Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE)
+Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
+Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 17 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 8
+ Reducer 9
File Output Operator [FS_74]
Limit [LIM_72] (rows=1 width=344)
Number of rows:100
Select Operator [SEL_71] (rows=1 width=344)
Output:["_col0","_col1","_col2"]
- <-Reducer 7 [SIMPLE_EDGE]
+ <-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_70]
Select Operator [SEL_69] (rows=1 width=344)
Output:["_col1","_col2","_col3"]
- Group By Operator [GBY_68] (rows=1 width=344)
- Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
- <-Reducer 6 [SIMPLE_EDGE]
- SHUFFLE [RS_67]
- Group By Operator [GBY_66] (rows=1395035081047425024 width=1)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4
- Select Operator [SEL_65] (rows=1395035081047425024 width=1)
- Output:["_col4","_col5","_col6"]
- Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
- predicate:_col16 is null
- Select Operator [SEL_63] (rows=2790070162094850048 width=1)
- Output:["_col4","_col5","_col6","_col16"]
- Merge Join Operator [MERGEJOIN_113] (rows=2790070162094850048 width=1)
- Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_61]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
- Output:["_col0","_col1"],keys:_col2, _col3
- Select Operator [SEL_43] (rows=5072854730221289472 width=1)
- Output:["_col2","_col3"]
- Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
- predicate:(_col2 <> _col0)
- Merge Join Operator [MERGEJOIN_111] (rows=5072854730221289472 width=1)
- Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
- <-Map 14 [SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_39]
- PartitionCols:_col1
- Select Operator [SEL_20] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- TableScan [TS_19] (rows=144002668 width=135)
- default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
- <-Reducer 18 [SIMPLE_EDGE]
- SHUFFLE [RS_40]
- PartitionCols:_col1
- Select Operator [SEL_38] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"]
- Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 17 [SIMPLE_EDGE]
- SHUFFLE [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
- Output:["_col0","_col1"],keys:_col4, _col3
- Merge Join Operator [MERGEJOIN_107] (rows=9223372036854775807 width=1)
- Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
- <-Map 14 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_32]
- Select Operator [SEL_28] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Please refer to the previous TableScan [TS_19]
- <-Map 19 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_29]
- Select Operator [SEL_22] (rows=73049 width=4)
- TableScan [TS_21] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
- <-Map 20 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_30]
- Select Operator [SEL_24] (rows=84 width=4)
- TableScan [TS_23] (rows=84 width=1850)
- default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE
- <-Map 21 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_31]
- Select Operator [SEL_26] (rows=40000000 width=4)
- TableScan [TS_25] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
- <-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col3, _col4
- Merge Join Operator [MERGEJOIN_112] (rows=210834322 width=135)
- Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_58]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=7199233 width=92)
- Output:["_col0","_col1"]
- Group By Operator [GBY_17] (rows=7199233 width=92)
- Output:["_col0"],keys:KEY._col0
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
+ Group By Operator [GBY_112] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
+ <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_111]
+ Group By Operator [GBY_110] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
+ Group By Operator [GBY_109] (rows=1395035081047425024 width=1)
+ Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_108]
+ PartitionCols:_col0
+ Group By Operator [GBY_107] (rows=1395035081047425024 width=1)
+ Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4
+ Select Operator [SEL_65] (rows=1395035081047425024 width=1)
+ Output:["_col4","_col5","_col6"]
+ Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
+ predicate:_col16 is null
+ Select Operator [SEL_63] (rows=2790070162094850048 width=1)
+ Output:["_col4","_col5","_col6","_col16"]
+ Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1)
+ Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_61]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
+ Output:["_col0","_col1"],keys:_col2, _col3
+ Select Operator [SEL_43] (rows=5072854730221289472 width=1)
+ Output:["_col2","_col3"]
+ Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
+ predicate:(_col2 <> _col0)
+ Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1)
+ Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
+ <-Map 15 [SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_39]
+ PartitionCols:_col1
+ Select Operator [SEL_20] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ TableScan [TS_19] (rows=144002668 width=135)
+ default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_40]
+ PartitionCols:_col1
+ Select Operator [SEL_38] (rows=4611686018427387903 width=1)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
+ Output:["_col0","_col1"],keys:_col4, _col3
+ Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1)
+ Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
+ <-Map 15 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_32]
+ Select Operator [SEL_28] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Please refer to the previous TableScan [TS_19]
+ <-Map 20 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_29]
+ Select Operator [SEL_22] (rows=73049 width=4)
+ TableScan [TS_21] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
+ <-Map 21 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_30]
+ Select Operator [SEL_24] (rows=84 width=4)
+ TableScan [TS_23] (rows=84 width=1850)
+ default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE
+ <-Map 22 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_31]
+ Select Operator [SEL_26] (rows=40000000 width=4)
+ TableScan [TS_25] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col3, _col4
+ Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135)
+ Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
PartitionCols:_col0
- Group By Operator [GBY_15] (rows=14398467 width=92)
- Output:["_col0"],keys:wr_order_number
- Filter Operator [FIL_104] (rows=14398467 width=92)
- predicate:wr_order_number is not null
- TableScan [TS_12] (rows=14398467 width=92)
- default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col4
- Merge Join Operator [MERGEJOIN_110] (rows=191667562 width=135)
- Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
- PartitionCols:_col0
- Select Operator [SEL_11] (rows=42 width=1850)
- Output:["_col0"]
- Filter Operator [FIL_103] (rows=42 width=1850)
- predicate:((web_company_name = 'pri') and web_site_sk is not null)
- TableScan [TS_9] (rows=84 width=1850)
- default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_54]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_109] (rows=174243235 width=135)
- Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_102] (rows=20000000 width=1014)
- predicate:((ca_state = 'TX') and ca_address_sk is not null)
- TableScan [TS_6] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135)
- Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
+ Select Operator [SEL_18] (rows=7199233 width=92)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_17] (rows=7199233 width=92)
+ Output:["_col0"],keys:KEY._col0
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Filter Operator [FIL_100] (rows=144002668 width=135)
- predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
- TableScan [TS_0] (rows=144002668 width=135)
- default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_101] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
- TableScan [TS_3] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+ Group By Operator [GBY_15] (rows=14398467 width=92)
+ Output:["_col0"],keys:wr_order_number
+ Filter Operator [FIL_104] (rows=14398467 width=92)
+ predicate:wr_order_number is not null
+ TableScan [TS_12] (rows=14398467 width=92)
+ default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col4
+ Merge Join Operator [MERGEJOIN_116] (rows=191667562 width=135)
+ Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:_col0
+ Select Operator [SEL_11] (rows=42 width=1850)
+ Output:["_col0"]
+ Filter Operator [FIL_103] (rows=42 width=1850)
+ predicate:((web_company_name = 'pri') and web_site_sk is not null)
+ TableScan [TS_9] (rows=84 width=1850)
+ default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_54]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_115] (rows=174243235 width=135)
+ Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_102] (rows=20000000 width=1014)
+ predicate:((ca_state = 'TX') and ca_address_sk is not null)
+ TableScan [TS_6] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_114] (rows=158402938 width=135)
+ Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_48]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Filter Operator [FIL_100] (rows=144002668 width=135)
+ predicate:(ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
+ TableScan [TS_0] (rows=144002668 width=135)
+ default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=8116 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_101] (rows=8116 width=1119)
+ predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/perf/query95.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query95.q.out b/ql/src/test/results/clientpositive/perf/query95.q.out
index 91d874e..332bef8 100644
--- a/ql/src/test/results/clientpositive/perf/query95.q.out
+++ b/ql/src/test/results/clientpositive/perf/query95.q.out
@@ -63,169 +63,177 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
-Reducer 14 <- Map 11 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
-Reducer 15 <- Map 18 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
-Reducer 16 <- Reducer 15 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE)
-Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
+Reducer 15 <- Map 12 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
+Reducer 16 <- Map 19 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
+Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 7
+ Reducer 8
File Output Operator [FS_71]
Limit [LIM_69] (rows=1 width=344)
Number of rows:100
Select Operator [SEL_68] (rows=1 width=344)
Output:["_col0","_col1","_col2"]
- <-Reducer 6 [SIMPLE_EDGE]
+ <-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_67]
Select Operator [SEL_66] (rows=1 width=344)
Output:["_col1","_col2","_col3"]
- Group By Operator [GBY_65] (rows=1 width=344)
- Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
- <-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_64]
- Group By Operator [GBY_63] (rows=421668645 width=135)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col3)","sum(_col4)","sum(_col5)"],keys:_col3
- Merge Join Operator [MERGEJOIN_121] (rows=421668645 width=135)
- Conds:RS_58._col3=RS_59._col0(Inner),RS_58._col3=RS_60._col0(Inner),Output:["_col3","_col4","_col5"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
+ Group By Operator [GBY_120] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
+ <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_119]
+ Group By Operator [GBY_118] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
+ Group By Operator [GBY_117] (rows=421668645 width=135)
+ Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_116]
PartitionCols:_col0
- Group By Operator [GBY_25] (rows=79201469 width=135)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_24]
- PartitionCols:_col0
- Group By Operator [GBY_23] (rows=158402938 width=135)
- Output:["_col0"],keys:_col1
- Select Operator [SEL_22] (rows=158402938 width=135)
- Output:["_col1"]
- Filter Operator [FIL_21] (rows=158402938 width=135)
- predicate:(_col0 <> _col2)
- Merge Join Operator [MERGEJOIN_118] (rows=158402938 width=135)
- Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col0","_col1","_col2"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
- PartitionCols:_col1
- Select Operator [SEL_14] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_110] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- TableScan [TS_12] (rows=144002668 width=135)
- default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
- <-Map 17 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col1
- Select Operator [SEL_17] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_111] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- TableScan [TS_15] (rows=144002668 width=135)
- default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col0
- Group By Operator [GBY_47] (rows=87121617 width=135)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_46]
- PartitionCols:_col0
- Group By Operator [GBY_45] (rows=174243235 width=135)
- Output:["_col0"],keys:_col1
- Merge Join Operator [MERGEJOIN_120] (rows=174243235 width=135)
- Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_42]
+ Group By Operator [GBY_115] (rows=421668645 width=135)
+ Output:["_col0","_col2","_col3"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col3
+ Merge Join Operator [MERGEJOIN_127] (rows=421668645 width=135)
+ Conds:RS_58._col3=RS_59._col0(Inner),RS_58._col3=RS_60._col0(Inner),Output:["_col3","_col4","_col5"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0
+ Group By Operator [GBY_25] (rows=79201469 width=135)
+ Output:["_col0"],keys:KEY._col0
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_24]
PartitionCols:_col0
- Select Operator [SEL_40] (rows=14398467 width=92)
- Output:["_col0"]
- Filter Operator [FIL_114] (rows=14398467 width=92)
- predicate:wr_order_number is not null
- TableScan [TS_38] (rows=14398467 width=92)
- default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_41]
+ Group By Operator [GBY_23] (rows=158402938 width=135)
+ Output:["_col0"],keys:_col1
+ Select Operator [SEL_22] (rows=158402938 width=135)
+ Output:["_col1"]
+ Filter Operator [FIL_21] (rows=158402938 width=135)
+ predicate:(_col0 <> _col2)
+ Merge Join Operator [MERGEJOIN_124] (rows=158402938 width=135)
+ Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col0","_col1","_col2"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
+ PartitionCols:_col1
+ Select Operator [SEL_14] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_110] (rows=144002668 width=135)
+ predicate:ws_order_number is not null
+ TableScan [TS_12] (rows=144002668 width=135)
+ default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
+ <-Map 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
+ PartitionCols:_col1
+ Select Operator [SEL_17] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_111] (rows=144002668 width=135)
+ predicate:ws_order_number is not null
+ TableScan [TS_15] (rows=144002668 width=135)
+ default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col0
+ Group By Operator [GBY_47] (rows=87121617 width=135)
+ Output:["_col0"],keys:KEY._col0
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_46]
+ PartitionCols:_col0
+ Group By Operator [GBY_45] (rows=174243235 width=135)
+ Output:["_col0"],keys:_col1
+ Merge Join Operator [MERGEJOIN_126] (rows=174243235 width=135)
+ Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"]
+ <-Map 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_42]
+ PartitionCols:_col0
+ Select Operator [SEL_40] (rows=14398467 width=92)
+ Output:["_col0"]
+ Filter Operator [FIL_114] (rows=14398467 width=92)
+ predicate:wr_order_number is not null
+ TableScan [TS_38] (rows=14398467 width=92)
+ default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_41]
+ PartitionCols:_col0
+ Select Operator [SEL_37] (rows=158402938 width=135)
+ Output:["_col0"]
+ Filter Operator [FIL_36] (rows=158402938 width=135)
+ predicate:(_col0 <> _col2)
+ Merge Join Operator [MERGEJOIN_125] (rows=158402938 width=135)
+ Conds:RS_33._col1=RS_34._col1(Inner),Output:["_col0","_col1","_col2"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_33]
+ PartitionCols:_col1
+ Select Operator [SEL_29] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_112] (rows=144002668 width=135)
+ predicate:ws_order_number is not null
+ Please refer to the previous TableScan [TS_12]
+ <-Map 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col1
+ Select Operator [SEL_32] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_113] (rows=144002668 width=135)
+ predicate:ws_order_number is not null
+ Please refer to the previous TableScan [TS_15]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
+ PartitionCols:_col3
+ Merge Join Operator [MERGEJOIN_123] (rows=191667562 width=135)
+ Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col3","_col4","_col5"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
PartitionCols:_col0
- Select Operator [SEL_37] (rows=158402938 width=135)
+ Select Operator [SEL_11] (rows=42 width=1850)
Output:["_col0"]
- Filter Operator [FIL_36] (rows=158402938 width=135)
- predicate:(_col0 <> _col2)
- Merge Join Operator [MERGEJOIN_119] (rows=158402938 width=135)
- Conds:RS_33._col1=RS_34._col1(Inner),Output:["_col0","_col1","_col2"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_33]
- PartitionCols:_col1
- Select Operator [SEL_29] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_112] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- Please refer to the previous TableScan [TS_12]
- <-Map 17 [SIMPLE_EDGE]
- SHUFFLE [RS_34]
- PartitionCols:_col1
- Select Operator [SEL_32] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_113] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- Please refer to the previous TableScan [TS_15]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_58]
- PartitionCols:_col3
- Merge Join Operator [MERGEJOIN_117] (rows=191667562 width=135)
- Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col3","_col4","_col5"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
- PartitionCols:_col0
- Select Operator [SEL_11] (rows=42 width=1850)
- Output:["_col0"]
- Filter Operator [FIL_109] (rows=42 width=1850)
- predicate:((web_company_name = 'pri') and web_site_sk is not null)
- TableScan [TS_9] (rows=84 width=1850)
- default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_116] (rows=174243235 width=135)
- Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col3","_col4","_col5"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_53]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_108] (rows=20000000 width=1014)
- predicate:((ca_state = 'TX') and ca_address_sk is not null)
- TableScan [TS_6] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_115] (rows=158402938 width=135)
- Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_106] (rows=144002668 width=135)
- predicate:(ws_order_number is not null and ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
- TableScan [TS_0] (rows=144002668 width=135)
- default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
- <-Map 8 [SIMPLE_EDGE]
- SHUFFLE [RS_50]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_107] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
- TableScan [TS_3] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+ Filter Operator [FIL_109] (rows=42 width=1850)
+ predicate:((web_company_name = 'pri') and web_site_sk is not null)
+ TableScan [TS_9] (rows=84 width=1850)
+ default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_122] (rows=174243235 width=135)
+ Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col2","_col3","_col4","_col5"]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_53]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_108] (rows=20000000 width=1014)
+ predicate:((ca_state = 'TX') and ca_address_sk is not null)
+ TableScan [TS_6] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_121] (rows=158402938 width=135)
+ Conds:RS_49._col0=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_106] (rows=144002668 width=135)
+ predicate:(ws_order_number is not null and ws_ship_date_sk is not null and ws_ship_addr_sk is not null and ws_web_site_sk is not null)
+ TableScan [TS_0] (rows=144002668 width=135)
+ default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
+ <-Map 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_50]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=8116 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_107] (rows=8116 width=1119)
+ predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/spark/nullgroup4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/nullgroup4.q.out b/ql/src/test/results/clientpositive/spark/nullgroup4.q.out
index 24f0291..63afe9b 100644
--- a/ql/src/test/results/clientpositive/spark/nullgroup4.q.out
+++ b/ql/src/test/results/clientpositive/spark/nullgroup4.q.out
@@ -95,7 +95,8 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -111,26 +112,44 @@ STAGE PLANS:
outputColumnNames: _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1), count(DISTINCT _col1)
+ aggregations: count(1)
keys: _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reducer 2
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0), count(DISTINCT KEY._col0:0._col0)
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/udf_count.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_count.q.out b/ql/src/test/results/clientpositive/udf_count.q.out
index f60ad04..c3903b2 100644
--- a/ql/src/test/results/clientpositive/udf_count.q.out
+++ b/ql/src/test/results/clientpositive/udf_count.q.out
@@ -43,7 +43,8 @@ POSTHOOK: query: EXPLAIN SELECT count(DISTINCT key) FROM src
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -57,24 +58,50 @@ STAGE PLANS:
outputColumnNames: key
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT key)
keys: key (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/vector_empty_where.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_empty_where.q.out b/ql/src/test/results/clientpositive/vector_empty_where.q.out
index b2dec6d..b7580f3 100644
--- a/ql/src/test/results/clientpositive/vector_empty_where.q.out
+++ b/ql/src/test/results/clientpositive/vector_empty_where.q.out
@@ -10,7 +10,8 @@ PLAN VECTORIZATION:
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -38,26 +39,25 @@ STAGE PLANS:
projectedOutputColumns: [2]
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT cint)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 2) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 2
native: false
- projectedOutputColumns: [0]
+ projectedOutputColumns: []
keys: cint (type: int)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Reduce Sink Vectorization:
className: VectorReduceSinkOperator
native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
@@ -74,17 +74,72 @@ STAGE PLANS:
enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: KEY._col0 (type: int)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0]
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
Group By Vectorization:
vectorOutput: false
native: false
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -117,7 +172,8 @@ PLAN VECTORIZATION:
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -137,26 +193,25 @@ STAGE PLANS:
predicate: cint (type: int)
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT cint)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 2) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 2
native: false
- projectedOutputColumns: [0]
+ projectedOutputColumns: []
keys: cint (type: int)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Reduce Sink Vectorization:
className: VectorReduceSinkOperator
native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
@@ -173,17 +228,72 @@ STAGE PLANS:
enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: KEY._col0 (type: int)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0]
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
Group By Vectorization:
vectorOutput: false
native: false
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -216,7 +326,8 @@ PLAN VECTORIZATION:
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -244,26 +355,25 @@ STAGE PLANS:
projectedOutputColumns: [2]
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT cint)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 2) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 2
native: false
- projectedOutputColumns: [0]
+ projectedOutputColumns: []
keys: cint (type: int)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Reduce Sink Vectorization:
className: VectorReduceSinkOperator
native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
@@ -280,17 +390,72 @@ STAGE PLANS:
enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: KEY._col0 (type: int)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0]
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
Group By Vectorization:
vectorOutput: false
native: false
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -323,7 +488,8 @@ PLAN VECTORIZATION:
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -351,26 +517,25 @@ STAGE PLANS:
projectedOutputColumns: [2]
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT cint)
Group By Vectorization:
- aggregators: VectorUDAFCount(col 2) -> bigint
className: VectorGroupByOperator
vectorOutput: true
keyExpressions: col 2
native: false
- projectedOutputColumns: [0]
+ projectedOutputColumns: []
keys: cint (type: int)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Reduce Sink Vectorization:
className: VectorReduceSinkOperator
native: false
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No DISTINCT columns IS false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
@@ -387,17 +552,72 @@ STAGE PLANS:
enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ keys: KEY._col0 (type: int)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ Group By Vectorization:
+ vectorOutput: false
+ native: false
+ projectedOutputColumns: null
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ TableScan Vectorization:
+ native: true
+ projectedOutputColumns: [0]
+ Reduce Output Operator
+ sort order:
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkOperator
+ native: false
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true
+ groupByVectorOutput: true
+ inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ Reduce Vectorization:
+ enabled: false
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true
+ enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
Group By Vectorization:
vectorOutput: false
native: false
projectedOutputColumns: null
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[26/36] hive git commit: Revert "HIVE-16744 : LLAP index update may
be broken after ORC switch (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)"
Posted by we...@apache.org.
Revert "HIVE-16744 : LLAP index update may be broken after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)"
This reverts commit 019ad8c822786b1bf195bcf0d4be94eec0449dda.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/38df8b4b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/38df8b4b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/38df8b4b
Branch: refs/heads/hive-14535
Commit: 38df8b4b8c624cb58b563a1eef43e7b417b9ab0a
Parents: 087aafc
Author: sergey <se...@apache.org>
Authored: Fri Jun 2 15:43:24 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Jun 2 15:43:24 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/38df8b4b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 7081140..9693826 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -394,7 +394,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}",
stripeKey.stripeIx, DebugUtils.toString(globalIncludes));
}
- assert isFoundInCache; // If it's not fresh from the cache, indexes should be present.
+ assert isFoundInCache;
counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
ensureMetadataReader();
updateLoadedIndexes(stripeMetadata, stripe, globalIncludes, sargColumns);
http://git-wip-us.apache.org/repos/asf/hive/blob/38df8b4b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index 96c4cce..1f3f7ea 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -113,7 +113,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
public void loadMissingIndexes(DataReader mr, StripeInformation stripe, boolean[] includes,
boolean[] sargColumns) throws IOException {
- // Do not lose the old indexes. Create a super set includes
+ // Do not loose the old indexes. Create a super set includes
OrcProto.RowIndex[] existing = getRowIndexes();
boolean superset[] = new boolean[Math.max(existing.length, includes.length)];
for (int i = 0; i < includes.length; i++) {
@@ -123,7 +123,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
superset[i] = superset[i] || (existing[i] != null);
}
// TODO: should we save footer to avoid a read here?
- rowIndex = mr.readRowIndex(stripe, schema, null, true, superset,
+ rowIndex = mr.readRowIndex(stripe, schema, null, true, includes,
rowIndex.getRowGroupIndex(),
sargColumns, writerVersion, rowIndex.getBloomFilterKinds(),
rowIndex.getBloomFilterIndex());
[07/36] hive git commit: HIVE-16782 : Flaky Test:
TestMiniLlapLocalCliDriver[subquery_scalar] (Vihang Karajgaonkar via Ashutosh
Chauhan)
Posted by we...@apache.org.
HIVE-16782 : Flaky Test: TestMiniLlapLocalCliDriver[subquery_scalar] (Vihang Karajgaonkar via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9e6c23d5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9e6c23d5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9e6c23d5
Branch: refs/heads/hive-14535
Commit: 9e6c23d58a443b06a1b744bf02d75d003a862324
Parents: cb815dc
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Wed May 31 13:30:49 2017 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed May 31 13:30:49 2017 -0700
----------------------------------------------------------------------
ql/src/test/results/clientpositive/llap/subquery_scalar.q.out | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9e6c23d5/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index 174492c..b78df8b 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -1514,7 +1514,7 @@ STAGE PLANS:
Reduce Operator Tree:
Merge Join Operator
condition map:
- Left Outer Join0 to 1
+ Left Outer Join 0 to 1
keys:
0 _col0 (type: string)
1 _col2 (type: string)
[22/36] hive git commit: HIVE-16566: Set column stats default as true
when creating new tables/partitions (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucketmapjoin5.q.out b/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
index febc2b6..4b98993 100644
--- a/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
+++ b/ql/src/test/results/clientpositive/bucketmapjoin5.q.out
@@ -251,7 +251,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -395,7 +395,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -432,7 +432,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -462,7 +462,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -483,7 +483,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -519,7 +519,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -549,7 +549,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -570,7 +570,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out b/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
index ee075c1..97cb1f1 100644
--- a/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
+++ b/ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out
@@ -191,7 +191,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -284,7 +284,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -321,7 +321,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -351,7 +351,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -372,7 +372,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -408,7 +408,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -438,7 +438,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -459,7 +459,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out b/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
index 2728bb7..724df73 100644
--- a/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
+++ b/ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out
@@ -254,7 +254,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -347,7 +347,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -384,7 +384,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -414,7 +414,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -435,7 +435,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -471,7 +471,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -501,7 +501,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -522,7 +522,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out b/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out
index bc9258f..fb833bc 100644
--- a/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out
+++ b/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out
@@ -25,7 +25,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\" left\":\"true\",\" middle \":\"true\",\"right \":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -51,7 +51,7 @@ POSTHOOK: Input: default@space
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
left string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\" left\":\"true\",\" middle \":\"true\",\"right \":\"true\"}}
PREHOOK: query: insert into space values ("1", "2", "3")
PREHOOK: type: QUERY
PREHOOK: Output: default@space
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/compustat_avro.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/compustat_avro.q.out b/ql/src/test/results/clientpositive/compustat_avro.q.out
index 016b8e7..2f8dc10 100644
--- a/ql/src/test/results/clientpositive/compustat_avro.q.out
+++ b/ql/src/test/results/clientpositive/compustat_avro.q.out
@@ -33,7 +33,7 @@ POSTHOOK: Input: default@testavro
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
col1 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\",\"col6\":\"true\"}}
PREHOOK: query: analyze table testAvro compute statistics for columns col1,col3
PREHOOK: type: QUERY
PREHOOK: Input: default@testavro
@@ -51,4 +51,4 @@ POSTHOOK: Input: default@testavro
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
col1 string 0 0 0.0 0 from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col3\":\"true\"}}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\",\"col4\":\"true\",\"col5\":\"true\",\"col6\":\"true\"}}
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out b/ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out
index 7d8e5c1..f661c5d 100644
--- a/ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out
+++ b/ql/src/test/results/clientpositive/create_alter_list_bucketing_table1.q.out
@@ -27,7 +27,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -74,7 +74,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -121,7 +121,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -175,7 +175,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -222,7 +222,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -268,7 +268,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -315,7 +315,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_like.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out
index 589ce59..a5c0c73 100644
--- a/ql/src/test/results/clientpositive/create_like.q.out
+++ b/ql/src/test/results/clientpositive/create_like.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -67,7 +67,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -118,7 +118,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
EXTERNAL TRUE
numFiles 0
numRows 0
@@ -354,7 +354,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\",\"number\":\"true\"}}
avro.schema.literal {
\"namespace\": \"testing.hive.avro.serde\",
\"name\": \"doctors\",
@@ -423,7 +423,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"first_name\":\"true\",\"last_name\":\"true\",\"number\":\"true\"}}
avro.schema.literal {
\"namespace\": \"testing.hive.avro.serde\",
\"name\": \"doctors\",
@@ -496,7 +496,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
parquet.compression LZO
@@ -540,7 +540,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -583,7 +583,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -693,7 +693,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"time\":\"true\"}}
numFiles 0
numRows 0
orc.compress SNAPPY
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_like2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_like2.q.out b/ql/src/test/results/clientpositive/create_like2.q.out
index 38bd499..157c3c9 100644
--- a/ql/src/test/results/clientpositive/create_like2.q.out
+++ b/ql/src/test/results/clientpositive/create_like2.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
a 1
c 3
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_like_tbl_props.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_like_tbl_props.q.out b/ql/src/test/results/clientpositive/create_like_tbl_props.q.out
index 19459fb..c242f3d 100644
--- a/ql/src/test/results/clientpositive/create_like_tbl_props.q.out
+++ b/ql/src/test/results/clientpositive/create_like_tbl_props.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
key value
numFiles 0
numRows 0
@@ -68,7 +68,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
key1 value1
numFiles 0
numRows 0
@@ -112,7 +112,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
key1 value1
key2 value2
numFiles 0
@@ -157,7 +157,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
key1 value1
key2 value3
numFiles 0
@@ -214,7 +214,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
key value
key1 value1
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_like_view.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_like_view.q.out b/ql/src/test/results/clientpositive/create_like_view.q.out
index 1e84b46..756934b 100644
--- a/ql/src/test/results/clientpositive/create_like_view.q.out
+++ b/ql/src/test/results/clientpositive/create_like_view.q.out
@@ -48,7 +48,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -111,7 +111,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -170,7 +170,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
EXTERNAL TRUE
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_skewed_table1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_skewed_table1.q.out b/ql/src/test/results/clientpositive/create_skewed_table1.q.out
index e4080b0..df927a7 100644
--- a/ql/src/test/results/clientpositive/create_skewed_table1.q.out
+++ b/ql/src/test/results/clientpositive/create_skewed_table1.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -77,7 +77,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -115,7 +115,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_table_like_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_table_like_stats.q.out b/ql/src/test/results/clientpositive/create_table_like_stats.q.out
index f5ec72c..bd559c0 100644
--- a/ql/src/test/results/clientpositive/create_table_like_stats.q.out
+++ b/ql/src/test/results/clientpositive/create_table_like_stats.q.out
@@ -28,7 +28,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/create_with_constraints.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/create_with_constraints.q.out b/ql/src/test/results/clientpositive/create_with_constraints.q.out
index 59b86b3..50a6d05 100644
--- a/ql/src/test/results/clientpositive/create_with_constraints.q.out
+++ b/ql/src/test/results/clientpositive/create_with_constraints.q.out
@@ -369,7 +369,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -411,7 +411,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -452,7 +452,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -506,7 +506,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\",\"y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -566,7 +566,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -614,7 +614,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\",\"y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -665,7 +665,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -707,7 +707,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -749,7 +749,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -791,7 +791,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -841,7 +841,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -890,7 +890,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -933,7 +933,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -976,7 +976,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1019,7 +1019,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1054,7 +1054,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1089,7 +1089,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1132,7 +1132,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1263,7 +1263,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1297,7 +1297,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1345,7 +1345,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\",\"y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1394,7 +1394,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\",\"y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1436,7 +1436,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1471,7 +1471,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1530,7 +1530,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1571,7 +1571,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1625,7 +1625,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x\":\"true\",\"y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1676,7 +1676,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -1712,7 +1712,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -1772,7 +1772,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -1819,7 +1819,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -1870,7 +1870,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -1952,7 +1952,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -2015,7 +2015,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -2062,7 +2062,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -2114,7 +2114,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/database_location.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/database_location.q.out b/ql/src/test/results/clientpositive/database_location.q.out
index 926db3a..e4db000 100644
--- a/ql/src/test/results/clientpositive/database_location.q.out
+++ b/ql/src/test/results/clientpositive/database_location.q.out
@@ -43,7 +43,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -118,7 +118,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/default_file_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/default_file_format.q.out b/ql/src/test/results/clientpositive/default_file_format.q.out
index 55401c0..9e8d5ba 100644
--- a/ql/src/test/results/clientpositive/default_file_format.q.out
+++ b/ql/src/test/results/clientpositive/default_file_format.q.out
@@ -69,7 +69,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -103,7 +103,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -137,7 +137,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -201,7 +201,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -369,7 +369,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -403,7 +403,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -437,7 +437,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -503,7 +503,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/describe_comment_indent.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/describe_comment_indent.q.out b/ql/src/test/results/clientpositive/describe_comment_indent.q.out
index 7b109b3..0049e2b 100644
--- a/ql/src/test/results/clientpositive/describe_comment_indent.q.out
+++ b/ql/src/test/results/clientpositive/describe_comment_indent.q.out
@@ -56,7 +56,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
comment table comment
two lines
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/describe_comment_nonascii.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/describe_comment_nonascii.q.out b/ql/src/test/results/clientpositive/describe_comment_nonascii.q.out
index 70f234e..4483e88 100644
--- a/ql/src/test/results/clientpositive/describe_comment_nonascii.q.out
+++ b/ql/src/test/results/clientpositive/describe_comment_nonascii.q.out
@@ -49,7 +49,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\",\"col3\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out b/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out
index c55239b..d47e325 100644
--- a/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out
+++ b/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out
@@ -57,7 +57,7 @@ Database: default
Table: view_partitioned
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/describe_syntax.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/describe_syntax.q.out b/ql/src/test/results/clientpositive/describe_syntax.q.out
index fe05b90..19147a1 100644
--- a/ql/src/test/results/clientpositive/describe_syntax.q.out
+++ b/ql/src/test/results/clientpositive/describe_syntax.q.out
@@ -320,7 +320,7 @@ Database: db1
Table: t1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key1\":\"true\",\"value1\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -394,7 +394,7 @@ Database: db1
Table: t1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key1\":\"true\",\"value1\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/describe_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/describe_table.q.out b/ql/src/test/results/clientpositive/describe_table.q.out
index 93b2030..7869494 100644
--- a/ql/src/test/results/clientpositive/describe_table.q.out
+++ b/ql/src/test/results/clientpositive/describe_table.q.out
@@ -572,7 +572,7 @@ POSTHOOK: query: DESCRIBE name1.name2 name3
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name3 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: DESCRIBE name1.name2 name4
PREHOOK: type: DESCTABLE
PREHOOK: Input: name1@name2
@@ -580,7 +580,7 @@ POSTHOOK: query: DESCRIBE name1.name2 name4
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name4 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: use name1
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:name1
@@ -672,7 +672,7 @@ POSTHOOK: query: DESCRIBE name2 name3
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name3 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: DESCRIBE name2 name4
PREHOOK: type: DESCTABLE
PREHOOK: Input: name1@name2
@@ -680,7 +680,7 @@ POSTHOOK: query: DESCRIBE name2 name4
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name4 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: DESCRIBE name1.name2
PREHOOK: type: DESCTABLE
PREHOOK: Input: name1@name2
@@ -696,7 +696,7 @@ POSTHOOK: query: DESCRIBE name1.name2 name3
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name3 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: DESCRIBE name1.name2 name4
PREHOOK: type: DESCTABLE
PREHOOK: Input: name1@name2
@@ -704,7 +704,7 @@ POSTHOOK: query: DESCRIBE name1.name2 name4
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name1@name2
name4 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"name3\":\"true\",\"name4\":\"true\"}}
PREHOOK: query: DESCRIBE name2.table1
PREHOOK: type: DESCTABLE
PREHOOK: Input: name2@table1
@@ -720,7 +720,7 @@ POSTHOOK: query: DESCRIBE name2.table1 col1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col1 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: DESCRIBE name2.table1 col2
PREHOOK: type: DESCTABLE
PREHOOK: Input: name2@table1
@@ -728,7 +728,7 @@ POSTHOOK: query: DESCRIBE name2.table1 col2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col2 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: use name2
PREHOOK: type: SWITCHDATABASE
PREHOOK: Input: database:name2
@@ -750,7 +750,7 @@ POSTHOOK: query: DESCRIBE table1 col1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col1 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: DESCRIBE table1 col2
PREHOOK: type: DESCTABLE
PREHOOK: Input: name2@table1
@@ -758,7 +758,7 @@ POSTHOOK: query: DESCRIBE table1 col2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col2 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: DESCRIBE name2.table1
PREHOOK: type: DESCTABLE
PREHOOK: Input: name2@table1
@@ -774,7 +774,7 @@ POSTHOOK: query: DESCRIBE name2.table1 col1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col1 int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: DESCRIBE name2.table1 col2
PREHOOK: type: DESCTABLE
PREHOOK: Input: name2@table1
@@ -782,7 +782,7 @@ POSTHOOK: query: DESCRIBE name2.table1 col2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: name2@table1
col2 string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\",\"col2\":\"true\"}}
PREHOOK: query: DROP TABLE IF EXISTS table1
PREHOOK: type: DROPTABLE
PREHOOK: Input: name2@table1
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out b/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
index 7b49e4a..a4b18d7 100644
--- a/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
+++ b/ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out
@@ -295,7 +295,7 @@ POSTHOOK: Input: default@empty_tab
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
a int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\"}}
PREHOOK: query: explain
analyze table empty_tab compute statistics for columns a,b,c,d,e
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/druid_basic1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_basic1.q.out b/ql/src/test/results/clientpositive/druid_basic1.q.out
index c16f81e..0ba61e5 100644
--- a/ql/src/test/results/clientpositive/druid_basic1.q.out
+++ b/ql/src/test/results/clientpositive/druid_basic1.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
@@ -102,7 +102,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/druid_basic2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_basic2.q.out b/ql/src/test/results/clientpositive/druid_basic2.q.out
index 0d9e683..38b07be 100644
--- a/ql/src/test/results/clientpositive/druid_basic2.q.out
+++ b/ql/src/test/results/clientpositive/druid_basic2.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
@@ -266,7 +266,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -292,7 +292,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -434,7 +434,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -460,7 +460,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -581,7 +581,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -607,7 +607,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -1022,7 +1022,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
@@ -1046,7 +1046,7 @@ STAGE PLANS:
input format: org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat
output format: org.apache.hadoop.hive.druid.io.DruidOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"__time":"true","added":"true","anonymous":"true","count":"true","deleted":"true","delta":"true","language":"true","namespace":"true","newpage":"true","page":"true","robot":"true","unpatrolled":"true","user":"true","variation":"true"}}
EXTERNAL TRUE
bucket_count -1
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/druid_intervals.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_intervals.q.out b/ql/src/test/results/clientpositive/druid_intervals.q.out
index db8633e..90013c6 100644
--- a/ql/src/test/results/clientpositive/druid_intervals.q.out
+++ b/ql/src/test/results/clientpositive/druid_intervals.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/druid_timeseries.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_timeseries.q.out b/ql/src/test/results/clientpositive/druid_timeseries.q.out
index ef3f8dd..bd4d6de 100644
--- a/ql/src/test/results/clientpositive/druid_timeseries.q.out
+++ b/ql/src/test/results/clientpositive/druid_timeseries.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/druid_topn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_topn.q.out b/ql/src/test/results/clientpositive/druid_topn.q.out
index 323eda2..5fa6446 100644
--- a/ql/src/test/results/clientpositive/druid_topn.q.out
+++ b/ql/src/test/results/clientpositive/druid_topn.q.out
@@ -40,7 +40,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"__time\":\"true\",\"added\":\"true\",\"anonymous\":\"true\",\"count\":\"true\",\"deleted\":\"true\",\"delta\":\"true\",\"language\":\"true\",\"namespace\":\"true\",\"newpage\":\"true\",\"page\":\"true\",\"robot\":\"true\",\"unpatrolled\":\"true\",\"user\":\"true\",\"variation\":\"true\"}}
EXTERNAL TRUE
druid.datasource wikipedia
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_map_ppr.q.out b/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
index 9be41df..0e5b394 100644
--- a/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
+++ b/ql/src/test/results/clientpositive/groupby_map_ppr.q.out
@@ -177,7 +177,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
@@ -208,7 +208,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out b/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
index 606ab4e..dbcef22 100644
--- a/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
@@ -177,7 +177,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
@@ -208,7 +208,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_ppr.q.out b/ql/src/test/results/clientpositive/groupby_ppr.q.out
index 8a75c2e..e645f5f 100644
--- a/ql/src/test/results/clientpositive/groupby_ppr.q.out
+++ b/ql/src/test/results/clientpositive/groupby_ppr.q.out
@@ -170,7 +170,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
@@ -201,7 +201,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out b/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
index 68d91eb..f300095 100644
--- a/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out
@@ -170,7 +170,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
@@ -201,7 +201,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
[10/36] hive git commit: HIVE-16654: Optimize a combination of avg(),
sum(), count(distinct) etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/llap/count_dist_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/count_dist_rewrite.q.out b/ql/src/test/results/clientpositive/llap/count_dist_rewrite.q.out
new file mode 100644
index 0000000..844c833
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/count_dist_rewrite.q.out
@@ -0,0 +1,1169 @@
+PREHOOK: query: explain select count(distinct key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(distinct key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col0)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(distinct key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(distinct key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+309
+PREHOOK: query: explain select max(key), count(distinct key) B1_CNTD from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key) B1_CNTD from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 205 Data size: 55555 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 55555 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 205 Data size: 55555 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key) B1_CNTD from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key) B1_CNTD from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309
+PREHOOK: query: explain select max(key), count(distinct key), min(key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(key), min(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 205 Data size: 93275 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 93275 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col3 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), min(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 205 Data size: 93275 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(_col1), count(_col0), min(_col2)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1), min(VALUE._col2)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 376 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309 0
+PREHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(key), min(key), avg(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 205 Data size: 145755 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 145755 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: struct<count:bigint,sum:double,input:string>)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), min(VALUE._col1), avg(VALUE._col2)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 205 Data size: 145755 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(_col1), count(_col0), min(_col2), avg(_col3)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 632 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 632 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: struct<count:bigint,sum:double,input:string>)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309 0 260.182
+PREHOOK: query: explain select count(1), count(distinct key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(distinct key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(1)
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(distinct key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(distinct key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 309
+PREHOOK: query: explain select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(), count(key), max(value), max(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5
+ Statistics: Num rows: 205 Data size: 96555 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 96555 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint), _col4 (type: string), _col5 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), max(VALUE._col2), max(VALUE._col3)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 205 Data size: 96555 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col1), count(_col2), count(_col0), max(_col3), max(_col4)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: string)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), max(VALUE._col3), max(VALUE._col4)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 500 309 val_98 98
+PREHOOK: query: explain select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(1), stddev(_col1)
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 205 Data size: 35875 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 205 Data size: 35875 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint), _col3 (type: struct<count:bigint,sum:double,variance:double>)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), stddev(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 205 Data size: 35875 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(_col1), count(_col0), stddev(_col2)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), stddev(VALUE._col2)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: bigint), _col1 (type: bigint), UDFToInteger(_col2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 309 142
+PREHOOK: query: select count(distinct key), count(1), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(distinct key), count(1), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+309 500 142
+PREHOOK: query: explain SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: substr(value, 5) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col0), avg(_col0), max(_col0), min(_col0), std(_col0), stddev_samp(_col0), variance(_col0), var_samp(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 214 Data size: 243104 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 214 Data size: 243104 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,input:string>), _col4 (type: string), _col5 (type: string), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>), _col9 (type: struct<count:bigint,sum:double,variance:double>)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), avg(VALUE._col1), max(VALUE._col2), min(VALUE._col3), std(VALUE._col4), stddev_samp(VALUE._col5), variance(VALUE._col6), var_samp(VALUE._col7)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 214 Data size: 243104 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: sum(_col1), avg(_col2), count(_col0), max(_col3), min(_col4), std(_col5), stddev_samp(_col6), variance(_col7), var_samp(_col8)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 960 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: bigint), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: double), _col2 (type: bigint), _col3 (type: string), _col4 (type: string), UDFToInteger(_col5) (type: int), UDFToInteger(_col6) (type: int), UDFToInteger(_col7) (type: int), UDFToInteger(_col8) (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 408 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+130091.0 260.182 309 98 0 142 143 20428 20469
+PREHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: max(key), count(DISTINCT key), min(key), avg(key)
+ keys: value (type: string), key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 250 Data size: 202500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 202500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,input:string>)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(DISTINCT KEY._col1:0._col0), min(VALUE._col2), avg(VALUE._col3)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 214 Data size: 101650 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: string), _col4 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 214 Data size: 82176 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 214 Data size: 82176 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 1 0 0.0
+10 1 10 10.0
+100 1 100 100.0
+103 1 103 103.0
+104 1 104 104.0
+105 1 105 105.0
+11 1 11 11.0
+111 1 111 111.0
+113 1 113 113.0
+114 1 114 114.0
+116 1 116 116.0
+118 1 118 118.0
+119 1 119 119.0
+12 1 12 12.0
+120 1 120 120.0
+125 1 125 125.0
+126 1 126 126.0
+128 1 128 128.0
+129 1 129 129.0
+131 1 131 131.0
+133 1 133 133.0
+134 1 134 134.0
+136 1 136 136.0
+137 1 137 137.0
+138 1 138 138.0
+143 1 143 143.0
+145 1 145 145.0
+146 1 146 146.0
+149 1 149 149.0
+15 1 15 15.0
+150 1 150 150.0
+152 1 152 152.0
+153 1 153 153.0
+155 1 155 155.0
+156 1 156 156.0
+157 1 157 157.0
+158 1 158 158.0
+160 1 160 160.0
+162 1 162 162.0
+163 1 163 163.0
+164 1 164 164.0
+165 1 165 165.0
+166 1 166 166.0
+167 1 167 167.0
+168 1 168 168.0
+169 1 169 169.0
+17 1 17 17.0
+170 1 170 170.0
+172 1 172 172.0
+174 1 174 174.0
+175 1 175 175.0
+176 1 176 176.0
+177 1 177 177.0
+178 1 178 178.0
+179 1 179 179.0
+18 1 18 18.0
+180 1 180 180.0
+181 1 181 181.0
+183 1 183 183.0
+186 1 186 186.0
+187 1 187 187.0
+189 1 189 189.0
+19 1 19 19.0
+190 1 190 190.0
+191 1 191 191.0
+192 1 192 192.0
+193 1 193 193.0
+194 1 194 194.0
+195 1 195 195.0
+196 1 196 196.0
+197 1 197 197.0
+199 1 199 199.0
+2 1 2 2.0
+20 1 20 20.0
+200 1 200 200.0
+201 1 201 201.0
+202 1 202 202.0
+203 1 203 203.0
+205 1 205 205.0
+207 1 207 207.0
+208 1 208 208.0
+209 1 209 209.0
+213 1 213 213.0
+214 1 214 214.0
+216 1 216 216.0
+217 1 217 217.0
+218 1 218 218.0
+219 1 219 219.0
+221 1 221 221.0
+222 1 222 222.0
+223 1 223 223.0
+224 1 224 224.0
+226 1 226 226.0
+228 1 228 228.0
+229 1 229 229.0
+230 1 230 230.0
+233 1 233 233.0
+235 1 235 235.0
+237 1 237 237.0
+238 1 238 238.0
+239 1 239 239.0
+24 1 24 24.0
+241 1 241 241.0
+242 1 242 242.0
+244 1 244 244.0
+247 1 247 247.0
+248 1 248 248.0
+249 1 249 249.0
+252 1 252 252.0
+255 1 255 255.0
+256 1 256 256.0
+257 1 257 257.0
+258 1 258 258.0
+26 1 26 26.0
+260 1 260 260.0
+262 1 262 262.0
+263 1 263 263.0
+265 1 265 265.0
+266 1 266 266.0
+27 1 27 27.0
+272 1 272 272.0
+273 1 273 273.0
+274 1 274 274.0
+275 1 275 275.0
+277 1 277 277.0
+278 1 278 278.0
+28 1 28 28.0
+280 1 280 280.0
+281 1 281 281.0
+282 1 282 282.0
+283 1 283 283.0
+284 1 284 284.0
+285 1 285 285.0
+286 1 286 286.0
+287 1 287 287.0
+288 1 288 288.0
+289 1 289 289.0
+291 1 291 291.0
+292 1 292 292.0
+296 1 296 296.0
+298 1 298 298.0
+30 1 30 30.0
+302 1 302 302.0
+305 1 305 305.0
+306 1 306 306.0
+307 1 307 307.0
+308 1 308 308.0
+309 1 309 309.0
+310 1 310 310.0
+311 1 311 311.0
+315 1 315 315.0
+316 1 316 316.0
+317 1 317 317.0
+318 1 318 318.0
+321 1 321 321.0
+322 1 322 322.0
+323 1 323 323.0
+325 1 325 325.0
+327 1 327 327.0
+33 1 33 33.0
+331 1 331 331.0
+332 1 332 332.0
+333 1 333 333.0
+335 1 335 335.0
+336 1 336 336.0
+338 1 338 338.0
+339 1 339 339.0
+34 1 34 34.0
+341 1 341 341.0
+342 1 342 342.0
+344 1 344 344.0
+345 1 345 345.0
+348 1 348 348.0
+35 1 35 35.0
+351 1 351 351.0
+353 1 353 353.0
+356 1 356 356.0
+360 1 360 360.0
+362 1 362 362.0
+364 1 364 364.0
+365 1 365 365.0
+366 1 366 366.0
+367 1 367 367.0
+368 1 368 368.0
+369 1 369 369.0
+37 1 37 37.0
+373 1 373 373.0
+374 1 374 374.0
+375 1 375 375.0
+377 1 377 377.0
+378 1 378 378.0
+379 1 379 379.0
+382 1 382 382.0
+384 1 384 384.0
+386 1 386 386.0
+389 1 389 389.0
+392 1 392 392.0
+393 1 393 393.0
+394 1 394 394.0
+395 1 395 395.0
+396 1 396 396.0
+397 1 397 397.0
+399 1 399 399.0
+4 1 4 4.0
+400 1 400 400.0
+401 1 401 401.0
+402 1 402 402.0
+403 1 403 403.0
+404 1 404 404.0
+406 1 406 406.0
+407 1 407 407.0
+409 1 409 409.0
+41 1 41 41.0
+411 1 411 411.0
+413 1 413 413.0
+414 1 414 414.0
+417 1 417 417.0
+418 1 418 418.0
+419 1 419 419.0
+42 1 42 42.0
+421 1 421 421.0
+424 1 424 424.0
+427 1 427 427.0
+429 1 429 429.0
+43 1 43 43.0
+430 1 430 430.0
+431 1 431 431.0
+432 1 432 432.0
+435 1 435 435.0
+436 1 436 436.0
+437 1 437 437.0
+438 1 438 438.0
+439 1 439 439.0
+44 1 44 44.0
+443 1 443 443.0
+444 1 444 444.0
+446 1 446 446.0
+448 1 448 448.0
+449 1 449 449.0
+452 1 452 452.0
+453 1 453 453.0
+454 1 454 454.0
+455 1 455 455.0
+457 1 457 457.0
+458 1 458 458.0
+459 1 459 459.0
+460 1 460 460.0
+462 1 462 462.0
+463 1 463 463.0
+466 1 466 466.0
+467 1 467 467.0
+468 1 468 468.0
+469 1 469 469.0
+47 1 47 47.0
+470 1 470 470.0
+472 1 472 472.0
+475 1 475 475.0
+477 1 477 477.0
+478 1 478 478.0
+479 1 479 479.0
+480 1 480 480.0
+481 1 481 481.0
+482 1 482 482.0
+483 1 483 483.0
+484 1 484 484.0
+485 1 485 485.0
+487 1 487 487.0
+489 1 489 489.0
+490 1 490 490.0
+491 1 491 491.0
+492 1 492 492.0
+493 1 493 493.0
+494 1 494 494.0
+495 1 495 495.0
+496 1 496 496.0
+497 1 497 497.0
+498 1 498 498.0
+5 1 5 5.0
+51 1 51 51.0
+53 1 53 53.0
+54 1 54 54.0
+57 1 57 57.0
+58 1 58 58.0
+64 1 64 64.0
+65 1 65 65.0
+66 1 66 66.0
+67 1 67 67.0
+69 1 69 69.0
+70 1 70 70.0
+72 1 72 72.0
+74 1 74 74.0
+76 1 76 76.0
+77 1 77 77.0
+78 1 78 78.0
+8 1 8 8.0
+80 1 80 80.0
+82 1 82 82.0
+83 1 83 83.0
+84 1 84 84.0
+85 1 85 85.0
+86 1 86 86.0
+87 1 87 87.0
+9 1 9 9.0
+90 1 90 90.0
+92 1 92 92.0
+95 1 95 95.0
+96 1 96 96.0
+97 1 97 97.0
+98 1 98 98.0
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/nullgroup4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nullgroup4.q.out b/ql/src/test/results/clientpositive/nullgroup4.q.out
index e5a8eee..d4c8e6a 100644
--- a/ql/src/test/results/clientpositive/nullgroup4.q.out
+++ b/ql/src/test/results/clientpositive/nullgroup4.q.out
@@ -93,7 +93,8 @@ select count(1), count(distinct x.value) from src x where x.key = 9999
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -110,25 +111,53 @@ STAGE PLANS:
outputColumnNames: _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1), count(DISTINCT _col1)
+ aggregations: count(1)
keys: _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0), count(DISTINCT KEY._col0:0._col0)
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/perf/query16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query16.q.out b/ql/src/test/results/clientpositive/perf/query16.q.out
index 449b9c8..a7f93f9 100644
--- a/ql/src/test/results/clientpositive/perf/query16.q.out
+++ b/ql/src/test/results/clientpositive/perf/query16.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join MERGEJOIN[107][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 17' is a cross product
+Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: explain
select
count(distinct cs_order_number) as `order count`
@@ -62,174 +62,182 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 13 <- Map 12 (SIMPLE_EDGE)
-Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
-Reducer 16 <- Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Map 14 (CUSTOM_SIMPLE_EDGE), Map 19 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE)
-Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
-Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 16 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+Reducer 14 <- Map 13 (SIMPLE_EDGE)
+Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
+Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE)
+Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
+Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 17 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 8
+ Reducer 9
File Output Operator [FS_74]
Limit [LIM_72] (rows=1 width=344)
Number of rows:100
Select Operator [SEL_71] (rows=1 width=344)
Output:["_col0","_col1","_col2"]
- <-Reducer 7 [SIMPLE_EDGE]
+ <-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_70]
Select Operator [SEL_69] (rows=1 width=344)
Output:["_col1","_col2","_col3"]
- Group By Operator [GBY_68] (rows=1 width=344)
- Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col0:0._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
- <-Reducer 6 [SIMPLE_EDGE]
- SHUFFLE [RS_67]
- Group By Operator [GBY_66] (rows=1395035081047425024 width=1)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT _col4)","sum(_col5)","sum(_col6)"],keys:_col4
- Select Operator [SEL_65] (rows=1395035081047425024 width=1)
- Output:["_col4","_col5","_col6"]
- Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
- predicate:_col16 is null
- Select Operator [SEL_63] (rows=2790070162094850048 width=1)
- Output:["_col4","_col5","_col6","_col16"]
- Merge Join Operator [MERGEJOIN_113] (rows=2790070162094850048 width=1)
- Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_61]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
- Output:["_col0","_col1"],keys:_col2, _col3
- Select Operator [SEL_43] (rows=5072854730221289472 width=1)
- Output:["_col2","_col3"]
- Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
- predicate:(_col2 <> _col0)
- Merge Join Operator [MERGEJOIN_111] (rows=5072854730221289472 width=1)
- Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
- <-Map 14 [SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_39]
- PartitionCols:_col1
- Select Operator [SEL_20] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- TableScan [TS_19] (rows=287989836 width=135)
- default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"]
- <-Reducer 18 [SIMPLE_EDGE]
- SHUFFLE [RS_40]
- PartitionCols:_col1
- Select Operator [SEL_38] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"]
- Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 17 [SIMPLE_EDGE]
- SHUFFLE [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
- Output:["_col0","_col1"],keys:_col4, _col3
- Merge Join Operator [MERGEJOIN_107] (rows=9223372036854775807 width=1)
- Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
- <-Map 14 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_32]
- Select Operator [SEL_28] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Please refer to the previous TableScan [TS_19]
- <-Map 19 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_29]
- Select Operator [SEL_22] (rows=73049 width=4)
- TableScan [TS_21] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
- <-Map 20 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_30]
- Select Operator [SEL_24] (rows=60 width=4)
- TableScan [TS_23] (rows=60 width=2045)
- default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE
- <-Map 21 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_31]
- Select Operator [SEL_26] (rows=40000000 width=4)
- TableScan [TS_25] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
- <-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col3, _col4
- Merge Join Operator [MERGEJOIN_112] (rows=421645953 width=135)
- Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_58]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=14399440 width=106)
- Output:["_col0","_col1"]
- Group By Operator [GBY_17] (rows=14399440 width=106)
- Output:["_col0"],keys:KEY._col0
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
+ Group By Operator [GBY_112] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
+ <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_111]
+ Group By Operator [GBY_110] (rows=1 width=344)
+ Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
+ Group By Operator [GBY_109] (rows=1395035081047425024 width=1)
+ Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_108]
+ PartitionCols:_col0
+ Group By Operator [GBY_107] (rows=1395035081047425024 width=1)
+ Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4
+ Select Operator [SEL_65] (rows=1395035081047425024 width=1)
+ Output:["_col4","_col5","_col6"]
+ Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
+ predicate:_col16 is null
+ Select Operator [SEL_63] (rows=2790070162094850048 width=1)
+ Output:["_col4","_col5","_col6","_col16"]
+ Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1)
+ Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_61]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
+ Output:["_col0","_col1"],keys:_col2, _col3
+ Select Operator [SEL_43] (rows=5072854730221289472 width=1)
+ Output:["_col2","_col3"]
+ Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
+ predicate:(_col2 <> _col0)
+ Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1)
+ Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
+ <-Map 15 [SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_39]
+ PartitionCols:_col1
+ Select Operator [SEL_20] (rows=287989836 width=135)
+ Output:["_col0","_col1"]
+ TableScan [TS_19] (rows=287989836 width=135)
+ default@catalog_sales,cs2,Tbl:COMPLETE,Col:NONE,Output:["cs_warehouse_sk","cs_order_number"]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_40]
+ PartitionCols:_col1
+ Select Operator [SEL_38] (rows=4611686018427387903 width=1)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_36]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
+ Output:["_col0","_col1"],keys:_col4, _col3
+ Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1)
+ Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
+ <-Map 15 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_32]
+ Select Operator [SEL_28] (rows=287989836 width=135)
+ Output:["_col0","_col1"]
+ Please refer to the previous TableScan [TS_19]
+ <-Map 20 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_29]
+ Select Operator [SEL_22] (rows=73049 width=4)
+ TableScan [TS_21] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
+ <-Map 21 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_30]
+ Select Operator [SEL_24] (rows=60 width=4)
+ TableScan [TS_23] (rows=60 width=2045)
+ default@call_center,call_center,Tbl:COMPLETE,Col:COMPLETE
+ <-Map 22 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_31]
+ Select Operator [SEL_26] (rows=40000000 width=4)
+ TableScan [TS_25] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col3, _col4
+ Merge Join Operator [MERGEJOIN_118] (rows=421645953 width=135)
+ Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
PartitionCols:_col0
- Group By Operator [GBY_15] (rows=28798881 width=106)
- Output:["_col0"],keys:cr_order_number
- Filter Operator [FIL_104] (rows=28798881 width=106)
- predicate:cr_order_number is not null
- TableScan [TS_12] (rows=28798881 width=106)
- default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col4
- Merge Join Operator [MERGEJOIN_110] (rows=383314495 width=135)
- Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
- PartitionCols:_col0
- Select Operator [SEL_11] (rows=30 width=2045)
- Output:["_col0"]
- Filter Operator [FIL_103] (rows=30 width=2045)
- predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null)
- TableScan [TS_9] (rows=60 width=2045)
- default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_54]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_109] (rows=348467716 width=135)
- Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_102] (rows=20000000 width=1014)
- predicate:((ca_state = 'NY') and ca_address_sk is not null)
- TableScan [TS_6] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_108] (rows=316788826 width=135)
- Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
+ Select Operator [SEL_18] (rows=14399440 width=106)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_17] (rows=14399440 width=106)
+ Output:["_col0"],keys:KEY._col0
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Filter Operator [FIL_100] (rows=287989836 width=135)
- predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null)
- TableScan [TS_0] (rows=287989836 width=135)
- default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_101] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null)
- TableScan [TS_3] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+ Group By Operator [GBY_15] (rows=28798881 width=106)
+ Output:["_col0"],keys:cr_order_number
+ Filter Operator [FIL_104] (rows=28798881 width=106)
+ predicate:cr_order_number is not null
+ TableScan [TS_12] (rows=28798881 width=106)
+ default@catalog_returns,cr1,Tbl:COMPLETE,Col:NONE,Output:["cr_order_number"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col4
+ Merge Join Operator [MERGEJOIN_116] (rows=383314495 width=135)
+ Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:_col0
+ Select Operator [SEL_11] (rows=30 width=2045)
+ Output:["_col0"]
+ Filter Operator [FIL_103] (rows=30 width=2045)
+ predicate:((cc_county) IN ('Ziebach County', 'Levy County', 'Huron County', 'Franklin Parish', 'Daviess County') and cc_call_center_sk is not null)
+ TableScan [TS_9] (rows=60 width=2045)
+ default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_county"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_54]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_115] (rows=348467716 width=135)
+ Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_102] (rows=20000000 width=1014)
+ predicate:((ca_state = 'NY') and ca_address_sk is not null)
+ TableScan [TS_6] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_114] (rows=316788826 width=135)
+ Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_48]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Filter Operator [FIL_100] (rows=287989836 width=135)
+ predicate:(cs_ship_date_sk is not null and cs_ship_addr_sk is not null and cs_call_center_sk is not null)
+ TableScan [TS_0] (rows=287989836 width=135)
+ default@catalog_sales,cs1,Tbl:COMPLETE,Col:NONE,Output:["cs_ship_date_sk","cs_ship_addr_sk","cs_call_center_sk","cs_warehouse_sk","cs_order_number","cs_ext_ship_cost","cs_net_profit"]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=8116 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_101] (rows=8116 width=1119)
+ predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 2001-04-01 00:00:00.0 AND 2001-05-31 01:00:00.0 and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/perf/query28.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query28.q.out b/ql/src/test/results/clientpositive/perf/query28.q.out
index 1fe7f15..d2978a5 100644
--- a/ql/src/test/results/clientpositive/perf/query28.q.out
+++ b/ql/src/test/results/clientpositive/perf/query28.q.out
@@ -1,4 +1,4 @@
-Warning: Shuffle Join MERGEJOIN[58][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 3' is a cross product
+Warning: Shuffle Join MERGEJOIN[64][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4, $hdt$_5]] in Stage 'Reducer 4' is a cross product
PREHOOK: query: explain
select *
from (select avg(ss_list_price) B1_LP
@@ -107,40 +107,48 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 2 <- Map 1 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
-Reducer 4 <- Map 1 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE), Reducer 7 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE)
Reducer 5 <- Map 1 (SIMPLE_EDGE)
Reducer 6 <- Map 1 (SIMPLE_EDGE)
Reducer 7 <- Map 1 (SIMPLE_EDGE)
Reducer 8 <- Map 1 (SIMPLE_EDGE)
+Reducer 9 <- Map 1 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:100
Stage-1
- Reducer 3
+ Reducer 4
File Output Operator [FS_51]
- Limit [LIM_50] (rows=1 width=2497)
+ Limit [LIM_50] (rows=1 width=2665)
Number of rows:100
- Select Operator [SEL_49] (rows=1 width=2497)
+ Select Operator [SEL_49] (rows=1 width=2665)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- Merge Join Operator [MERGEJOIN_58] (rows=1 width=2497)
+ Merge Join Operator [MERGEJOIN_64] (rows=1 width=2665)
Conds:(Inner),(Inner),(Inner),(Inner),(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- <-Reducer 2 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 3 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_42]
- Group By Operator [GBY_5] (rows=1 width=416)
- Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_4]
- Group By Operator [GBY_3] (rows=21333171 width=88)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["avg(ss_list_price)","count(ss_list_price)","count(DISTINCT ss_list_price)"],keys:ss_list_price
- Select Operator [SEL_2] (rows=21333171 width=88)
- Output:["ss_list_price"]
- Filter Operator [FIL_52] (rows=21333171 width=88)
- predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34))
- TableScan [TS_0] (rows=575995635 width=88)
- default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
- <-Reducer 4 [CUSTOM_SIMPLE_EDGE]
+ Group By Operator [GBY_63] (rows=1 width=584)
+ Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(VALUE._col2)"]
+ <-Reducer 2 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_62]
+ Group By Operator [GBY_61] (rows=1 width=584)
+ Output:["_col0","_col1","_col2"],aggregations:["avg(_col1)","count(_col2)","count(_col0)"]
+ Group By Operator [GBY_60] (rows=21333171 width=88)
+ Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0
+ Group By Operator [GBY_58] (rows=21333171 width=88)
+ Output:["_col0","_col1","_col2"],aggregations:["avg(ss_list_price)","count(ss_list_price)"],keys:ss_list_price
+ Select Operator [SEL_2] (rows=21333171 width=88)
+ Output:["ss_list_price"]
+ Filter Operator [FIL_52] (rows=21333171 width=88)
+ predicate:(ss_quantity BETWEEN 0 AND 5 and (ss_list_price BETWEEN 11 AND 21 or ss_coupon_amt BETWEEN 460 AND 1460 or ss_wholesale_cost BETWEEN 14 AND 34))
+ TableScan [TS_0] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_quantity","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
+ <-Reducer 5 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_43]
Group By Operator [GBY_12] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
@@ -153,7 +161,7 @@ Stage-0
Filter Operator [FIL_53] (rows=21333171 width=88)
predicate:(ss_quantity BETWEEN 26 AND 30 and (ss_list_price BETWEEN 28 AND 38 or ss_coupon_amt BETWEEN 2513 AND 3513 or ss_wholesale_cost BETWEEN 42 AND 62))
Please refer to the previous TableScan [TS_0]
- <-Reducer 5 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_44]
Group By Operator [GBY_19] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
@@ -166,7 +174,7 @@ Stage-0
Filter Operator [FIL_54] (rows=21333171 width=88)
predicate:(ss_quantity BETWEEN 21 AND 25 and (ss_list_price BETWEEN 135 AND 145 or ss_coupon_amt BETWEEN 14180 AND 15180 or ss_wholesale_cost BETWEEN 38 AND 58))
Please refer to the previous TableScan [TS_0]
- <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_45]
Group By Operator [GBY_26] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
@@ -179,7 +187,7 @@ Stage-0
Filter Operator [FIL_55] (rows=21333171 width=88)
predicate:(ss_quantity BETWEEN 16 AND 20 and (ss_list_price BETWEEN 142 AND 152 or ss_coupon_amt BETWEEN 3054 AND 4054 or ss_wholesale_cost BETWEEN 80 AND 100))
Please refer to the previous TableScan [TS_0]
- <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 8 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_46]
Group By Operator [GBY_33] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
@@ -192,7 +200,7 @@ Stage-0
Filter Operator [FIL_56] (rows=21333171 width=88)
predicate:(ss_quantity BETWEEN 11 AND 15 and (ss_list_price BETWEEN 66 AND 76 or ss_coupon_amt BETWEEN 920 AND 1920 or ss_wholesale_cost BETWEEN 4 AND 24))
Please refer to the previous TableScan [TS_0]
- <-Reducer 8 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 9 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_47]
Group By Operator [GBY_40] (rows=1 width=416)
Output:["_col0","_col1","_col2"],aggregations:["avg(VALUE._col0)","count(VALUE._col1)","count(DISTINCT KEY._col0:0._col0)"]
[15/36] hive git commit: HIVE-16803: Alter table change column
comment should not try to get column stats for update (Chaoyu Tang,
reviewed by Pengcheng Xiong)
Posted by we...@apache.org.
HIVE-16803: Alter table change column comment should not try to get column stats for update (Chaoyu Tang, reviewed by Pengcheng Xiong)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/690a9f8e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/690a9f8e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/690a9f8e
Branch: refs/heads/hive-14535
Commit: 690a9f8e2a2ce3ef773565e5ab3dc3b664e3412a
Parents: 0155a34
Author: Chaoyu Tang <ct...@cloudera.com>
Authored: Fri Jun 2 14:25:17 2017 -0400
Committer: Chaoyu Tang <ct...@cloudera.com>
Committed: Fri Jun 2 14:25:17 2017 -0400
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 ++--
.../hadoop/hive/metastore/MetaStoreUtils.java | 14 +++++++++++---
.../hadoop/hive/metastore/TestMetaStoreUtils.java | 16 +++++++++-------
3 files changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/690a9f8e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 7978a40..7c1be8c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -727,7 +727,7 @@ public class HiveAlterHandler implements AlterHandler {
// Nothing to update if everything is the same
if (newDbName.equals(dbName) &&
newTableName.equals(tableName) &&
- MetaStoreUtils.columnsIncluded(oldCols, newCols)) {
+ MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) {
updateColumnStats = false;
}
@@ -802,7 +802,7 @@ public class HiveAlterHandler implements AlterHandler {
|| !oldPartName.equals(newPartName);
// do not need to update column stats if alter partition is not for rename or changing existing columns
- if (!rename && MetaStoreUtils.columnsIncluded(oldCols, newCols)) {
+ if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) {
return newPartsColStats;
}
List<String> oldColNames = new ArrayList<String>(oldCols.size());
http://git-wip-us.apache.org/repos/asf/hive/blob/690a9f8e/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 8328428..a8adb61 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -652,14 +652,22 @@ public class MetaStoreUtils {
return true;
}
- static boolean columnsIncluded(List<FieldSchema> oldCols, List<FieldSchema> newCols) {
+ /*
+ * This method is to check if the new column list includes all the old columns with same name and
+ * type. The column comment does not count.
+ */
+ static boolean columnsIncludedByNameType(List<FieldSchema> oldCols, List<FieldSchema> newCols) {
if (oldCols.size() > newCols.size()) {
return false;
}
- Set<FieldSchema> newColsSet = new HashSet<FieldSchema>(newCols);
+ Map<String, String> columnNameTypePairMap = new HashMap<String, String>(newCols.size());
+ for (FieldSchema newCol : newCols) {
+ columnNameTypePairMap.put(newCol.getName().toLowerCase(), newCol.getType());
+ }
for (final FieldSchema oldCol : oldCols) {
- if (!newColsSet.contains(oldCol)) {
+ if (!columnNameTypePairMap.containsKey(oldCol.getName())
+ || !columnNameTypePairMap.get(oldCol.getName()).equalsIgnoreCase(oldCol.getType())) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/690a9f8e/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
index 21f9054..e5c8a40 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreUtils.java
@@ -27,15 +27,17 @@ import java.util.Arrays;
public class TestMetaStoreUtils {
@Test
- public void testColumnsIncluded() {
+ public void testcolumnsIncludedByNameType() {
FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment");
+ FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment");
FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment");
FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment");
- Assert.assertTrue(MetaStoreUtils.columnsIncluded(Arrays.asList(col1), Arrays.asList(col1)));
- Assert.assertTrue(MetaStoreUtils.columnsIncluded(Arrays.asList(col1, col2), Arrays.asList(col1, col2)));
- Assert.assertTrue(MetaStoreUtils.columnsIncluded(Arrays.asList(col1, col2), Arrays.asList(col2, col1)));
- Assert.assertTrue(MetaStoreUtils.columnsIncluded(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3)));
- Assert.assertTrue(MetaStoreUtils.columnsIncluded(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1)));
- Assert.assertFalse(MetaStoreUtils.columnsIncluded(Arrays.asList(col1, col2), Arrays.asList(col1)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3)));
+ Assert.assertTrue(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1)));
+ Assert.assertFalse(MetaStoreUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1)));
}
}
[20/36] hive git commit: HIVE-16566: Set column stats default as true
when creating new tables/partitions (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out
index 5d5f8c7..c226eed 100644
--- a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out
+++ b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out
@@ -311,7 +311,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -345,7 +345,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -1350,7 +1350,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -1384,7 +1384,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/metadataonly1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out
index 27218cf..e6853b2 100644
--- a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out
+++ b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out
@@ -150,7 +150,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -291,7 +291,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -432,7 +432,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -580,7 +580,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -627,7 +627,7 @@ STAGE PLANS:
partition values:
ds 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -704,7 +704,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -751,7 +751,7 @@ STAGE PLANS:
partition values:
ds 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -988,7 +988,7 @@ STAGE PLANS:
ds 1
hr 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1035,7 +1035,7 @@ STAGE PLANS:
ds 1
hr 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1082,7 +1082,7 @@ STAGE PLANS:
ds 1
hr 3
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1233,7 +1233,7 @@ STAGE PLANS:
ds 1
hr 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1281,7 +1281,7 @@ STAGE PLANS:
ds 1
hr 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1329,7 +1329,7 @@ STAGE PLANS:
ds 1
hr 3
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1476,7 +1476,7 @@ STAGE PLANS:
partition values:
ds 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1522,7 +1522,7 @@ STAGE PLANS:
partition values:
ds 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1721,7 +1721,7 @@ STAGE PLANS:
ds 01:10:10
hr 01
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1768,7 +1768,7 @@ STAGE PLANS:
ds 01:10:20
hr 02
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1815,7 +1815,7 @@ STAGE PLANS:
ds 1
hr 1
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1862,7 +1862,7 @@ STAGE PLANS:
ds 1
hr 2
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
@@ -1909,7 +1909,7 @@ STAGE PLANS:
ds 1
hr 3
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true"}}
bucket_count -1
column.name.delimiter ,
columns a,b
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/orc_create.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_create.q.out b/ql/src/test/results/clientpositive/llap/orc_create.q.out
index 893ca41..849fc46 100644
--- a/ql/src/test/results/clientpositive/llap/orc_create.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_create.q.out
@@ -66,7 +66,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"lst\":\"true\",\"mp\":\"true\",\"str\":\"true\",\"strct\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -345,7 +345,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"lst\":\"true\",\"mp\":\"true\",\"str\":\"true\",\"strct\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out
index 2cc6f5d..d74e4c7 100644
--- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out
+++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -154,7 +154,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/sample1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sample1.q.out b/ql/src/test/results/clientpositive/llap/sample1.q.out
index 61a3a0f..1a7fb32 100644
--- a/ql/src/test/results/clientpositive/llap/sample1.q.out
+++ b/ql/src/test/results/clientpositive/llap/sample1.q.out
@@ -52,7 +52,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -141,7 +141,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/stats11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/stats11.q.out b/ql/src/test/results/clientpositive/llap/stats11.q.out
index c71a71e..48d76cf 100644
--- a/ql/src/test/results/clientpositive/llap/stats11.q.out
+++ b/ql/src/test/results/clientpositive/llap/stats11.q.out
@@ -498,7 +498,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -532,7 +532,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/sysdb.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index e6f408b..64b97ee 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -3152,11 +3152,11 @@ POSTHOOK: query: select param_key, param_value from table_params order by param_
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@table_params
#### A masked pattern was here ####
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c_boolean":"true","c_float":"true","c_int":"true","key":"true","value":"true"}}
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}}
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"json":"true"}}
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true","c":"true","d":"true","e":"true","f":"true","g":"true"}}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"add_time":"true","grant_option":"true","grantor":"true","grantor_type":"true","principal_name":"true","principal_type":"true","role_grant_id":"true","role_id":"true"}}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"avg_col_len":"true","big_decimal_high_value":"true","big_decimal_low_value":"true","column_name":"true","column_type":"true","cs_id":"true","db_name":"true","double_high_value":"true","double_low_value":"true","last_analyzed":"true","long_high_value":"true","long_low_value":"true","max_col_len":"true","num_distincts":"true","num_falses":"true","num_nulls":"true","num_trues":"true","part_id":"true","partition_name":"true","table_name":"true"}}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"avg_col_len":"true","big_decimal_high_value":"true","big_decimal_low_value":"true","column_name":"true","column_type":"true","cs_id":"true","db_name":"true","double_high_value":"true","double_low_value":"true","last_analyzed":"true","long_high_value":"true","long_low_value":"true","max_col_len":"true","num_distincts":"true","num_falses":"true","num_nulls":"true","num_trues":"true","table_name":"true","tbl_id":"true"}}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"bucket_col_name":"true","integer_idx":"true","sd_id":"true"}}
PREHOOK: query: select tbl_name from tbls order by tbl_name limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@tbls
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/parquet_array_null_element.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_array_null_element.q.out b/ql/src/test/results/clientpositive/parquet_array_null_element.q.out
index 75d2d27..f05ecc6 100644
--- a/ql/src/test/results/clientpositive/parquet_array_null_element.q.out
+++ b/ql/src/test/results/clientpositive/parquet_array_null_element.q.out
@@ -70,7 +70,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"lstint\":\"true\",\"lststr\":\"true\",\"mp\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/parquet_create.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_create.q.out b/ql/src/test/results/clientpositive/parquet_create.q.out
index cc4b735..b4222b3 100644
--- a/ql/src/test/results/clientpositive/parquet_create.q.out
+++ b/ql/src/test/results/clientpositive/parquet_create.q.out
@@ -73,7 +73,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"lst\":\"true\",\"mp\":\"true\",\"str\":\"true\",\"strct\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out b/ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out
index 83f1a69..d15fd81 100644
--- a/ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out
@@ -639,7 +639,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"ts\":\"true\"}}
numFiles 0
numRows 0
parquet.mr.int96.write.zone GMT+10
@@ -674,7 +674,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"ts\":\"true\"}}
numFiles 0
numRows 0
parquet.mr.int96.write.zone GMT+10
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out b/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
index 077b1f1..8b42dc3 100644
--- a/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
+++ b/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
@@ -34,7 +34,7 @@ Database: default
Table: mytbl
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\"}}
a myval
b yourval
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out b/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
index 95c2d6e..5c2c8a6 100644
--- a/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
+++ b/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
@@ -34,7 +34,7 @@ Database: default
Table: mytbl
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out b/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
index 6036e5c..9e00fe6 100644
--- a/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
+++ b/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
@@ -34,7 +34,7 @@ Database: default
Table: mytbl
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\"}}
a myval
b yourval
c noval
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/partial_column_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/partial_column_stats.q.out b/ql/src/test/results/clientpositive/partial_column_stats.q.out
index f204a18..5876efa 100644
--- a/ql/src/test/results/clientpositive/partial_column_stats.q.out
+++ b/ql/src/test/results/clientpositive/partial_column_stats.q.out
@@ -72,4 +72,4 @@ POSTHOOK: Input: default@t1
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
value string 0 0 0.0 0 from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"data\":\"true\",\"key\":\"true\",\"value\":\"true\"}}
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/partition_coltype_literals.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/partition_coltype_literals.q.out b/ql/src/test/results/clientpositive/partition_coltype_literals.q.out
index 3affc94..3505556 100644
--- a/ql/src/test/results/clientpositive/partition_coltype_literals.q.out
+++ b/ql/src/test/results/clientpositive/partition_coltype_literals.q.out
@@ -41,7 +41,7 @@ Database: default
Table: partcoltypenum
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -91,7 +91,7 @@ Database: default
Table: partcoltypenum
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -142,7 +142,7 @@ Database: default
Table: partcoltypenum
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -194,7 +194,7 @@ Database: default
Table: partcoltypenum
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/pcr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/pcr.q.out b/ql/src/test/results/clientpositive/pcr.q.out
index ffb1d91..5d0d170 100644
--- a/ql/src/test/results/clientpositive/pcr.q.out
+++ b/ql/src/test/results/clientpositive/pcr.q.out
@@ -3501,7 +3501,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3537,7 +3537,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3629,7 +3629,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3666,7 +3666,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3696,7 +3696,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3717,7 +3717,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3753,7 +3753,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3783,7 +3783,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3804,7 +3804,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3850,7 +3850,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3887,7 +3887,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3917,7 +3917,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3938,7 +3938,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3974,7 +3974,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -4004,7 +4004,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -4025,7 +4025,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/quotedid_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/quotedid_stats.q.out b/ql/src/test/results/clientpositive/quotedid_stats.q.out
index a8cbb74..40e23bf 100644
--- a/ql/src/test/results/clientpositive/quotedid_stats.q.out
+++ b/ql/src/test/results/clientpositive/quotedid_stats.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"x+1`\":\"true\",\"y&y\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
index fa4aae8..9b29136 100644
--- a/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
+++ b/ql/src/test/results/clientpositive/rand_partitionpruner2.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -193,7 +193,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -230,7 +230,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -260,7 +260,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -281,7 +281,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -317,7 +317,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -347,7 +347,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -368,7 +368,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/rcfile_default_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/rcfile_default_format.q.out b/ql/src/test/results/clientpositive/rcfile_default_format.q.out
index bb846c0..97bc8d2 100644
--- a/ql/src/test/results/clientpositive/rcfile_default_format.q.out
+++ b/ql/src/test/results/clientpositive/rcfile_default_format.q.out
@@ -23,7 +23,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -257,7 +257,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -346,7 +346,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample1.q.out b/ql/src/test/results/clientpositive/sample1.q.out
index 1947e92..dec9b23 100644
--- a/ql/src/test/results/clientpositive/sample1.q.out
+++ b/ql/src/test/results/clientpositive/sample1.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -146,7 +146,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -183,7 +183,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -213,7 +213,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -234,7 +234,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -270,7 +270,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -300,7 +300,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -321,7 +321,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample2.q.out b/ql/src/test/results/clientpositive/sample2.q.out
index 9c5e238..f54c573 100644
--- a/ql/src/test/results/clientpositive/sample2.q.out
+++ b/ql/src/test/results/clientpositive/sample2.q.out
@@ -52,7 +52,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -145,7 +145,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -182,7 +182,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -212,7 +212,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -233,7 +233,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -269,7 +269,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -299,7 +299,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -320,7 +320,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample4.q.out b/ql/src/test/results/clientpositive/sample4.q.out
index 20bd86c..675fda9 100644
--- a/ql/src/test/results/clientpositive/sample4.q.out
+++ b/ql/src/test/results/clientpositive/sample4.q.out
@@ -52,7 +52,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -145,7 +145,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -182,7 +182,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -212,7 +212,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -233,7 +233,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -269,7 +269,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -299,7 +299,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -320,7 +320,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample5.q.out b/ql/src/test/results/clientpositive/sample5.q.out
index 0ed1fb2..583784e 100644
--- a/ql/src/test/results/clientpositive/sample5.q.out
+++ b/ql/src/test/results/clientpositive/sample5.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -146,7 +146,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -183,7 +183,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -213,7 +213,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -234,7 +234,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -270,7 +270,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -300,7 +300,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -321,7 +321,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample6.q.out b/ql/src/test/results/clientpositive/sample6.q.out
index 1988a90..36e6906 100644
--- a/ql/src/test/results/clientpositive/sample6.q.out
+++ b/ql/src/test/results/clientpositive/sample6.q.out
@@ -52,7 +52,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -145,7 +145,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -182,7 +182,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -212,7 +212,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -233,7 +233,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -269,7 +269,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -299,7 +299,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -320,7 +320,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/sample7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/sample7.q.out b/ql/src/test/results/clientpositive/sample7.q.out
index 7de68a8..f0d9088 100644
--- a/ql/src/test/results/clientpositive/sample7.q.out
+++ b/ql/src/test/results/clientpositive/sample7.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -146,7 +146,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -183,7 +183,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -213,7 +213,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -234,7 +234,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -270,7 +270,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -300,7 +300,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -321,7 +321,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/show_tblproperties.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_tblproperties.q.out b/ql/src/test/results/clientpositive/show_tblproperties.q.out
index 005de89..1ef1b9b 100644
--- a/ql/src/test/results/clientpositive/show_tblproperties.q.out
+++ b/ql/src/test/results/clientpositive/show_tblproperties.q.out
@@ -36,7 +36,7 @@ PREHOOK: query: show tblproperties tmpfoo
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties tmpfoo
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true"}}
bar bar value
#### A masked pattern was here ####
numFiles 0
@@ -54,7 +54,7 @@ PREHOOK: query: show tblproperties default.tmpfoo
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties default.tmpfoo
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true"}}
bar bar value
#### A masked pattern was here ####
numFiles 0
@@ -108,7 +108,7 @@ PREHOOK: query: show tblproperties default.tmpfoo
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties default.tmpfoo
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"a":"true"}}
bar bar value
#### A masked pattern was here ####
numFiles 0
@@ -126,7 +126,7 @@ PREHOOK: query: show tblproperties tmpfoo
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties tmpfoo
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"b":"true"}}
bar bar value1
#### A masked pattern was here ####
numFiles 0
@@ -150,7 +150,7 @@ PREHOOK: query: show tblproperties db1.tmpfoo
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: show tblproperties db1.tmpfoo
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"b":"true"}}
bar bar value1
#### A masked pattern was here ####
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket2.q.out b/ql/src/test/results/clientpositive/spark/bucket2.q.out
index 540e6f8..90c9e54 100644
--- a/ql/src/test/results/clientpositive/spark/bucket2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket2.q.out
@@ -113,7 +113,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -145,7 +145,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket4.q.out b/ql/src/test/results/clientpositive/spark/bucket4.q.out
index d944a5e..13e21b6 100644
--- a/ql/src/test/results/clientpositive/spark/bucket4.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket4.q.out
@@ -114,7 +114,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -147,7 +147,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket5.q.out b/ql/src/test/results/clientpositive/spark/bucket5.q.out
index 1d0438c..dd24db8 100644
--- a/ql/src/test/results/clientpositive/spark/bucket5.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket5.q.out
@@ -197,7 +197,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -238,7 +238,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -269,7 +269,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -303,7 +303,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out
index 0fcf844..7229734 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out
@@ -252,7 +252,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -343,7 +343,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out
index 01f1003..4592f21 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out
@@ -236,7 +236,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -327,7 +327,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out
index 244ac03..add6ac4 100644
--- a/ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out
@@ -236,7 +236,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -327,7 +327,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
index 1b47864..5943fdf 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out
@@ -562,7 +562,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -593,7 +593,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
index 58a2935..ed95995 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out
@@ -290,7 +290,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -321,7 +321,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
[27/36] hive git commit: Revert "Revert "HIVE-16744 : LLAP index
update may be broken after ORC switch (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)""
Posted by we...@apache.org.
Revert "Revert "HIVE-16744 : LLAP index update may be broken after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)""
This reverts commit 38df8b4b8c624cb58b563a1eef43e7b417b9ab0a.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7916965e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7916965e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7916965e
Branch: refs/heads/hive-14535
Commit: 7916965eaaf6f86a0d20121d9e8fdea283a02458
Parents: 38df8b4
Author: sergey <se...@apache.org>
Authored: Fri Jun 2 15:43:54 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Jun 2 15:43:54 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7916965e/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 9693826..7081140 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -394,7 +394,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}",
stripeKey.stripeIx, DebugUtils.toString(globalIncludes));
}
- assert isFoundInCache;
+ assert isFoundInCache; // If it's not fresh from the cache, indexes should be present.
counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
ensureMetadataReader();
updateLoadedIndexes(stripeMetadata, stripe, globalIncludes, sargColumns);
http://git-wip-us.apache.org/repos/asf/hive/blob/7916965e/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index 1f3f7ea..96c4cce 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -113,7 +113,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
public void loadMissingIndexes(DataReader mr, StripeInformation stripe, boolean[] includes,
boolean[] sargColumns) throws IOException {
- // Do not loose the old indexes. Create a super set includes
+ // Do not lose the old indexes. Create a super set includes
OrcProto.RowIndex[] existing = getRowIndexes();
boolean superset[] = new boolean[Math.max(existing.length, includes.length)];
for (int i = 0; i < includes.length; i++) {
@@ -123,7 +123,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
superset[i] = superset[i] || (existing[i] != null);
}
// TODO: should we save footer to avoid a read here?
- rowIndex = mr.readRowIndex(stripe, schema, null, true, includes,
+ rowIndex = mr.readRowIndex(stripe, schema, null, true, superset,
rowIndex.getRowGroupIndex(),
sargColumns, writerVersion, rowIndex.getBloomFilterKinds(),
rowIndex.getBloomFilterIndex());
[25/36] hive git commit: HIVE-16460: In the console output,
show vertex list in topological order instead of an alphabetical sort
(Prasanth Jayachandran reviewed by Siddharth Seth)
Posted by we...@apache.org.
HIVE-16460: In the console output, show vertex list in topological order instead of an alphabetical sort (Prasanth Jayachandran reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/087aafca
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/087aafca
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/087aafca
Branch: refs/heads/hive-14535
Commit: 087aafca763ed1f49cab2c389d2e00425c48a6d0
Parents: cea9ea7
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Fri Jun 2 15:30:51 2017 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Fri Jun 2 15:30:51 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +-
.../ql/exec/tez/monitoring/TezJobMonitor.java | 8 ++--
.../exec/tez/monitoring/TezProgressMonitor.java | 42 ++++++++++----------
.../tez/monitoring/TestTezProgressMonitor.java | 3 +-
4 files changed, 28 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3356dc9..7ee6186 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -194,7 +194,7 @@ public class TezTask extends Task<TezWork> {
additionalLr, inputOutputJars, inputOutputLocalResources);
// finally monitor will print progress until the job is done
- TezJobMonitor monitor = new TezJobMonitor(work.getWorkMap(),dagClient, conf, dag, ctx);
+ TezJobMonitor monitor = new TezJobMonitor(work.getAllWork(),dagClient, conf, dag, ctx);
rc = monitor.monitorExecution();
if (rc != 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
index 049d7fd..9e2846c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
@@ -68,7 +68,7 @@ public class TezJobMonitor {
private final PerfLogger perfLogger = SessionState.getPerfLogger();
private static final List<DAGClient> shutdownList;
- private final Map<String, BaseWork> workMap;
+ private final List<BaseWork> topSortedWorks;
transient LogHelper console;
@@ -101,9 +101,9 @@ public class TezJobMonitor {
private long executionStartTime = 0;
private final RenderStrategy.UpdateFunction updateFunction;
- public TezJobMonitor(Map<String, BaseWork> workMap, final DAGClient dagClient, HiveConf conf, DAG dag,
+ public TezJobMonitor(List<BaseWork> topSortedWorks, final DAGClient dagClient, HiveConf conf, DAG dag,
Context ctx) {
- this.workMap = workMap;
+ this.topSortedWorks = topSortedWorks;
this.dagClient = dagClient;
this.hiveConf = conf;
this.dag = dag;
@@ -323,7 +323,7 @@ public class TezJobMonitor {
ProgressMonitor progressMonitor(DAGStatus status, Map<String, Progress> progressMap) {
try {
- return new TezProgressMonitor(dagClient, status, workMap, progressMap, console,
+ return new TezProgressMonitor(dagClient, status, topSortedWorks, progressMap, console,
executionStartTime);
} catch (IOException | TezException e) {
console.printInfo("Getting Progress Information: " + e.getMessage() + " stack trace: " +
http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java
index 9739ad7..7fbd80a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java
@@ -42,7 +42,7 @@ import static org.apache.tez.dag.api.client.DAGStatus.State.KILLED;
class TezProgressMonitor implements ProgressMonitor {
private static final int COLUMN_1_WIDTH = 16;
- private final Map<String, BaseWork> workMap;
+ private final List<BaseWork> topSortedWork;
private final SessionState.LogHelper console;
private final long executionStartTime;
private final DAGStatus status;
@@ -53,11 +53,11 @@ class TezProgressMonitor implements ProgressMonitor {
* Try to get most the data required from dagClient in the constructor itself so that even after
* the tez job has finished this object can be used for later use.s
*/
- TezProgressMonitor(DAGClient dagClient, DAGStatus status, Map<String, BaseWork> workMap,
+ TezProgressMonitor(DAGClient dagClient, DAGStatus status, List<BaseWork> topSortedWork,
Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime)
throws IOException, TezException {
this.status = status;
- this.workMap = workMap;
+ this.topSortedWork = topSortedWork;
this.console = console;
this.executionStartTime = executionStartTime;
for (Map.Entry<String, Progress> entry : progressMap.entrySet()) {
@@ -88,25 +88,26 @@ class TezProgressMonitor implements ProgressMonitor {
public List<List<String>> rows() {
try {
List<List<String>> results = new ArrayList<>();
- SortedSet<String> keys = new TreeSet<>(progressCountsMap.keySet());
- for (String s : keys) {
- VertexProgress progress = progressCountsMap.get(s);
+ for (BaseWork baseWork : topSortedWork) {
+ String vertexName = baseWork.getName();
+ VertexProgress progress = progressCountsMap.get(vertexName);
+ if (progress != null) {
+ // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0
- // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0
-
- results.add(
+ results.add(
Arrays.asList(
- getNameWithProgress(s, progress.succeededTaskCount, progress.totalTaskCount),
- getMode(s, workMap),
- progress.vertexStatus(vertexStatusMap.get(s)),
- progress.total(),
- progress.completed(),
- progress.running(),
- progress.pending(),
- progress.failed(),
- progress.killed()
+ getNameWithProgress(vertexName, progress.succeededTaskCount, progress.totalTaskCount),
+ getMode(baseWork),
+ progress.vertexStatus(vertexStatusMap.get(vertexName)),
+ progress.total(),
+ progress.completed(),
+ progress.running(),
+ progress.pending(),
+ progress.failed(),
+ progress.killed()
)
- );
+ );
+ }
}
return results;
} catch (Exception e) {
@@ -194,9 +195,8 @@ class TezProgressMonitor implements ProgressMonitor {
return result;
}
- private String getMode(String name, Map<String, BaseWork> workMap) {
+ private String getMode(BaseWork work) {
String mode = "container";
- BaseWork work = workMap.get(name);
if (work != null) {
// uber > llap > container
if (work.getUberMode()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java
index 1e0fa14..98067df 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java
@@ -32,6 +32,7 @@ import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -92,7 +93,7 @@ public class TestTezProgressMonitor {
when(dagClient.getVertexStatus(eq(REDUCER), anySet())).thenReturn(running);
TezProgressMonitor monitor =
- new TezProgressMonitor(dagClient, dagStatus, new HashMap<String, BaseWork>(), progressMap(), console,
+ new TezProgressMonitor(dagClient, dagStatus, new ArrayList<BaseWork>(), progressMap(), console,
Long.MAX_VALUE);
verify(dagClient).getVertexStatus(eq(MAPPER), isNull(Set.class));
[36/36] hive git commit: HIVE-14671 : Merge branch 'master' into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/78f28c88
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/78f28c88
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/78f28c88
Branch: refs/heads/hive-14535
Commit: 78f28c8855f9a2867c81f911cf56684c09a35ea9
Parents: d9e4ae0 ee3a3ed
Author: Wei Zheng <we...@apache.org>
Authored: Tue Jun 6 11:34:43 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Tue Jun 6 11:34:43 2017 -0700
----------------------------------------------------------------------
.../apache/hive/beeline/HiveSchemaHelper.java | 581 ------
.../org/apache/hive/beeline/HiveSchemaTool.java | 79 +-
.../beeline/SeparatedValuesOutputFormat.java | 11 +-
.../apache/hive/beeline/TestHiveSchemaTool.java | 1 +
bin/ext/version.sh | 2 +-
bin/hive | 20 +-
.../hadoop/hive/common/StatsSetupConst.java | 7 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +
.../hive/http/Log4j2ConfiguratorServlet.java | 18 +-
.../hadoop/hive/druid/serde/DruidSerDe.java | 95 +-
.../hadoop/hive/druid/TestDruidSerDe.java | 169 +-
.../results/positive/external_table_ppd.q.out | 2 +-
.../positive/hbase_binary_storage_queries.q.out | 4 +-
.../src/test/results/positive/hbasestats.q.out | 2 +-
.../hcatalog/templeton/tool/TempletonUtils.java | 28 +-
.../templeton/tool/TestTempletonUtils.java | 11 +-
.../write_final_output_blobstore.q.out | 16 +-
itests/hive-unit/pom.xml | 6 +
.../hive/metastore/TestReplChangeManager.java | 89 +-
...TestReplicationScenariosAcrossInstances.java | 166 +-
.../hadoop/hive/ql/parse/WarehouseInstance.java | 176 +-
.../org/apache/hive/beeline/TestSchemaTool.java | 6 +-
.../test/resources/testconfiguration.properties | 6 +-
.../hadoop/hive/cli/control/CliConfigs.java | 1 +
.../java/org/apache/hive/beeline/QFile.java | 1 +
.../llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../llap/io/metadata/OrcStripeMetadata.java | 4 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 14 +-
.../hive/metastore/IMetaStoreSchemaInfo.java | 15 +
.../hive/metastore/MetaStoreSchemaInfo.java | 33 +
.../hadoop/hive/metastore/MetaStoreUtils.java | 22 +-
.../hadoop/hive/metastore/ObjectStore.java | 12 +-
.../hive/metastore/ReplChangeManager.java | 51 +-
.../apache/hadoop/hive/metastore/Warehouse.java | 4 +
.../hive/metastore/tools/HiveSchemaHelper.java | 640 +++++++
.../hive/metastore/TestMetaStoreUtils.java | 16 +-
.../hadoop/hive/metastore/TestObjectStore.java | 48 +-
pom.xml | 4 +-
ql/pom.xml | 13 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 4 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 10 +-
.../hadoop/hive/ql/exec/ReplCopyTask.java | 22 +-
.../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +-
.../spark/status/RemoteSparkJobMonitor.java | 14 +
.../ql/exec/spark/status/SparkJobMonitor.java | 12 +
.../hive/ql/exec/tez/TezSessionPoolManager.java | 9 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +-
.../ql/exec/tez/monitoring/TezJobMonitor.java | 8 +-
.../exec/tez/monitoring/TezProgressMonitor.java | 42 +-
.../fast/VectorMapJoinFastHashTable.java | 3 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../ql/optimizer/CountDistinctRewriteProc.java | 504 +++++
.../hadoop/hive/ql/optimizer/MergeJoinProc.java | 23 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +
.../calcite/cost/HiveAlgorithmsUtil.java | 4 +-
.../optimizer/calcite/cost/HiveCostModel.java | 7 +-
.../calcite/cost/HiveDefaultCostModel.java | 2 +-
.../calcite/cost/HiveOnTezCostModel.java | 4 +-
.../optimizer/calcite/cost/HiveRelMdCost.java | 2 +-
.../rules/HiveFilterAggregateTransposeRule.java | 6 +-
.../calcite/rules/HiveRelDecorrelator.java | 46 +-
.../stats/FilterSelectivityEstimator.java | 9 +-
.../stats/HiveRelMdDistinctRowCount.java | 2 +-
.../calcite/stats/HiveRelMdRowCount.java | 3 +-
.../calcite/stats/HiveRelMdSelectivity.java | 4 +-
.../calcite/stats/HiveRelMdUniqueKeys.java | 2 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 88 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 4 +-
.../ql/parse/ReplicationSemanticAnalyzer.java | 50 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 36 +-
.../hadoop/hive/ql/parse/repl/PathBuilder.java | 66 +
.../repl/dump/events/CreateFunctionHandler.java | 9 +-
.../parse/repl/dump/io/FunctionSerializer.java | 40 +-
.../hive/ql/parse/repl/load/MetaData.java | 2 +-
.../load/message/CreateFunctionHandler.java | 156 +-
.../hadoop/hive/ql/plan/CreateTableDesc.java | 7 +-
.../apache/hadoop/hive/ql/plan/GroupByDesc.java | 15 +
.../hive/ql/exec/tez/TestTezSessionPool.java | 43 +-
.../tez/monitoring/TestTezProgressMonitor.java | 3 +-
.../PrimaryToReplicaResourceFunctionTest.java | 87 +
.../orderby_position_unsupported.q | 3 +-
.../clientnegative/spark_stage_max_tasks.q | 6 +
.../queries/clientpositive/alterColumnStats.q | 21 +
.../clientpositive/alterColumnStatsPart.q | 47 +
.../queries/clientpositive/count_dist_rewrite.q | 65 +
.../test/queries/clientpositive/druid_basic2.q | 4 +
.../test/queries/clientpositive/filter_aggr.q | 18 +
.../test/queries/clientpositive/order_by_pos.q | 20 +
.../test/queries/clientpositive/perf/query4.q | 111 ++
.../test/queries/clientpositive/perf/query47.q | 53 +
.../test/queries/clientpositive/perf/query57.q | 50 +
.../test/queries/clientpositive/perf/query74.q | 63 +
ql/src/test/queries/clientpositive/tez-tag.q | 68 +
.../clientnegative/alter_file_format.q.out | 2 +-
.../orderby_position_unsupported.q.out | 4 +-
.../spark/spark_stage_max_tasks.q.out | 77 +
.../clientnegative/unset_table_property.q.out | 2 +-
.../clientpositive/alterColumnStats.q.out | 158 ++
.../clientpositive/alterColumnStatsPart.q.out | 507 +++++
.../clientpositive/alter_file_format.q.out | 26 +-
.../alter_partition_clusterby_sortby.q.out | 6 +-
.../clientpositive/alter_skewed_table.q.out | 12 +-
.../alter_table_add_partition.q.out | 2 +-
.../clientpositive/alter_table_not_sorted.q.out | 4 +-
.../clientpositive/autoColumnStats_3.q.out | 2 +-
.../clientpositive/autoColumnStats_4.q.out | 2 +-
.../insert_overwrite_local_directory_1.q.out | 1769 ++++++++++++++++++
.../clientpositive/binary_output_format.q.out | 16 +-
.../test/results/clientpositive/bucket1.q.out | 4 +-
.../test/results/clientpositive/bucket2.q.out | 4 +-
.../clientpositive/bucket_map_join_spark1.q.out | 4 +-
.../clientpositive/bucket_map_join_spark2.q.out | 4 +-
.../clientpositive/bucket_map_join_spark3.q.out | 4 +-
.../results/clientpositive/bucketmapjoin5.q.out | 16 +-
.../clientpositive/bucketmapjoin_negative.q.out | 16 +-
.../bucketmapjoin_negative2.q.out | 16 +-
...names_with_leading_and_trailing_spaces.q.out | 4 +-
.../results/clientpositive/compustat_avro.q.out | 4 +-
.../clientpositive/count_dist_rewrite.q.out | 1151 ++++++++++++
.../create_alter_list_bucketing_table1.q.out | 14 +-
.../results/clientpositive/create_like.q.out | 18 +-
.../results/clientpositive/create_like2.q.out | 2 +-
.../clientpositive/create_like_tbl_props.q.out | 10 +-
.../clientpositive/create_like_view.q.out | 6 +-
.../clientpositive/create_skewed_table1.q.out | 6 +-
.../create_table_like_stats.q.out | 2 +-
.../create_with_constraints.q.out | 72 +-
.../clientpositive/database_location.q.out | 4 +-
.../clientpositive/default_file_format.q.out | 16 +-
.../describe_comment_indent.q.out | 2 +-
.../describe_comment_nonascii.q.out | 2 +-
.../describe_formatted_view_partitioned.q.out | 2 +-
.../clientpositive/describe_syntax.q.out | 4 +-
.../results/clientpositive/describe_table.q.out | 24 +-
.../display_colstats_tbllvl.q.out | 2 +-
.../results/clientpositive/druid_basic1.q.out | 4 +-
.../results/clientpositive/druid_basic2.q.out | 58 +-
.../clientpositive/druid_intervals.q.out | 2 +-
.../clientpositive/druid_timeseries.q.out | 2 +-
.../results/clientpositive/druid_topn.q.out | 2 +-
.../results/clientpositive/filter_aggr.q.out | 680 +++++++
.../clientpositive/groupby_map_ppr.q.out | 4 +-
.../groupby_map_ppr_multi_distinct.q.out | 4 +-
.../results/clientpositive/groupby_ppr.q.out | 4 +-
.../groupby_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/groupby_sort_11.q.out | 39 +-
.../clientpositive/groupby_sort_1_23.q.out | 56 +-
.../results/clientpositive/groupby_sort_6.q.out | 4 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 56 +-
.../results/clientpositive/input_part1.q.out | 16 +-
.../results/clientpositive/input_part2.q.out | 32 +-
.../insert_values_orig_table_use_metadata.q.out | 4 +-
ql/src/test/results/clientpositive/join17.q.out | 4 +-
ql/src/test/results/clientpositive/join26.q.out | 4 +-
ql/src/test/results/clientpositive/join32.q.out | 4 +-
ql/src/test/results/clientpositive/join33.q.out | 4 +-
ql/src/test/results/clientpositive/join34.q.out | 6 +-
ql/src/test/results/clientpositive/join35.q.out | 6 +-
ql/src/test/results/clientpositive/join9.q.out | 4 +-
.../results/clientpositive/join_map_ppr.q.out | 16 +-
.../clientpositive/list_bucket_dml_14.q.out | 4 +-
.../clientpositive/llap/autoColumnStats_2.q.out | 2 +-
.../results/clientpositive/llap/bucket2.q.out | 4 +-
.../results/clientpositive/llap/bucket4.q.out | 4 +-
.../results/clientpositive/llap/bucket5.q.out | 20 +-
.../clientpositive/llap/bucket_many.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin1.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin2.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin3.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin4.q.out | 4 +-
...names_with_leading_and_trailing_spaces.q.out | 4 +-
.../llap/count_dist_rewrite.q.out | 1169 ++++++++++++
.../llap/disable_merge_for_bucketing.q.out | 4 +-
.../llap/dynamic_semijoin_reduction.q.out | 2 +-
.../llap/dynamic_semijoin_user_level.q.out | 2 +-
.../clientpositive/llap/join32_lessSize.q.out | 8 +-
.../clientpositive/llap/metadataonly1.q.out | 40 +-
.../clientpositive/llap/orc_create.q.out | 4 +-
.../clientpositive/llap/orc_llap_counters.q.out | 2 +-
.../clientpositive/llap/orc_ppd_basic.q.out | 230 ---
.../llap/orc_ppd_schema_evol_3a.q.out | 6 +-
.../llap/reduce_deduplicate.q.out | 4 +-
.../results/clientpositive/llap/sample1.q.out | 4 +-
.../results/clientpositive/llap/stats11.q.out | 4 +-
.../clientpositive/llap/subquery_scalar.q.out | 2 +-
.../results/clientpositive/llap/sysdb.q.out | 10 +-
ql/src/test/results/clientpositive/mm_all.q.out | 35 +-
.../results/clientpositive/nullgroup4.q.out | 41 +-
.../results/clientpositive/order_by_pos.q.out | 129 ++
.../parquet_array_null_element.q.out | 2 +-
.../results/clientpositive/parquet_create.q.out | 2 +-
.../parquet_int96_timestamp.q.out | 4 +-
.../clientpositive/part_inherit_tbl_props.q.out | 2 +-
.../part_inherit_tbl_props_empty.q.out | 2 +-
.../part_inherit_tbl_props_with_star.q.out | 2 +-
.../clientpositive/partial_column_stats.q.out | 2 +-
.../partition_coltype_literals.q.out | 8 +-
ql/src/test/results/clientpositive/pcr.q.out | 32 +-
.../results/clientpositive/perf/query16.q.out | 318 ++--
.../results/clientpositive/perf/query28.q.out | 58 +-
.../results/clientpositive/perf/query4.q.out | 1128 +++++++++++
.../results/clientpositive/perf/query47.q.out | 357 ++++
.../results/clientpositive/perf/query57.q.out | 351 ++++
.../results/clientpositive/perf/query74.q.out | 529 ++++++
.../results/clientpositive/perf/query94.q.out | 318 ++--
.../results/clientpositive/perf/query95.q.out | 302 +--
.../results/clientpositive/quotedid_stats.q.out | 2 +-
.../clientpositive/rand_partitionpruner2.q.out | 16 +-
.../clientpositive/rcfile_default_format.q.out | 6 +-
.../test/results/clientpositive/sample1.q.out | 16 +-
.../test/results/clientpositive/sample2.q.out | 16 +-
.../test/results/clientpositive/sample4.q.out | 16 +-
.../test/results/clientpositive/sample5.q.out | 16 +-
.../test/results/clientpositive/sample6.q.out | 16 +-
.../test/results/clientpositive/sample7.q.out | 16 +-
.../clientpositive/show_tblproperties.q.out | 10 +-
.../results/clientpositive/spark/bucket2.q.out | 4 +-
.../results/clientpositive/spark/bucket4.q.out | 4 +-
.../results/clientpositive/spark/bucket5.q.out | 8 +-
.../spark/bucket_map_join_spark1.q.out | 4 +-
.../spark/bucket_map_join_spark2.q.out | 4 +-
.../spark/bucket_map_join_spark3.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin5.q.out | 4 +-
.../spark/bucketmapjoin_negative.q.out | 4 +-
.../spark/bucketmapjoin_negative2.q.out | 4 +-
.../spark/disable_merge_for_bucketing.q.out | 4 +-
.../clientpositive/spark/groupby_map_ppr.q.out | 4 +-
.../spark/groupby_map_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/spark/groupby_ppr.q.out | 4 +-
.../spark/groupby_ppr_multi_distinct.q.out | 4 +-
.../spark/groupby_sort_1_23.q.out | 20 +-
.../spark/groupby_sort_skew_1_23.q.out | 20 +-
.../clientpositive/spark/input_part2.q.out | 8 +-
.../results/clientpositive/spark/join17.q.out | 4 +-
.../results/clientpositive/spark/join26.q.out | 4 +-
.../results/clientpositive/spark/join32.q.out | 4 +-
.../clientpositive/spark/join32_lessSize.q.out | 8 +-
.../results/clientpositive/spark/join33.q.out | 4 +-
.../results/clientpositive/spark/join34.q.out | 4 +-
.../results/clientpositive/spark/join35.q.out | 4 +-
.../results/clientpositive/spark/join9.q.out | 4 +-
.../clientpositive/spark/join_map_ppr.q.out | 4 +-
.../clientpositive/spark/nullgroup4.q.out | 31 +-
.../test/results/clientpositive/spark/pcr.q.out | 8 +-
.../spark/reduce_deduplicate.q.out | 4 +-
.../results/clientpositive/spark/sample1.q.out | 4 +-
.../results/clientpositive/spark/sample2.q.out | 4 +-
.../results/clientpositive/spark/sample4.q.out | 4 +-
.../results/clientpositive/spark/sample5.q.out | 4 +-
.../results/clientpositive/spark/sample6.q.out | 4 +-
.../results/clientpositive/spark/sample7.q.out | 4 +-
.../results/clientpositive/spark/stats0.q.out | 8 +-
.../results/clientpositive/spark/stats3.q.out | 2 +-
ql/src/test/results/clientpositive/stats0.q.out | 20 +-
ql/src/test/results/clientpositive/stats3.q.out | 2 +-
.../temp_table_display_colstats_tbllvl.q.out | 6 +-
.../results/clientpositive/tez/tez-tag.q.out | 301 +++
.../test/results/clientpositive/udf_count.q.out | 39 +-
.../clientpositive/unicode_notation.q.out | 6 +-
.../unset_table_view_property.q.out | 18 +-
.../clientpositive/vector_empty_where.q.out | 300 ++-
266 files changed, 13050 insertions(+), 2477 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index b13b0a4,ff8412c..e7bbd62
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@@ -1964,60 -1969,13 +1972,68 @@@ public class MetaStoreUtils
}
return metaException;
}
+
+ public static List<String> getColumnNames(List<FieldSchema> schema) {
+ List<String> cols = new ArrayList<>();
+ for (FieldSchema fs : schema) {
+ cols.add(fs.getName());
+ }
+ return cols;
+ }
+ // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
+ // right now ObjectStore.getAllMmTablesForCleanup is calling these method
+ /**
+ * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
+ * @param params table properties
+ * @return true if table is an INSERT_ONLY table, false otherwise
+ */
+ // TODO# also check that transactional is true
+ public static boolean isInsertOnlyTable(Map<String, String> params) {
+ return isInsertOnlyTable(params, false);
+ }
+
+ public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
+ String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
+ }
+
+ public static boolean isInsertOnlyTable(Properties params) {
+ // TODO# redirect for now - fix before merge
+ HashMap<String, String> testMap = new HashMap<String, String>();
+ for (String n : params.stringPropertyNames()) {
+ testMap.put(n, params.getProperty(n));
+ }
+ return isInsertOnlyTable(testMap);
+ }
+
+ /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
+ public static Boolean isToInsertOnlyTable(Map<String, String> props) {
+ // TODO# Setting these separately is a very hairy issue in certain combinations, since we
+ // cannot decide what type of table this becomes without taking both into account, and
+ // in many cases the conversion might be illegal.
+ // The only thing we allow is tx = true w/o tx-props, for backward compat.
+ String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+ String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ if (transactional == null && transactionalProp == null) return null; // Not affected.
+ boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
+ if (transactionalProp == null) {
+ if (isSetToTxn) return false; // Assume the full ACID table.
+ throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
+ + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
+ }
+ if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
+ if (!isSetToTxn) {
+ throw new RuntimeException("Cannot set '"
+ + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
+ + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
+ }
+ return true;
+ }
+
+ public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
+ boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
+ hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ return hasTxn || hasProps;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index e0302fb,b28ea73..9542990
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -63,7 -63,8 +63,8 @@@ import org.mockito.Mockito
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Supplier;
+ import com.google.common.collect.ImmutableList;
-
import javax.jdo.Query;
public class TestObjectStore {
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------
[02/36] hive git commit: HIVE-16788: ODBC call SQLForeignKeys leads
to NPE if you use PK arguments rather than FK arguments (Jesus Camacho
Rodriguez, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16788: ODBC call SQLForeignKeys leads to NPE if you use PK arguments rather than FK arguments (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/509308f6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/509308f6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/509308f6
Branch: refs/heads/hive-14535
Commit: 509308f642f4af8eb44a9fb7f0f105198df9fac6
Parents: 8aee8d4
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue May 30 20:20:19 2017 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 31 08:53:14 2017 +0100
----------------------------------------------------------------------
.../hadoop/hive/metastore/ObjectStore.java | 12 ++++-
.../hadoop/hive/metastore/TestObjectStore.java | 49 ++++++++++++++++++--
2 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/509308f6/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b16218d..4676e15 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -8539,7 +8539,17 @@ public class ObjectStore implements RawStore, Configurable {
final String parent_tbl_name = parent_tbl_name_input;
final String foreign_db_name = foreign_db_name_input;
final String foreign_tbl_name = foreign_tbl_name_input;
- return new GetListHelper<SQLForeignKey>(foreign_db_name, foreign_tbl_name, allowSql, allowJdo) {
+ final String db_name;
+ final String tbl_name;
+ if (foreign_tbl_name == null) {
+ // The FK table name might be null if we are retrieving the constraint from the PK side
+ db_name = parent_db_name_input;
+ tbl_name = parent_tbl_name_input;
+ } else {
+ db_name = foreign_db_name_input;
+ tbl_name = foreign_tbl_name_input;
+ }
+ return new GetListHelper<SQLForeignKey>(db_name, tbl_name, allowSql, allowJdo) {
@Override
protected List<SQLForeignKey> getSqlResult(GetHelper<List<SQLForeignKey>> ctx) throws MetaException {
http://git-wip-us.apache.org/repos/asf/hive/blob/509308f6/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 69e8826..b28ea73 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -63,6 +63,8 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableList;
+
import javax.jdo.Query;
public class TestObjectStore {
@@ -204,22 +206,63 @@ public class TestObjectStore {
public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
Database db1 = new Database(DB1, "description", "locationurl", null);
objectStore.createDatabase(db1);
- StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+ StorageDescriptor sd1 = new StorageDescriptor(ImmutableList.of(new FieldSchema("pk_col", "double", null)),
+ "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null),
+ null, null, null);
HashMap<String,String> params = new HashMap<String,String>();
params.put("EXTERNAL", "false");
- Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, null, null, "MANAGED_TABLE");
+ Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd1, null, params, null, null, "MANAGED_TABLE");
objectStore.createTable(tbl1);
List<String> tables = objectStore.getAllTables(DB1);
Assert.assertEquals(1, tables.size());
Assert.assertEquals(TABLE1, tables.get(0));
- Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, null, null, "MANAGED_TABLE");
+ StorageDescriptor sd2 = new StorageDescriptor(ImmutableList.of(new FieldSchema("fk_col", "double", null)),
+ "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null),
+ null, null, null);
+ Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd2, null, params, null, null, "MANAGED_TABLE");
objectStore.alterTable(DB1, TABLE1, newTbl1);
tables = objectStore.getTables(DB1, "new*");
Assert.assertEquals(1, tables.size());
Assert.assertEquals("new" + TABLE1, tables.get(0));
+ objectStore.createTable(tbl1);
+ tables = objectStore.getAllTables(DB1);
+ Assert.assertEquals(2, tables.size());
+
+ List<SQLForeignKey> foreignKeys = objectStore.getForeignKeys(DB1, TABLE1, null, null);
+ Assert.assertEquals(0, foreignKeys.size());
+
+ SQLPrimaryKey pk = new SQLPrimaryKey(DB1, TABLE1, "pk_col", 1,
+ "pk_const_1", false, false, false);
+ objectStore.addPrimaryKeys(ImmutableList.of(pk));
+ SQLForeignKey fk = new SQLForeignKey(DB1, TABLE1, "pk_col",
+ DB1, "new" + TABLE1, "fk_col", 1,
+ 0, 0, "fk_const_1", "pk_const_1", false, false, false);
+ objectStore.addForeignKeys(ImmutableList.of(fk));
+
+ // Retrieve from PK side
+ foreignKeys = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1);
+ Assert.assertEquals(1, foreignKeys.size());
+
+ List<SQLForeignKey> fks = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1);
+ if (fks != null) {
+ for (SQLForeignKey fkcol : fks) {
+ objectStore.dropConstraint(fkcol.getFktable_db(), fkcol.getFktable_name(), fkcol.getFk_name());
+ }
+ }
+ // Retrieve from FK side
+ foreignKeys = objectStore.getForeignKeys(DB1, TABLE1, null, null);
+ Assert.assertEquals(0, foreignKeys.size());
+ // Retrieve from PK side
+ foreignKeys = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1);
+ Assert.assertEquals(0, foreignKeys.size());
+
+ objectStore.dropTable(DB1, TABLE1);
+ tables = objectStore.getAllTables(DB1);
+ Assert.assertEquals(1, tables.size());
+
objectStore.dropTable(DB1, "new" + TABLE1);
tables = objectStore.getAllTables(DB1);
Assert.assertEquals(0, tables.size());
[18/36] hive git commit: HIVE-16744 : LLAP index update may be broken
after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Posted by we...@apache.org.
HIVE-16744 : LLAP index update may be broken after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/019ad8c8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/019ad8c8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/019ad8c8
Branch: refs/heads/hive-14535
Commit: 019ad8c822786b1bf195bcf0d4be94eec0449dda
Parents: 788d486
Author: sergey <se...@apache.org>
Authored: Fri Jun 2 13:09:08 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Jun 2 13:17:01 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/019ad8c8/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 9693826..7081140 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -394,7 +394,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}",
stripeKey.stripeIx, DebugUtils.toString(globalIncludes));
}
- assert isFoundInCache;
+ assert isFoundInCache; // If it's not fresh from the cache, indexes should be present.
counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
ensureMetadataReader();
updateLoadedIndexes(stripeMetadata, stripe, globalIncludes, sargColumns);
http://git-wip-us.apache.org/repos/asf/hive/blob/019ad8c8/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
index 1f3f7ea..96c4cce 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java
@@ -113,7 +113,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
public void loadMissingIndexes(DataReader mr, StripeInformation stripe, boolean[] includes,
boolean[] sargColumns) throws IOException {
- // Do not loose the old indexes. Create a super set includes
+ // Do not lose the old indexes. Create a super set includes
OrcProto.RowIndex[] existing = getRowIndexes();
boolean superset[] = new boolean[Math.max(existing.length, includes.length)];
for (int i = 0; i < includes.length; i++) {
@@ -123,7 +123,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt
superset[i] = superset[i] || (existing[i] != null);
}
// TODO: should we save footer to avoid a read here?
- rowIndex = mr.readRowIndex(stripe, schema, null, true, includes,
+ rowIndex = mr.readRowIndex(stripe, schema, null, true, superset,
rowIndex.getRowGroupIndex(),
sargColumns, writerVersion, rowIndex.getBloomFilterKinds(),
rowIndex.getBloomFilterIndex());
[16/36] hive git commit: HIVE-16799: Control the max number of task
for a stage in a spark job (Reviewed by Rui)
Posted by we...@apache.org.
HIVE-16799: Control the max number of task for a stage in a spark job (Reviewed by Rui)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/363ffe0a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/363ffe0a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/363ffe0a
Branch: refs/heads/hive-14535
Commit: 363ffe0ac7dec7e4804c1eb2ba76cb07660ae020
Parents: b560f49
Author: Xuefu Zhang <xu...@uber.com>
Authored: Fri Jun 2 11:26:33 2017 -0700
Committer: Xuefu Zhang <xu...@uber.com>
Committed: Fri Jun 2 11:26:33 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../test/resources/testconfiguration.properties | 3 +-
.../hadoop/hive/cli/control/CliConfigs.java | 1 +
.../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +-
.../spark/status/RemoteSparkJobMonitor.java | 14 ++++
.../ql/exec/spark/status/SparkJobMonitor.java | 12 +++
.../clientnegative/spark_stage_max_tasks.q | 6 ++
.../spark/spark_stage_max_tasks.q.out | 77 ++++++++++++++++++++
8 files changed, 115 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 176d36f..fce8db3 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3371,6 +3371,8 @@ public class HiveConf extends Configuration {
"Turn this off when there is a memory issue."),
SPARK_JOB_MAX_TASKS("hive.spark.job.max.tasks", -1, "The maximum number of tasks a Spark job may have.\n" +
"If a Spark job contains more tasks than the maximum, it will be cancelled. A value of -1 means no limit."),
+ SPARK_STAGE_MAX_TASKS("hive.spark.stage.max.tasks", -1, "The maximum number of tasks a stage in a Spark job may have.\n" +
+ "If a Spark job stage contains more tasks than the maximum, the job will be cancelled. A value of -1 means no limit."),
NWAYJOINREORDER("hive.reorder.nway.joins", true,
"Runs reordering of tables within single n-way join (i.e.: picks streamtable)"),
HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", true,
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index e613374..62462bd 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1459,5 +1459,6 @@ spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
groupby3_map_skew_multi_distinct.q,\
groupby3_multi_distinct.q,\
groupby_grouping_sets7.q,\
- spark_job_max_tasks.q
+ spark_job_max_tasks.q,\
+ spark_stage_max_tasks.q
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 1457db0..27b87fb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -288,6 +288,7 @@ public class CliConfigs {
excludesFrom(testConfigProps, "minimr.query.negative.files");
excludeQuery("authorization_uri_import.q");
excludeQuery("spark_job_max_tasks.q");
+ excludeQuery("spark_stage_max_tasks.q");
setResultsDir("ql/src/test/results/clientnegative");
setLogDir("itests/qtest/target/qfile-results/clientnegative");
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index b4fb49f..2ee8c93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -138,7 +138,7 @@ public class SparkTask extends Task<SparkWork> {
LOG.info("Failed to submit Spark job " + sparkJobID);
killJob();
} else if (rc == 4) {
- LOG.info("The number of tasks reaches above the limit " + conf.getIntVar(HiveConf.ConfVars.SPARK_JOB_MAX_TASKS) +
+ LOG.info("The spark job or one stage of it has too many tasks" +
". Cancelling Spark job " + sparkJobID + " with application ID " + jobID );
killJob();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
index 9dfb65e..37b8363 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
@@ -35,7 +35,9 @@ import org.apache.spark.JobExecutionStatus;
*/
public class RemoteSparkJobMonitor extends SparkJobMonitor {
private int sparkJobMaxTaskCount = -1;
+ private int sparkStageMaxTaskCount = -1;
private int totalTaskCount = 0;
+ private int stageMaxTaskCount = 0;
private RemoteSparkJobStatus sparkJobStatus;
private final HiveConf hiveConf;
@@ -44,6 +46,7 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
this.sparkJobStatus = sparkJobStatus;
this.hiveConf = hiveConf;
sparkJobMaxTaskCount = hiveConf.getIntVar(HiveConf.ConfVars.SPARK_JOB_MAX_TASKS);
+ sparkStageMaxTaskCount = hiveConf.getIntVar(HiveConf.ConfVars.SPARK_STAGE_MAX_TASKS);
}
@Override
@@ -103,6 +106,17 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
console.logInfo(format);
}
} else {
+ // Get the maximum of the number of tasks in the stages of the job and cancel the job if it goes beyond the limit.
+ if (sparkStageMaxTaskCount != -1 && stageMaxTaskCount == 0) {
+ stageMaxTaskCount = getStageMaxTaskCount(progressMap);
+ if (stageMaxTaskCount > sparkStageMaxTaskCount) {
+ rc = 4;
+ done = true;
+ console.printInfo("\nThe number of task in one stage of the Spark job [" + stageMaxTaskCount + "] is greater than the limit [" +
+ sparkStageMaxTaskCount + "]. The Spark job will be cancelled.");
+ }
+ }
+
// Count the number of tasks, and kill application if it goes beyond the limit.
if (sparkJobMaxTaskCount != -1 && totalTaskCount == 0) {
totalTaskCount = getTotalTaskCount(progressMap);
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index 41730b5..078a57d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -181,6 +181,18 @@ abstract class SparkJobMonitor {
return totalTasks;
}
+ protected int getStageMaxTaskCount(Map<String, SparkStageProgress> progressMap) {
+ int stageMaxTasks = 0;
+ for (SparkStageProgress progress: progressMap.values() ) {
+ int tasks = progress.getTotalTaskCount();
+ if (tasks > stageMaxTasks) {
+ stageMaxTasks = tasks;
+ }
+ }
+
+ return stageMaxTasks;
+ }
+
private String getReport(Map<String, SparkStageProgress> progressMap) {
StringBuilder reportBuffer = new StringBuilder();
SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/ql/src/test/queries/clientnegative/spark_stage_max_tasks.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/spark_stage_max_tasks.q b/ql/src/test/queries/clientnegative/spark_stage_max_tasks.q
new file mode 100644
index 0000000..5bdb014
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/spark_stage_max_tasks.q
@@ -0,0 +1,6 @@
+set hive.spark.stage.max.tasks=1;
+
+EXPLAIN
+SELECT key, sum(value) AS s FROM src1 GROUP BY key ORDER BY s;
+
+SELECT key, sum(value) AS s FROM src1 GROUP BY key ORDER BY s;
http://git-wip-us.apache.org/repos/asf/hive/blob/363ffe0a/ql/src/test/results/clientnegative/spark/spark_stage_max_tasks.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/spark/spark_stage_max_tasks.q.out b/ql/src/test/results/clientnegative/spark/spark_stage_max_tasks.q.out
new file mode 100644
index 0000000..ba2f09e
--- /dev/null
+++ b/ql/src/test/results/clientnegative/spark/spark_stage_max_tasks.q.out
@@ -0,0 +1,77 @@
+PREHOOK: query: EXPLAIN
+SELECT key, sum(value) AS s FROM src1 GROUP BY key ORDER BY s
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT key, sum(value) AS s FROM src1 GROUP BY key ORDER BY s
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (SORT, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src1
+ Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(value)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double)
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: double)
+ sort order: +
+ Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT key, sum(value) AS s FROM src1 GROUP BY key ORDER BY s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+#### A masked pattern was here ####
+FAILED: Execution Error, return code 4 from org.apache.hadoop.hive.ql.exec.spark.SparkTask
[31/36] hive git commit: HIVE-16775 : Fix
HiveFilterAggregateTransposeRule when filter is always false (Pengcheng Xiong
via Jesus Camacho Rodriguez)
Posted by we...@apache.org.
HIVE-16775 : Fix HiveFilterAggregateTransposeRule when filter is always false (Pengcheng Xiong via Jesus Camacho Rodriguez)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b754cb7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b754cb7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b754cb7d
Branch: refs/heads/hive-14535
Commit: b754cb7dc26fda6eb4d89a17b92cd76ca3f87b97
Parents: 9a69169
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Sat Jun 3 13:08:18 2017 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 3 13:08:18 2017 -0700
----------------------------------------------------------------------
.../rules/HiveFilterAggregateTransposeRule.java | 6 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +-
.../test/queries/clientpositive/filter_aggr.q | 18 +
.../test/queries/clientpositive/perf/query4.q | 111 ++
.../test/queries/clientpositive/perf/query74.q | 63 +
.../results/clientpositive/filter_aggr.q.out | 680 +++++++++++
.../results/clientpositive/perf/query4.q.out | 1128 ++++++++++++++++++
.../results/clientpositive/perf/query74.q.out | 529 ++++++++
8 files changed, 2534 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
index 0e5c731..89293a7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
@@ -20,16 +20,16 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.RelFactories.FilterFactory;
import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
public class HiveFilterAggregateTransposeRule extends FilterAggregateTransposeRule {
public HiveFilterAggregateTransposeRule(Class<? extends Filter> filterClass,
- FilterFactory filterFactory, Class<? extends Aggregate> aggregateClass) {
- super(filterClass, filterFactory, aggregateClass);
+ RelBuilderFactory builderFactory, Class<? extends Aggregate> aggregateClass) {
+ super(filterClass, builderFactory, aggregateClass);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 7f583ed..348331e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1670,7 +1670,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
rules.add(HiveFilterSortTransposeRule.INSTANCE);
rules.add(HiveFilterJoinRule.JOIN);
rules.add(HiveFilterJoinRule.FILTER_ON_JOIN);
- rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class));
+ rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_BUILDER,
+ Aggregate.class));
rules.add(new FilterMergeRule(HiveRelFactories.HIVE_BUILDER));
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_REDUCE_WITH_STATS)) {
rules.add(HiveReduceExpressionsWithStatsRule.INSTANCE);
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/filter_aggr.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/filter_aggr.q b/ql/src/test/queries/clientpositive/filter_aggr.q
new file mode 100644
index 0000000..44e088b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/filter_aggr.q
@@ -0,0 +1,18 @@
+set hive.mapred.mode=nonstrict;
+
+explain extended
+select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1;
+
+select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/perf/query4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/perf/query4.q b/ql/src/test/queries/clientpositive/perf/query4.q
new file mode 100644
index 0000000..631a464
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/perf/query4.q
@@ -0,0 +1,111 @@
+set hive.mapred.mode=nonstrict;
+-- start query 1 in stream 0 using template query4.tpl and seed 1819994127
+explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total
+ ,'c' sale_type
+ from customer
+ ,catalog_sales
+ ,date_dim
+ where c_customer_sk = cs_bill_customer_sk
+ and cs_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,c_preferred_cust_flag customer_preferred_cust_flag
+ ,c_birth_country customer_birth_country
+ ,c_login customer_login
+ ,c_email_address customer_email_address
+ ,d_year dyear
+ ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,c_preferred_cust_flag
+ ,c_birth_country
+ ,c_login
+ ,c_email_address
+ ,d_year
+ )
+ select t_s_secyear.customer_preferred_cust_flag
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_c_firstyear
+ ,year_total t_c_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_c_secyear.customer_id
+ and t_s_firstyear.customer_id = t_c_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_c_firstyear.sale_type = 'c'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_c_secyear.sale_type = 'c'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.dyear = 2001
+ and t_s_secyear.dyear = 2001+1
+ and t_c_firstyear.dyear = 2001
+ and t_c_secyear.dyear = 2001+1
+ and t_w_firstyear.dyear = 2001
+ and t_w_secyear.dyear = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_c_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end
+ > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ order by t_s_secyear.customer_preferred_cust_flag
+limit 100;
+
+-- end query 1 in stream 0 using template query4.tpl
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/perf/query74.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/perf/query74.q b/ql/src/test/queries/clientpositive/perf/query74.q
new file mode 100644
index 0000000..b25db9c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/perf/query74.q
@@ -0,0 +1,63 @@
+set hive.mapred.mode=nonstrict;
+-- start query 1 in stream 0 using template query74.tpl and seed 1556717815
+explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ss_net_paid) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ws_net_paid) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ )
+ select
+ t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.year = 2001
+ and t_s_secyear.year = 2001+1
+ and t_w_firstyear.year = 2001
+ and t_w_secyear.year = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ order by 2,1,3
+limit 100;
+
+-- end query 1 in stream 0 using template query74.tpl
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/results/clientpositive/filter_aggr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/filter_aggr.q.out b/ql/src/test/results/clientpositive/filter_aggr.q.out
new file mode 100644
index 0000000..db7dcae
--- /dev/null
+++ b/ql/src/test/results/clientpositive/filter_aggr.q.out
@@ -0,0 +1,680 @@
+PREHOOK: query: explain extended
+select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended
+select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-3
+ Stage-3 is a root stage
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [null-subquery1:$hdt$_0-subquery1:src]
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint), 1 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ GatherStats: false
+ Union
+ Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2
+ columns.types string:bigint:int
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ TableScan
+ GatherStats: false
+ Union
+ Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ columns _col0,_col1,_col2
+ columns.types string:bigint:int
+ escape.delim \
+ hive.serialization.extend.additional.nesting.levels true
+ serialization.escape.crlf true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10004
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+#### A masked pattern was here ####
+ Partition
+ base file name: -mr-10005
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Truncated Path -> Alias:
+#### A masked pattern was here ####
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: false (type: boolean)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col1 (type: bigint)
+ auto parallelism: false
+ Path -> Alias:
+ nullscan://null/default.src/part_ [null-subquery2:$hdt$_0-subquery2:src]
+ Path -> Partition:
+ nullscan://null/default.src/part_
+ Partition
+ input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.NullStructSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ column.name.delimiter ,
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ nullscan://null/default.src/part_ [null-subquery2:$hdt$_0-subquery2:src]
+ Needs Tagging: false
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint), 2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ properties:
+ column.name.delimiter ,
+ columns _col0,_col1,_col2
+ columns.types string,bigint,int
+ escape.delim \
+ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key, c, m from
+(
+select key, c, 1 as m from (select key, count(key) as c from src group by key)s1
+union all
+select key, c, 2 as m from (select key, count(key) as c from src group by key)s2
+)sub
+where m = 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 3 1
+10 1 1
+100 2 1
+103 2 1
+104 2 1
+105 1 1
+11 1 1
+111 1 1
+113 2 1
+114 1 1
+116 1 1
+118 2 1
+119 3 1
+12 2 1
+120 2 1
+125 2 1
+126 1 1
+128 3 1
+129 2 1
+131 1 1
+133 1 1
+134 2 1
+136 1 1
+137 2 1
+138 4 1
+143 1 1
+145 1 1
+146 2 1
+149 2 1
+15 2 1
+150 1 1
+152 2 1
+153 1 1
+155 1 1
+156 1 1
+157 1 1
+158 1 1
+160 1 1
+162 1 1
+163 1 1
+164 2 1
+165 2 1
+166 1 1
+167 3 1
+168 1 1
+169 4 1
+17 1 1
+170 1 1
+172 2 1
+174 2 1
+175 2 1
+176 2 1
+177 1 1
+178 1 1
+179 2 1
+18 2 1
+180 1 1
+181 1 1
+183 1 1
+186 1 1
+187 3 1
+189 1 1
+19 1 1
+190 1 1
+191 2 1
+192 1 1
+193 3 1
+194 1 1
+195 2 1
+196 1 1
+197 2 1
+199 3 1
+2 1 1
+20 1 1
+200 2 1
+201 1 1
+202 1 1
+203 2 1
+205 2 1
+207 2 1
+208 3 1
+209 2 1
+213 2 1
+214 1 1
+216 2 1
+217 2 1
+218 1 1
+219 2 1
+221 2 1
+222 1 1
+223 2 1
+224 2 1
+226 1 1
+228 1 1
+229 2 1
+230 5 1
+233 2 1
+235 1 1
+237 2 1
+238 2 1
+239 2 1
+24 2 1
+241 1 1
+242 2 1
+244 1 1
+247 1 1
+248 1 1
+249 1 1
+252 1 1
+255 2 1
+256 2 1
+257 1 1
+258 1 1
+26 2 1
+260 1 1
+262 1 1
+263 1 1
+265 2 1
+266 1 1
+27 1 1
+272 2 1
+273 3 1
+274 1 1
+275 1 1
+277 4 1
+278 2 1
+28 1 1
+280 2 1
+281 2 1
+282 2 1
+283 1 1
+284 1 1
+285 1 1
+286 1 1
+287 1 1
+288 2 1
+289 1 1
+291 1 1
+292 1 1
+296 1 1
+298 3 1
+30 1 1
+302 1 1
+305 1 1
+306 1 1
+307 2 1
+308 1 1
+309 2 1
+310 1 1
+311 3 1
+315 1 1
+316 3 1
+317 2 1
+318 3 1
+321 2 1
+322 2 1
+323 1 1
+325 2 1
+327 3 1
+33 1 1
+331 2 1
+332 1 1
+333 2 1
+335 1 1
+336 1 1
+338 1 1
+339 1 1
+34 1 1
+341 1 1
+342 2 1
+344 2 1
+345 1 1
+348 5 1
+35 3 1
+351 1 1
+353 2 1
+356 1 1
+360 1 1
+362 1 1
+364 1 1
+365 1 1
+366 1 1
+367 2 1
+368 1 1
+369 3 1
+37 2 1
+373 1 1
+374 1 1
+375 1 1
+377 1 1
+378 1 1
+379 1 1
+382 2 1
+384 3 1
+386 1 1
+389 1 1
+392 1 1
+393 1 1
+394 1 1
+395 2 1
+396 3 1
+397 2 1
+399 2 1
+4 1 1
+400 1 1
+401 5 1
+402 1 1
+403 3 1
+404 2 1
+406 4 1
+407 1 1
+409 3 1
+41 1 1
+411 1 1
+413 2 1
+414 2 1
+417 3 1
+418 1 1
+419 1 1
+42 2 1
+421 1 1
+424 2 1
+427 1 1
+429 2 1
+43 1 1
+430 3 1
+431 3 1
+432 1 1
+435 1 1
+436 1 1
+437 1 1
+438 3 1
+439 2 1
+44 1 1
+443 1 1
+444 1 1
+446 1 1
+448 1 1
+449 1 1
+452 1 1
+453 1 1
+454 3 1
+455 1 1
+457 1 1
+458 2 1
+459 2 1
+460 1 1
+462 2 1
+463 2 1
+466 3 1
+467 1 1
+468 4 1
+469 5 1
+47 1 1
+470 1 1
+472 1 1
+475 1 1
+477 1 1
+478 2 1
+479 1 1
+480 3 1
+481 1 1
+482 1 1
+483 1 1
+484 1 1
+485 1 1
+487 1 1
+489 4 1
+490 1 1
+491 1 1
+492 2 1
+493 1 1
+494 1 1
+495 1 1
+496 1 1
+497 1 1
+498 3 1
+5 3 1
+51 2 1
+53 1 1
+54 1 1
+57 1 1
+58 2 1
+64 1 1
+65 1 1
+66 1 1
+67 2 1
+69 1 1
+70 3 1
+72 2 1
+74 1 1
+76 2 1
+77 1 1
+78 1 1
+8 1 1
+80 1 1
+82 1 1
+83 2 1
+84 2 1
+85 1 1
+86 1 1
+87 1 1
+9 1 1
+90 3 1
+92 1 1
+95 2 1
+96 1 1
+97 2 1
+98 2 1
[05/36] hive git commit: HIVE-16625: Extra '\0' characters in the
output,
when SeparatedValuesOutputFormat is used and the quoting is disabled (Peter
Vary, reviewed by Zoltan Haindrich and Aihua Xu)
Posted by we...@apache.org.
HIVE-16625: Extra '\0' characters in the output, when SeparatedValuesOutputFormat is used and the quoting is disabled (Peter Vary, reviewed by Zoltan Haindrich and Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/396cba45
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/396cba45
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/396cba45
Branch: refs/heads/hive-14535
Commit: 396cba45822ab9e1519504cdde82df1d89b60c8f
Parents: f5b2250
Author: Aihua Xu <ai...@apache.org>
Authored: Wed May 31 13:00:04 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Wed May 31 13:01:47 2017 -0400
----------------------------------------------------------------------
.../beeline/SeparatedValuesOutputFormat.java | 11 +-
.../test/resources/testconfiguration.properties | 1 +
.../java/org/apache/hive/beeline/QFile.java | 1 +
.../insert_overwrite_local_directory_1.q.out | 1769 ++++++++++++++++++
4 files changed, 1780 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
index 66d9fd0..9c24a20 100644
--- a/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
+++ b/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
@@ -26,6 +26,8 @@ import java.io.IOException;
import java.io.StringWriter;
import org.apache.hadoop.io.IOUtils;
+import org.supercsv.encoder.CsvEncoder;
+import org.supercsv.encoder.SelectiveCsvEncoder;
import org.supercsv.io.CsvListWriter;
import org.supercsv.prefs.CsvPreference;
@@ -40,10 +42,15 @@ class SeparatedValuesOutputFormat implements OutputFormat {
SeparatedValuesOutputFormat(BeeLine beeLine, char separator) {
this.beeLine = beeLine;
- unquotedCsvPreference = new CsvPreference.Builder('\0', separator, "").build();
+ unquotedCsvPreference = getUnquotedCsvPreference(separator);
quotedCsvPreference = new CsvPreference.Builder('"', separator, "").build();
}
+ private static CsvPreference getUnquotedCsvPreference(char delimiter) {
+ CsvEncoder noEncoder = new SelectiveCsvEncoder();
+ return new CsvPreference.Builder('\0', delimiter, "").useEncoder(noEncoder).build();
+ }
+
private void updateCsvPreference() {
if (beeLine.getOpts().getOutputFormat().equals("dsv")) {
// check whether delimiter changed by user
@@ -54,7 +61,7 @@ class SeparatedValuesOutputFormat implements OutputFormat {
// "" is passed as the end of line symbol in following function, as
// beeline itself adds newline
if (isQuotingDisabled()) {
- unquotedCsvPreference = new CsvPreference.Builder('\0', newDel, "").build();
+ unquotedCsvPreference = getUnquotedCsvPreference(newDel);
} else {
quotedCsvPreference = new CsvPreference.Builder('"', newDel, "").build();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 47a13c9..f4a53df 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -759,6 +759,7 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
beeline.positive.include=create_merge_compressed.q,\
drop_with_concurrency.q,\
escape_comments.q,\
+ insert_overwrite_local_directory_1.q,\
mapjoin2.q,\
materialized_view_create_rewrite.q,\
smb_mapjoin_1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
index 0d00a9b..e70ac38 100644
--- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -304,6 +304,7 @@ public final class QFile {
.addFilter(".*/tmp/.*\n", MASK_PATTERN)
.addFilter(".*file:.*\n", MASK_PATTERN)
.addFilter(".*file\\..*\n", MASK_PATTERN)
+ .addFilter(".*Output:.*/data/files/.*\n", MASK_PATTERN)
.addFilter(".*CreateTime.*\n", MASK_PATTERN)
.addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
.addFilter(".*lastUpdateTime.*\n", MASK_PATTERN)
http://git-wip-us.apache.org/repos/asf/hive/blob/396cba45/ql/src/test/results/clientpositive/beeline/insert_overwrite_local_directory_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/beeline/insert_overwrite_local_directory_1.q.out b/ql/src/test/results/clientpositive/beeline/insert_overwrite_local_directory_1.q.out
new file mode 100644
index 0000000..9a4e861
--- /dev/null
+++ b/ql/src/test/results/clientpositive/beeline/insert_overwrite_local_directory_1.q.out
@@ -0,0 +1,1769 @@
+PREHOOK: query: insert overwrite local directory '../../data/files/local_src_table_1'
+select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_src_table_1'
+select * from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238val_238
+86val_86
+311val_311
+27val_27
+165val_165
+409val_409
+255val_255
+278val_278
+98val_98
+484val_484
+265val_265
+193val_193
+401val_401
+150val_150
+273val_273
+224val_224
+369val_369
+66val_66
+128val_128
+213val_213
+146val_146
+406val_406
+429val_429
+374val_374
+152val_152
+469val_469
+145val_145
+495val_495
+37val_37
+327val_327
+281val_281
+277val_277
+209val_209
+15val_15
+82val_82
+403val_403
+166val_166
+417val_417
+430val_430
+252val_252
+292val_292
+219val_219
+287val_287
+153val_153
+193val_193
+338val_338
+446val_446
+459val_459
+394val_394
+237val_237
+482val_482
+174val_174
+413val_413
+494val_494
+207val_207
+199val_199
+466val_466
+208val_208
+174val_174
+399val_399
+396val_396
+247val_247
+417val_417
+489val_489
+162val_162
+377val_377
+397val_397
+309val_309
+365val_365
+266val_266
+439val_439
+342val_342
+367val_367
+325val_325
+167val_167
+195val_195
+475val_475
+17val_17
+113val_113
+155val_155
+203val_203
+339val_339
+0val_0
+455val_455
+128val_128
+311val_311
+316val_316
+57val_57
+302val_302
+205val_205
+149val_149
+438val_438
+345val_345
+129val_129
+170val_170
+20val_20
+489val_489
+157val_157
+378val_378
+221val_221
+92val_92
+111val_111
+47val_47
+72val_72
+4val_4
+280val_280
+35val_35
+427val_427
+277val_277
+208val_208
+356val_356
+399val_399
+169val_169
+382val_382
+498val_498
+125val_125
+386val_386
+437val_437
+469val_469
+192val_192
+286val_286
+187val_187
+176val_176
+54val_54
+459val_459
+51val_51
+138val_138
+103val_103
+239val_239
+213val_213
+216val_216
+430val_430
+278val_278
+176val_176
+289val_289
+221val_221
+65val_65
+318val_318
+332val_332
+311val_311
+275val_275
+137val_137
+241val_241
+83val_83
+333val_333
+180val_180
+284val_284
+12val_12
+230val_230
+181val_181
+67val_67
+260val_260
+404val_404
+384val_384
+489val_489
+353val_353
+373val_373
+272val_272
+138val_138
+217val_217
+84val_84
+348val_348
+466val_466
+58val_58
+8val_8
+411val_411
+230val_230
+208val_208
+348val_348
+24val_24
+463val_463
+431val_431
+179val_179
+172val_172
+42val_42
+129val_129
+158val_158
+119val_119
+496val_496
+0val_0
+322val_322
+197val_197
+468val_468
+393val_393
+454val_454
+100val_100
+298val_298
+199val_199
+191val_191
+418val_418
+96val_96
+26val_26
+165val_165
+327val_327
+230val_230
+205val_205
+120val_120
+131val_131
+51val_51
+404val_404
+43val_43
+436val_436
+156val_156
+469val_469
+468val_468
+308val_308
+95val_95
+196val_196
+288val_288
+481val_481
+457val_457
+98val_98
+282val_282
+197val_197
+187val_187
+318val_318
+318val_318
+409val_409
+470val_470
+137val_137
+369val_369
+316val_316
+169val_169
+413val_413
+85val_85
+77val_77
+0val_0
+490val_490
+87val_87
+364val_364
+179val_179
+118val_118
+134val_134
+395val_395
+282val_282
+138val_138
+238val_238
+419val_419
+15val_15
+118val_118
+72val_72
+90val_90
+307val_307
+19val_19
+435val_435
+10val_10
+277val_277
+273val_273
+306val_306
+224val_224
+309val_309
+389val_389
+327val_327
+242val_242
+369val_369
+392val_392
+272val_272
+331val_331
+401val_401
+242val_242
+452val_452
+177val_177
+226val_226
+5val_5
+497val_497
+402val_402
+396val_396
+317val_317
+395val_395
+58val_58
+35val_35
+336val_336
+95val_95
+11val_11
+168val_168
+34val_34
+229val_229
+233val_233
+143val_143
+472val_472
+322val_322
+498val_498
+160val_160
+195val_195
+42val_42
+321val_321
+430val_430
+119val_119
+489val_489
+458val_458
+78val_78
+76val_76
+41val_41
+223val_223
+492val_492
+149val_149
+449val_449
+218val_218
+228val_228
+138val_138
+453val_453
+30val_30
+209val_209
+64val_64
+468val_468
+76val_76
+74val_74
+342val_342
+69val_69
+230val_230
+33val_33
+368val_368
+103val_103
+296val_296
+113val_113
+216val_216
+367val_367
+344val_344
+167val_167
+274val_274
+219val_219
+239val_239
+485val_485
+116val_116
+223val_223
+256val_256
+263val_263
+70val_70
+487val_487
+480val_480
+401val_401
+288val_288
+191val_191
+5val_5
+244val_244
+438val_438
+128val_128
+467val_467
+432val_432
+202val_202
+316val_316
+229val_229
+469val_469
+463val_463
+280val_280
+2val_2
+35val_35
+283val_283
+331val_331
+235val_235
+80val_80
+44val_44
+193val_193
+321val_321
+335val_335
+104val_104
+466val_466
+366val_366
+175val_175
+403val_403
+483val_483
+53val_53
+105val_105
+257val_257
+406val_406
+409val_409
+190val_190
+406val_406
+401val_401
+114val_114
+258val_258
+90val_90
+203val_203
+262val_262
+348val_348
+424val_424
+12val_12
+396val_396
+201val_201
+217val_217
+164val_164
+431val_431
+454val_454
+478val_478
+298val_298
+125val_125
+431val_431
+164val_164
+424val_424
+187val_187
+382val_382
+5val_5
+70val_70
+397val_397
+480val_480
+291val_291
+24val_24
+351val_351
+255val_255
+104val_104
+70val_70
+163val_163
+438val_438
+119val_119
+414val_414
+200val_200
+491val_491
+237val_237
+439val_439
+360val_360
+248val_248
+479val_479
+305val_305
+417val_417
+199val_199
+444val_444
+120val_120
+429val_429
+169val_169
+443val_443
+323val_323
+325val_325
+277val_277
+230val_230
+478val_478
+178val_178
+468val_468
+310val_310
+317val_317
+333val_333
+493val_493
+460val_460
+207val_207
+249val_249
+265val_265
+480val_480
+83val_83
+136val_136
+353val_353
+172val_172
+214val_214
+462val_462
+233val_233
+406val_406
+133val_133
+175val_175
+189val_189
+454val_454
+375val_375
+401val_401
+421val_421
+407val_407
+384val_384
+256val_256
+26val_26
+134val_134
+67val_67
+384val_384
+379val_379
+18val_18
+462val_462
+492val_492
+100val_100
+298val_298
+9val_9
+341val_341
+498val_498
+146val_146
+458val_458
+362val_362
+186val_186
+285val_285
+348val_348
+167val_167
+18val_18
+273val_273
+183val_183
+281val_281
+344val_344
+97val_97
+469val_469
+315val_315
+84val_84
+28val_28
+37val_37
+448val_448
+152val_152
+348val_348
+307val_307
+194val_194
+414val_414
+477val_477
+222val_222
+126val_126
+90val_90
+169val_169
+403val_403
+400val_400
+200val_200
+97val_97
+PREHOOK: query: insert overwrite local directory '../../data/files/local_src_table_2'
+row format delimited
+FIELDS TERMINATED BY ':'
+select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_src_table_2'
+row format delimited
+FIELDS TERMINATED BY ':'
+select * from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+238:val_238
+86:val_86
+311:val_311
+27:val_27
+165:val_165
+409:val_409
+255:val_255
+278:val_278
+98:val_98
+484:val_484
+265:val_265
+193:val_193
+401:val_401
+150:val_150
+273:val_273
+224:val_224
+369:val_369
+66:val_66
+128:val_128
+213:val_213
+146:val_146
+406:val_406
+429:val_429
+374:val_374
+152:val_152
+469:val_469
+145:val_145
+495:val_495
+37:val_37
+327:val_327
+281:val_281
+277:val_277
+209:val_209
+15:val_15
+82:val_82
+403:val_403
+166:val_166
+417:val_417
+430:val_430
+252:val_252
+292:val_292
+219:val_219
+287:val_287
+153:val_153
+193:val_193
+338:val_338
+446:val_446
+459:val_459
+394:val_394
+237:val_237
+482:val_482
+174:val_174
+413:val_413
+494:val_494
+207:val_207
+199:val_199
+466:val_466
+208:val_208
+174:val_174
+399:val_399
+396:val_396
+247:val_247
+417:val_417
+489:val_489
+162:val_162
+377:val_377
+397:val_397
+309:val_309
+365:val_365
+266:val_266
+439:val_439
+342:val_342
+367:val_367
+325:val_325
+167:val_167
+195:val_195
+475:val_475
+17:val_17
+113:val_113
+155:val_155
+203:val_203
+339:val_339
+0:val_0
+455:val_455
+128:val_128
+311:val_311
+316:val_316
+57:val_57
+302:val_302
+205:val_205
+149:val_149
+438:val_438
+345:val_345
+129:val_129
+170:val_170
+20:val_20
+489:val_489
+157:val_157
+378:val_378
+221:val_221
+92:val_92
+111:val_111
+47:val_47
+72:val_72
+4:val_4
+280:val_280
+35:val_35
+427:val_427
+277:val_277
+208:val_208
+356:val_356
+399:val_399
+169:val_169
+382:val_382
+498:val_498
+125:val_125
+386:val_386
+437:val_437
+469:val_469
+192:val_192
+286:val_286
+187:val_187
+176:val_176
+54:val_54
+459:val_459
+51:val_51
+138:val_138
+103:val_103
+239:val_239
+213:val_213
+216:val_216
+430:val_430
+278:val_278
+176:val_176
+289:val_289
+221:val_221
+65:val_65
+318:val_318
+332:val_332
+311:val_311
+275:val_275
+137:val_137
+241:val_241
+83:val_83
+333:val_333
+180:val_180
+284:val_284
+12:val_12
+230:val_230
+181:val_181
+67:val_67
+260:val_260
+404:val_404
+384:val_384
+489:val_489
+353:val_353
+373:val_373
+272:val_272
+138:val_138
+217:val_217
+84:val_84
+348:val_348
+466:val_466
+58:val_58
+8:val_8
+411:val_411
+230:val_230
+208:val_208
+348:val_348
+24:val_24
+463:val_463
+431:val_431
+179:val_179
+172:val_172
+42:val_42
+129:val_129
+158:val_158
+119:val_119
+496:val_496
+0:val_0
+322:val_322
+197:val_197
+468:val_468
+393:val_393
+454:val_454
+100:val_100
+298:val_298
+199:val_199
+191:val_191
+418:val_418
+96:val_96
+26:val_26
+165:val_165
+327:val_327
+230:val_230
+205:val_205
+120:val_120
+131:val_131
+51:val_51
+404:val_404
+43:val_43
+436:val_436
+156:val_156
+469:val_469
+468:val_468
+308:val_308
+95:val_95
+196:val_196
+288:val_288
+481:val_481
+457:val_457
+98:val_98
+282:val_282
+197:val_197
+187:val_187
+318:val_318
+318:val_318
+409:val_409
+470:val_470
+137:val_137
+369:val_369
+316:val_316
+169:val_169
+413:val_413
+85:val_85
+77:val_77
+0:val_0
+490:val_490
+87:val_87
+364:val_364
+179:val_179
+118:val_118
+134:val_134
+395:val_395
+282:val_282
+138:val_138
+238:val_238
+419:val_419
+15:val_15
+118:val_118
+72:val_72
+90:val_90
+307:val_307
+19:val_19
+435:val_435
+10:val_10
+277:val_277
+273:val_273
+306:val_306
+224:val_224
+309:val_309
+389:val_389
+327:val_327
+242:val_242
+369:val_369
+392:val_392
+272:val_272
+331:val_331
+401:val_401
+242:val_242
+452:val_452
+177:val_177
+226:val_226
+5:val_5
+497:val_497
+402:val_402
+396:val_396
+317:val_317
+395:val_395
+58:val_58
+35:val_35
+336:val_336
+95:val_95
+11:val_11
+168:val_168
+34:val_34
+229:val_229
+233:val_233
+143:val_143
+472:val_472
+322:val_322
+498:val_498
+160:val_160
+195:val_195
+42:val_42
+321:val_321
+430:val_430
+119:val_119
+489:val_489
+458:val_458
+78:val_78
+76:val_76
+41:val_41
+223:val_223
+492:val_492
+149:val_149
+449:val_449
+218:val_218
+228:val_228
+138:val_138
+453:val_453
+30:val_30
+209:val_209
+64:val_64
+468:val_468
+76:val_76
+74:val_74
+342:val_342
+69:val_69
+230:val_230
+33:val_33
+368:val_368
+103:val_103
+296:val_296
+113:val_113
+216:val_216
+367:val_367
+344:val_344
+167:val_167
+274:val_274
+219:val_219
+239:val_239
+485:val_485
+116:val_116
+223:val_223
+256:val_256
+263:val_263
+70:val_70
+487:val_487
+480:val_480
+401:val_401
+288:val_288
+191:val_191
+5:val_5
+244:val_244
+438:val_438
+128:val_128
+467:val_467
+432:val_432
+202:val_202
+316:val_316
+229:val_229
+469:val_469
+463:val_463
+280:val_280
+2:val_2
+35:val_35
+283:val_283
+331:val_331
+235:val_235
+80:val_80
+44:val_44
+193:val_193
+321:val_321
+335:val_335
+104:val_104
+466:val_466
+366:val_366
+175:val_175
+403:val_403
+483:val_483
+53:val_53
+105:val_105
+257:val_257
+406:val_406
+409:val_409
+190:val_190
+406:val_406
+401:val_401
+114:val_114
+258:val_258
+90:val_90
+203:val_203
+262:val_262
+348:val_348
+424:val_424
+12:val_12
+396:val_396
+201:val_201
+217:val_217
+164:val_164
+431:val_431
+454:val_454
+478:val_478
+298:val_298
+125:val_125
+431:val_431
+164:val_164
+424:val_424
+187:val_187
+382:val_382
+5:val_5
+70:val_70
+397:val_397
+480:val_480
+291:val_291
+24:val_24
+351:val_351
+255:val_255
+104:val_104
+70:val_70
+163:val_163
+438:val_438
+119:val_119
+414:val_414
+200:val_200
+491:val_491
+237:val_237
+439:val_439
+360:val_360
+248:val_248
+479:val_479
+305:val_305
+417:val_417
+199:val_199
+444:val_444
+120:val_120
+429:val_429
+169:val_169
+443:val_443
+323:val_323
+325:val_325
+277:val_277
+230:val_230
+478:val_478
+178:val_178
+468:val_468
+310:val_310
+317:val_317
+333:val_333
+493:val_493
+460:val_460
+207:val_207
+249:val_249
+265:val_265
+480:val_480
+83:val_83
+136:val_136
+353:val_353
+172:val_172
+214:val_214
+462:val_462
+233:val_233
+406:val_406
+133:val_133
+175:val_175
+189:val_189
+454:val_454
+375:val_375
+401:val_401
+421:val_421
+407:val_407
+384:val_384
+256:val_256
+26:val_26
+134:val_134
+67:val_67
+384:val_384
+379:val_379
+18:val_18
+462:val_462
+492:val_492
+100:val_100
+298:val_298
+9:val_9
+341:val_341
+498:val_498
+146:val_146
+458:val_458
+362:val_362
+186:val_186
+285:val_285
+348:val_348
+167:val_167
+18:val_18
+273:val_273
+183:val_183
+281:val_281
+344:val_344
+97:val_97
+469:val_469
+315:val_315
+84:val_84
+28:val_28
+37:val_37
+448:val_448
+152:val_152
+348:val_348
+307:val_307
+194:val_194
+414:val_414
+477:val_477
+222:val_222
+126:val_126
+90:val_90
+169:val_169
+403:val_403
+400:val_400
+200:val_200
+97:val_97
+PREHOOK: query: create table array_table (a array<string>, b array<string>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY ','
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@array_table
+POSTHOOK: query: create table array_table (a array<string>, b array<string>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY ','
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@array_table
+PREHOOK: query: load data local inpath "../../data/files/array_table.txt" overwrite into table array_table
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@array_table
+POSTHOOK: query: load data local inpath "../../data/files/array_table.txt" overwrite into table array_table
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@array_table
+PREHOOK: query: insert overwrite local directory '../../data/files/local_array_table_1'
+select * from array_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@array_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_array_table_1'
+select * from array_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@array_table
+#### A masked pattern was here ####
+a1a2a3b1b2b3b4
+a21a22a23b21b22b23b24
+PREHOOK: query: insert overwrite local directory '../../data/files/local_array_table_2'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+select * from array_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@array_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_array_table_2'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+select * from array_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@array_table
+#### A masked pattern was here ####
+a1#a2#a3:b1#b2#b3#b4
+a21#a22#a23:b21#b22#b23#b24
+PREHOOK: query: insert overwrite local directory '../../data/files/local_array_table_2_withfields'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+select b,a from array_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@array_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_array_table_2_withfields'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+select b,a from array_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@array_table
+#### A masked pattern was here ####
+b1#b2#b3#b4:a1#a2#a3
+b21#b22#b23#b24:a21#a22#a23
+PREHOOK: query: create table map_table (foo STRING , bar MAP<STRING, STRING>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY ','
+MAP KEYS TERMINATED BY ':'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@map_table
+POSTHOOK: query: create table map_table (foo STRING , bar MAP<STRING, STRING>)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY ','
+MAP KEYS TERMINATED BY ':'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@map_table
+PREHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@map_table
+POSTHOOK: query: load data local inpath "../../data/files/map_table.txt" overwrite into table map_table
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@map_table
+PREHOOK: query: insert overwrite local directory '../../data/files/local_map_table_1'
+select * from map_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@map_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_map_table_1'
+select * from map_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@map_table
+#### A masked pattern was here ####
+foo1k1v1k2v2k3v3
+foo2k21v21k22v22k31v31
+PREHOOK: query: insert overwrite local directory '../../data/files/local_map_table_2'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+MAP KEYS TERMINATED BY '='
+select * from map_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@map_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_map_table_2'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+MAP KEYS TERMINATED BY '='
+select * from map_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@map_table
+#### A masked pattern was here ####
+foo1:k1=v1#k2=v2#k3=v3
+foo2:k21=v21#k22=v22#k31=v31
+PREHOOK: query: insert overwrite local directory '../../data/files/local_map_table_2_withfields'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+MAP KEYS TERMINATED BY '='
+select bar,foo from map_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@map_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_map_table_2_withfields'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY ':'
+COLLECTION ITEMS TERMINATED BY '#'
+MAP KEYS TERMINATED BY '='
+select bar,foo from map_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@map_table
+#### A masked pattern was here ####
+k1=v1#k2=v2#k3=v3:foo1
+k21=v21#k22=v22#k31=v31:foo2
+PREHOOK: query: insert overwrite local directory '../../data/files/local_array_table_3'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe'
+STORED AS TEXTFILE
+select * from array_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@array_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_array_table_3'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe'
+STORED AS TEXTFILE
+select * from array_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@array_table
+#### A masked pattern was here ####
+["a1","a2","a3"]["b1","b2","b3","b4"]
+["a21","a22","a23"]["b21","b22","b23","b24"]
+PREHOOK: query: insert overwrite local directory '../../data/files/local_map_table_3'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe'
+STORED AS TEXTFILE
+select * from map_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@map_table
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_map_table_3'
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe'
+STORED AS TEXTFILE
+select * from map_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@map_table
+#### A masked pattern was here ####
+foo1{"k1":"v1","k2":"v2","k3":"v3"}
+foo2{"k21":"v21","k22":"v22","k31":"v31"}
+PREHOOK: query: insert overwrite local directory '../../data/files/local_rctable'
+STORED AS RCFILE
+select value,key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_rctable'
+STORED AS RCFILE
+select value,key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: query: create external table local_rctable(value string, key string)
+STORED AS RCFILE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@local_rctable
+POSTHOOK: query: create external table local_rctable(value string, key string)
+STORED AS RCFILE
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@local_rctable
+PREHOOK: query: insert overwrite local directory '../../data/files/local_rctable_out'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+select key,value from local_rctable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@local_rctable
+#### A masked pattern was here ####
+POSTHOOK: query: insert overwrite local directory '../../data/files/local_rctable_out'
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+select key,value from local_rctable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@local_rctable
+#### A masked pattern was here ####
+238 val_238
+86 val_86
+311 val_311
+27 val_27
+165 val_165
+409 val_409
+255 val_255
+278 val_278
+98 val_98
+484 val_484
+265 val_265
+193 val_193
+401 val_401
+150 val_150
+273 val_273
+224 val_224
+369 val_369
+66 val_66
+128 val_128
+213 val_213
+146 val_146
+406 val_406
+429 val_429
+374 val_374
+152 val_152
+469 val_469
+145 val_145
+495 val_495
+37 val_37
+327 val_327
+281 val_281
+277 val_277
+209 val_209
+15 val_15
+82 val_82
+403 val_403
+166 val_166
+417 val_417
+430 val_430
+252 val_252
+292 val_292
+219 val_219
+287 val_287
+153 val_153
+193 val_193
+338 val_338
+446 val_446
+459 val_459
+394 val_394
+237 val_237
+482 val_482
+174 val_174
+413 val_413
+494 val_494
+207 val_207
+199 val_199
+466 val_466
+208 val_208
+174 val_174
+399 val_399
+396 val_396
+247 val_247
+417 val_417
+489 val_489
+162 val_162
+377 val_377
+397 val_397
+309 val_309
+365 val_365
+266 val_266
+439 val_439
+342 val_342
+367 val_367
+325 val_325
+167 val_167
+195 val_195
+475 val_475
+17 val_17
+113 val_113
+155 val_155
+203 val_203
+339 val_339
+0 val_0
+455 val_455
+128 val_128
+311 val_311
+316 val_316
+57 val_57
+302 val_302
+205 val_205
+149 val_149
+438 val_438
+345 val_345
+129 val_129
+170 val_170
+20 val_20
+489 val_489
+157 val_157
+378 val_378
+221 val_221
+92 val_92
+111 val_111
+47 val_47
+72 val_72
+4 val_4
+280 val_280
+35 val_35
+427 val_427
+277 val_277
+208 val_208
+356 val_356
+399 val_399
+169 val_169
+382 val_382
+498 val_498
+125 val_125
+386 val_386
+437 val_437
+469 val_469
+192 val_192
+286 val_286
+187 val_187
+176 val_176
+54 val_54
+459 val_459
+51 val_51
+138 val_138
+103 val_103
+239 val_239
+213 val_213
+216 val_216
+430 val_430
+278 val_278
+176 val_176
+289 val_289
+221 val_221
+65 val_65
+318 val_318
+332 val_332
+311 val_311
+275 val_275
+137 val_137
+241 val_241
+83 val_83
+333 val_333
+180 val_180
+284 val_284
+12 val_12
+230 val_230
+181 val_181
+67 val_67
+260 val_260
+404 val_404
+384 val_384
+489 val_489
+353 val_353
+373 val_373
+272 val_272
+138 val_138
+217 val_217
+84 val_84
+348 val_348
+466 val_466
+58 val_58
+8 val_8
+411 val_411
+230 val_230
+208 val_208
+348 val_348
+24 val_24
+463 val_463
+431 val_431
+179 val_179
+172 val_172
+42 val_42
+129 val_129
+158 val_158
+119 val_119
+496 val_496
+0 val_0
+322 val_322
+197 val_197
+468 val_468
+393 val_393
+454 val_454
+100 val_100
+298 val_298
+199 val_199
+191 val_191
+418 val_418
+96 val_96
+26 val_26
+165 val_165
+327 val_327
+230 val_230
+205 val_205
+120 val_120
+131 val_131
+51 val_51
+404 val_404
+43 val_43
+436 val_436
+156 val_156
+469 val_469
+468 val_468
+308 val_308
+95 val_95
+196 val_196
+288 val_288
+481 val_481
+457 val_457
+98 val_98
+282 val_282
+197 val_197
+187 val_187
+318 val_318
+318 val_318
+409 val_409
+470 val_470
+137 val_137
+369 val_369
+316 val_316
+169 val_169
+413 val_413
+85 val_85
+77 val_77
+0 val_0
+490 val_490
+87 val_87
+364 val_364
+179 val_179
+118 val_118
+134 val_134
+395 val_395
+282 val_282
+138 val_138
+238 val_238
+419 val_419
+15 val_15
+118 val_118
+72 val_72
+90 val_90
+307 val_307
+19 val_19
+435 val_435
+10 val_10
+277 val_277
+273 val_273
+306 val_306
+224 val_224
+309 val_309
+389 val_389
+327 val_327
+242 val_242
+369 val_369
+392 val_392
+272 val_272
+331 val_331
+401 val_401
+242 val_242
+452 val_452
+177 val_177
+226 val_226
+5 val_5
+497 val_497
+402 val_402
+396 val_396
+317 val_317
+395 val_395
+58 val_58
+35 val_35
+336 val_336
+95 val_95
+11 val_11
+168 val_168
+34 val_34
+229 val_229
+233 val_233
+143 val_143
+472 val_472
+322 val_322
+498 val_498
+160 val_160
+195 val_195
+42 val_42
+321 val_321
+430 val_430
+119 val_119
+489 val_489
+458 val_458
+78 val_78
+76 val_76
+41 val_41
+223 val_223
+492 val_492
+149 val_149
+449 val_449
+218 val_218
+228 val_228
+138 val_138
+453 val_453
+30 val_30
+209 val_209
+64 val_64
+468 val_468
+76 val_76
+74 val_74
+342 val_342
+69 val_69
+230 val_230
+33 val_33
+368 val_368
+103 val_103
+296 val_296
+113 val_113
+216 val_216
+367 val_367
+344 val_344
+167 val_167
+274 val_274
+219 val_219
+239 val_239
+485 val_485
+116 val_116
+223 val_223
+256 val_256
+263 val_263
+70 val_70
+487 val_487
+480 val_480
+401 val_401
+288 val_288
+191 val_191
+5 val_5
+244 val_244
+438 val_438
+128 val_128
+467 val_467
+432 val_432
+202 val_202
+316 val_316
+229 val_229
+469 val_469
+463 val_463
+280 val_280
+2 val_2
+35 val_35
+283 val_283
+331 val_331
+235 val_235
+80 val_80
+44 val_44
+193 val_193
+321 val_321
+335 val_335
+104 val_104
+466 val_466
+366 val_366
+175 val_175
+403 val_403
+483 val_483
+53 val_53
+105 val_105
+257 val_257
+406 val_406
+409 val_409
+190 val_190
+406 val_406
+401 val_401
+114 val_114
+258 val_258
+90 val_90
+203 val_203
+262 val_262
+348 val_348
+424 val_424
+12 val_12
+396 val_396
+201 val_201
+217 val_217
+164 val_164
+431 val_431
+454 val_454
+478 val_478
+298 val_298
+125 val_125
+431 val_431
+164 val_164
+424 val_424
+187 val_187
+382 val_382
+5 val_5
+70 val_70
+397 val_397
+480 val_480
+291 val_291
+24 val_24
+351 val_351
+255 val_255
+104 val_104
+70 val_70
+163 val_163
+438 val_438
+119 val_119
+414 val_414
+200 val_200
+491 val_491
+237 val_237
+439 val_439
+360 val_360
+248 val_248
+479 val_479
+305 val_305
+417 val_417
+199 val_199
+444 val_444
+120 val_120
+429 val_429
+169 val_169
+443 val_443
+323 val_323
+325 val_325
+277 val_277
+230 val_230
+478 val_478
+178 val_178
+468 val_468
+310 val_310
+317 val_317
+333 val_333
+493 val_493
+460 val_460
+207 val_207
+249 val_249
+265 val_265
+480 val_480
+83 val_83
+136 val_136
+353 val_353
+172 val_172
+214 val_214
+462 val_462
+233 val_233
+406 val_406
+133 val_133
+175 val_175
+189 val_189
+454 val_454
+375 val_375
+401 val_401
+421 val_421
+407 val_407
+384 val_384
+256 val_256
+26 val_26
+134 val_134
+67 val_67
+384 val_384
+379 val_379
+18 val_18
+462 val_462
+492 val_492
+100 val_100
+298 val_298
+9 val_9
+341 val_341
+498 val_498
+146 val_146
+458 val_458
+362 val_362
+186 val_186
+285 val_285
+348 val_348
+167 val_167
+18 val_18
+273 val_273
+183 val_183
+281 val_281
+344 val_344
+97 val_97
+469 val_469
+315 val_315
+84 val_84
+28 val_28
+37 val_37
+448 val_448
+152 val_152
+348 val_348
+307 val_307
+194 val_194
+414 val_414
+477 val_477
+222 val_222
+126 val_126
+90 val_90
+169 val_169
+403 val_403
+400 val_400
+200 val_200
+97 val_97
+PREHOOK: query: drop table local_rctable
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@local_rctable
+PREHOOK: Output: default@local_rctable
+POSTHOOK: query: drop table local_rctable
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@local_rctable
+POSTHOOK: Output: default@local_rctable
+PREHOOK: query: drop table array_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@array_table
+PREHOOK: Output: default@array_table
+POSTHOOK: query: drop table array_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@array_table
+POSTHOOK: Output: default@array_table
+PREHOOK: query: drop table map_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@map_table
+PREHOOK: Output: default@map_table
+POSTHOOK: query: drop table map_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@map_table
+POSTHOOK: Output: default@map_table
+#### A masked pattern was here ####
[23/36] hive git commit: HIVE-16566: Set column stats default as true
when creating new tables/partitions (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16566: Set column stats default as true when creating new tables/partitions (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52a71e96
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52a71e96
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52a71e96
Branch: refs/heads/hive-14535
Commit: 52a71e9608928b41ea6a4ae7bf2365641009ce56
Parents: 019ad8c
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Fri Jun 2 13:26:02 2017 -0700
Committer: Pengcheng Xiong <px...@hortonworks.com>
Committed: Fri Jun 2 13:26:02 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/common/StatsSetupConst.java | 7 +-
.../results/positive/external_table_ppd.q.out | 2 +-
.../positive/hbase_binary_storage_queries.q.out | 4 +-
.../src/test/results/positive/hbasestats.q.out | 2 +-
.../write_final_output_blobstore.q.out | 16 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 8 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 4 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/plan/CreateTableDesc.java | 7 +-
.../queries/clientpositive/alterColumnStats.q | 21 +
.../clientpositive/alterColumnStatsPart.q | 47 ++
.../clientnegative/alter_file_format.q.out | 2 +-
.../clientnegative/unset_table_property.q.out | 2 +-
.../clientpositive/alterColumnStats.q.out | 158 ++++++
.../clientpositive/alterColumnStatsPart.q.out | 507 +++++++++++++++++++
.../clientpositive/alter_file_format.q.out | 26 +-
.../alter_partition_clusterby_sortby.q.out | 6 +-
.../clientpositive/alter_skewed_table.q.out | 12 +-
.../alter_table_add_partition.q.out | 2 +-
.../clientpositive/alter_table_not_sorted.q.out | 4 +-
.../clientpositive/autoColumnStats_3.q.out | 2 +-
.../clientpositive/autoColumnStats_4.q.out | 2 +-
.../clientpositive/binary_output_format.q.out | 16 +-
.../test/results/clientpositive/bucket1.q.out | 4 +-
.../test/results/clientpositive/bucket2.q.out | 4 +-
.../clientpositive/bucket_map_join_spark1.q.out | 4 +-
.../clientpositive/bucket_map_join_spark2.q.out | 4 +-
.../clientpositive/bucket_map_join_spark3.q.out | 4 +-
.../results/clientpositive/bucketmapjoin5.q.out | 16 +-
.../clientpositive/bucketmapjoin_negative.q.out | 16 +-
.../bucketmapjoin_negative2.q.out | 16 +-
...names_with_leading_and_trailing_spaces.q.out | 4 +-
.../results/clientpositive/compustat_avro.q.out | 4 +-
.../create_alter_list_bucketing_table1.q.out | 14 +-
.../results/clientpositive/create_like.q.out | 18 +-
.../results/clientpositive/create_like2.q.out | 2 +-
.../clientpositive/create_like_tbl_props.q.out | 10 +-
.../clientpositive/create_like_view.q.out | 6 +-
.../clientpositive/create_skewed_table1.q.out | 6 +-
.../create_table_like_stats.q.out | 2 +-
.../create_with_constraints.q.out | 72 +--
.../clientpositive/database_location.q.out | 4 +-
.../clientpositive/default_file_format.q.out | 16 +-
.../describe_comment_indent.q.out | 2 +-
.../describe_comment_nonascii.q.out | 2 +-
.../describe_formatted_view_partitioned.q.out | 2 +-
.../clientpositive/describe_syntax.q.out | 4 +-
.../results/clientpositive/describe_table.q.out | 24 +-
.../display_colstats_tbllvl.q.out | 2 +-
.../results/clientpositive/druid_basic1.q.out | 4 +-
.../results/clientpositive/druid_basic2.q.out | 18 +-
.../clientpositive/druid_intervals.q.out | 2 +-
.../clientpositive/druid_timeseries.q.out | 2 +-
.../results/clientpositive/druid_topn.q.out | 2 +-
.../clientpositive/groupby_map_ppr.q.out | 4 +-
.../groupby_map_ppr_multi_distinct.q.out | 4 +-
.../results/clientpositive/groupby_ppr.q.out | 4 +-
.../groupby_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/groupby_sort_1_23.q.out | 56 +-
.../results/clientpositive/groupby_sort_6.q.out | 4 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 56 +-
.../results/clientpositive/input_part1.q.out | 16 +-
.../results/clientpositive/input_part2.q.out | 32 +-
.../insert_values_orig_table_use_metadata.q.out | 4 +-
ql/src/test/results/clientpositive/join17.q.out | 4 +-
ql/src/test/results/clientpositive/join26.q.out | 4 +-
ql/src/test/results/clientpositive/join32.q.out | 4 +-
ql/src/test/results/clientpositive/join33.q.out | 4 +-
ql/src/test/results/clientpositive/join34.q.out | 6 +-
ql/src/test/results/clientpositive/join35.q.out | 6 +-
ql/src/test/results/clientpositive/join9.q.out | 4 +-
.../results/clientpositive/join_map_ppr.q.out | 16 +-
.../clientpositive/list_bucket_dml_14.q.out | 4 +-
.../clientpositive/llap/autoColumnStats_2.q.out | 2 +-
.../results/clientpositive/llap/bucket2.q.out | 4 +-
.../results/clientpositive/llap/bucket4.q.out | 4 +-
.../results/clientpositive/llap/bucket5.q.out | 20 +-
.../clientpositive/llap/bucket_many.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin1.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin2.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin3.q.out | 4 +-
.../clientpositive/llap/bucketmapjoin4.q.out | 4 +-
...names_with_leading_and_trailing_spaces.q.out | 4 +-
.../llap/disable_merge_for_bucketing.q.out | 4 +-
.../llap/dynamic_semijoin_reduction.q.out | 2 +-
.../llap/dynamic_semijoin_user_level.q.out | 2 +-
.../clientpositive/llap/join32_lessSize.q.out | 8 +-
.../clientpositive/llap/metadataonly1.q.out | 40 +-
.../clientpositive/llap/orc_create.q.out | 4 +-
.../llap/reduce_deduplicate.q.out | 4 +-
.../results/clientpositive/llap/sample1.q.out | 4 +-
.../results/clientpositive/llap/stats11.q.out | 4 +-
.../results/clientpositive/llap/sysdb.q.out | 10 +-
.../parquet_array_null_element.q.out | 2 +-
.../results/clientpositive/parquet_create.q.out | 2 +-
.../parquet_int96_timestamp.q.out | 4 +-
.../clientpositive/part_inherit_tbl_props.q.out | 2 +-
.../part_inherit_tbl_props_empty.q.out | 2 +-
.../part_inherit_tbl_props_with_star.q.out | 2 +-
.../clientpositive/partial_column_stats.q.out | 2 +-
.../partition_coltype_literals.q.out | 8 +-
ql/src/test/results/clientpositive/pcr.q.out | 32 +-
.../results/clientpositive/quotedid_stats.q.out | 2 +-
.../clientpositive/rand_partitionpruner2.q.out | 16 +-
.../clientpositive/rcfile_default_format.q.out | 6 +-
.../test/results/clientpositive/sample1.q.out | 16 +-
.../test/results/clientpositive/sample2.q.out | 16 +-
.../test/results/clientpositive/sample4.q.out | 16 +-
.../test/results/clientpositive/sample5.q.out | 16 +-
.../test/results/clientpositive/sample6.q.out | 16 +-
.../test/results/clientpositive/sample7.q.out | 16 +-
.../clientpositive/show_tblproperties.q.out | 10 +-
.../results/clientpositive/spark/bucket2.q.out | 4 +-
.../results/clientpositive/spark/bucket4.q.out | 4 +-
.../results/clientpositive/spark/bucket5.q.out | 8 +-
.../spark/bucket_map_join_spark1.q.out | 4 +-
.../spark/bucket_map_join_spark2.q.out | 4 +-
.../spark/bucket_map_join_spark3.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 4 +-
.../clientpositive/spark/bucketmapjoin5.q.out | 4 +-
.../spark/bucketmapjoin_negative.q.out | 4 +-
.../spark/bucketmapjoin_negative2.q.out | 4 +-
.../spark/disable_merge_for_bucketing.q.out | 4 +-
.../clientpositive/spark/groupby_map_ppr.q.out | 4 +-
.../spark/groupby_map_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/spark/groupby_ppr.q.out | 4 +-
.../spark/groupby_ppr_multi_distinct.q.out | 4 +-
.../spark/groupby_sort_1_23.q.out | 20 +-
.../spark/groupby_sort_skew_1_23.q.out | 20 +-
.../clientpositive/spark/input_part2.q.out | 8 +-
.../results/clientpositive/spark/join17.q.out | 4 +-
.../results/clientpositive/spark/join26.q.out | 4 +-
.../results/clientpositive/spark/join32.q.out | 4 +-
.../clientpositive/spark/join32_lessSize.q.out | 8 +-
.../results/clientpositive/spark/join33.q.out | 4 +-
.../results/clientpositive/spark/join34.q.out | 4 +-
.../results/clientpositive/spark/join35.q.out | 4 +-
.../results/clientpositive/spark/join9.q.out | 4 +-
.../clientpositive/spark/join_map_ppr.q.out | 4 +-
.../test/results/clientpositive/spark/pcr.q.out | 8 +-
.../spark/reduce_deduplicate.q.out | 4 +-
.../results/clientpositive/spark/sample1.q.out | 4 +-
.../results/clientpositive/spark/sample2.q.out | 4 +-
.../results/clientpositive/spark/sample4.q.out | 4 +-
.../results/clientpositive/spark/sample5.q.out | 4 +-
.../results/clientpositive/spark/sample6.q.out | 4 +-
.../results/clientpositive/spark/sample7.q.out | 4 +-
.../results/clientpositive/spark/stats0.q.out | 8 +-
.../results/clientpositive/spark/stats3.q.out | 2 +-
ql/src/test/results/clientpositive/stats0.q.out | 20 +-
ql/src/test/results/clientpositive/stats3.q.out | 2 +-
.../temp_table_display_colstats_tbllvl.q.out | 6 +-
.../clientpositive/unicode_notation.q.out | 6 +-
.../unset_table_view_property.q.out | 18 +-
158 files changed, 1360 insertions(+), 613 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index 2387407..7c27d07 100644
--- a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -240,6 +240,9 @@ public class StatsSetupConst {
if (params == null) {
throw new RuntimeException("params are null...cant set columnstatstate!");
}
+ if (colNames == null) {
+ return;
+ }
ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE));
for (String colName : colNames) {
@@ -283,13 +286,15 @@ public class StatsSetupConst {
}
}
- public static void setBasicStatsStateForCreateTable(Map<String, String> params, String setting) {
+ public static void setStatsStateForCreateTable(Map<String, String> params,
+ List<String> cols, String setting) {
if (TRUE.equals(setting)) {
for (String stat : StatsSetupConst.supportedStats) {
params.put(stat, "0");
}
}
setBasicStatsState(params, setting);
+ setColumnStatsState(params, cols);
}
private static ColumnStatsAccurate parseStatsAcc(String statsAcc) {
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/hbase-handler/src/test/results/positive/external_table_ppd.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/external_table_ppd.q.out b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
index f09b880..328131a 100644
--- a/hbase-handler/src/test/results/positive/external_table_ppd.q.out
+++ b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
@@ -56,7 +56,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
hbase.table.default.storage.type binary
hbase.table.name t_hive
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
index 3d3e82d..5404cd9 100644
--- a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
@@ -56,7 +56,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
hbase.table.default.storage.type binary
hbase.table.name t_hive
numFiles 0
@@ -236,7 +236,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
EXTERNAL TRUE
hbase.table.name t_hive
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/hbase-handler/src/test/results/positive/hbasestats.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbasestats.q.out b/hbase-handler/src/test/results/positive/hbasestats.q.out
index bf902e4..b7ef954 100644
--- a/hbase-handler/src/test/results/positive/hbasestats.q.out
+++ b/hbase-handler/src/test/results/positive/hbasestats.q.out
@@ -38,7 +38,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"country\":\"true\",\"country_id\":\"true\",\"key\":\"true\",\"state\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
index 46bfef5..2ababb1 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -89,7 +89,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -192,7 +192,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -224,7 +224,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -292,7 +292,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -313,7 +313,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -416,7 +416,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
@@ -448,7 +448,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index a8adb61..ff8412c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1969,5 +1969,13 @@ public class MetaStoreUtils {
}
return metaException;
}
+
+ public static List<String> getColumnNames(List<FieldSchema> schema) {
+ List<String> cols = new ArrayList<>();
+ for (FieldSchema fs : schema) {
+ cols.add(fs.getName());
+ }
+ return cols;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 617555e..87928ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4566,8 +4566,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (crtTbl.getLocation() == null && !tbl.isPartitioned()
&& conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(),
- StatsSetupConst.TRUE);
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(),
+ MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE);
}
// create the table
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 3f032c8..ceb8079 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1752,7 +1752,7 @@ public class Hive {
if (oldPart == null) {
newTPart.getTPartition().setParameters(new HashMap<String,String>());
if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- StatsSetupConst.setBasicStatsStateForCreateTable(newTPart.getParameters(),
+ StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), null,
StatsSetupConst.TRUE);
}
MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters());
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index dee6a10..e9a4ff0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -2951,8 +2951,8 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
if (desc.getPartParams() == null) {
desc.setPartParams(new HashMap<String, String>());
}
- StatsSetupConst.setBasicStatsStateForCreateTable(desc.getPartParams(),
- StatsSetupConst.TRUE);
+ StatsSetupConst.setStatsStateForCreateTable(desc.getPartParams(),
+ MetaStoreUtils.getColumnNames(tab.getCols()), StatsSetupConst.TRUE);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
index 7b46fcd..4320421 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
@@ -28,6 +28,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -848,11 +849,11 @@ public class CreateTableDesc extends DDLDesc implements Serializable {
if (!this.isCTAS && (tbl.getPath() == null || (tbl.isEmpty() && !isExternal()))) {
if (!tbl.isPartitioned() && conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(),
- StatsSetupConst.TRUE);
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(),
+ MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE);
}
} else {
- StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(),
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null,
StatsSetupConst.FALSE);
}
return tbl;
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/queries/clientpositive/alterColumnStats.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alterColumnStats.q b/ql/src/test/queries/clientpositive/alterColumnStats.q
new file mode 100644
index 0000000..4456088
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alterColumnStats.q
@@ -0,0 +1,21 @@
+set hive.mapred.mode=nonstrict;
+
+drop table p;
+
+CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint);
+
+desc formatted p;
+
+insert into p values (1,22,333);
+
+desc formatted p;
+
+alter table p replace columns (insert_num int, c1 STRING, c2 STRING);
+
+desc formatted p;
+
+desc formatted p c1;
+
+desc formatted p c2;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/queries/clientpositive/alterColumnStatsPart.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alterColumnStatsPart.q b/ql/src/test/queries/clientpositive/alterColumnStatsPart.q
new file mode 100644
index 0000000..f46d9d5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alterColumnStatsPart.q
@@ -0,0 +1,47 @@
+set hive.mapred.mode=nonstrict;
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+drop table p;
+
+CREATE TABLE p(a int, b int) partitioned by (c int);
+
+desc formatted p;
+
+insert into p partition (c=1) values (1,2);
+
+desc formatted p partition (c=1) a;
+
+desc formatted p partition (c=1);
+
+explain select max(a) from p where c=1;
+
+analyze table p partition(c=1) compute statistics for columns a;
+
+explain select max(a) from p where c=1;
+
+insert into p partition (c) values (2,3,4);
+
+insert into p partition (c) values (4,5,6);
+
+desc formatted p partition(c=4);
+
+explain select max(a) from p where c=4;
+
+alter table p add partition (c=100);
+
+desc formatted p partition (c=100);
+
+explain select max(a) from p where c=100;
+
+analyze table p partition(c=100) compute statistics for columns a;
+
+explain select max(a) from p where c=100;
+
+desc formatted p partition(c=100);
+
+insert into p partition (c=100) values (1,2);
+
+analyze table p partition(c=100) compute statistics for columns a;
+
+explain select max(a) from p where c=100;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientnegative/alter_file_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/alter_file_format.q.out b/ql/src/test/results/clientnegative/alter_file_format.q.out
index e3f3b4c..4a79eaa 100644
--- a/ql/src/test/results/clientnegative/alter_file_format.q.out
+++ b/ql/src/test/results/clientnegative/alter_file_format.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientnegative/unset_table_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/unset_table_property.q.out b/ql/src/test/results/clientnegative/unset_table_property.q.out
index 4aedfc5..2dd047a 100644
--- a/ql/src/test/results/clientnegative/unset_table_property.q.out
+++ b/ql/src/test/results/clientnegative/unset_table_property.q.out
@@ -18,7 +18,7 @@ PREHOOK: query: SHOW TBLPROPERTIES testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
c 3
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alterColumnStats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alterColumnStats.q.out b/ql/src/test/results/clientpositive/alterColumnStats.q.out
new file mode 100644
index 0000000..519a62a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alterColumnStats.q.out
@@ -0,0 +1,158 @@
+PREHOOK: query: drop table p
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table p
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@p
+POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@p
+PREHOOK: query: desc formatted p
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+insert_num int
+c1 tinyint
+c2 smallint
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}}
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: insert into p values (1,22,333)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@p
+POSTHOOK: query: insert into p values (1,22,333)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@p
+POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: desc formatted p
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+insert_num int
+c1 tinyint
+c2 smallint
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles 1
+ numRows 1
+ rawDataSize 8
+ totalSize 9
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING)
+PREHOOK: type: ALTERTABLE_REPLACECOLS
+PREHOOK: Input: default@p
+PREHOOK: Output: default@p
+POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING)
+POSTHOOK: type: ALTERTABLE_REPLACECOLS
+POSTHOOK: Input: default@p
+POSTHOOK: Output: default@p
+PREHOOK: query: desc formatted p
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+insert_num int
+c1 string
+c2 string
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+#### A masked pattern was here ####
+ numFiles 1
+ numRows 1
+ rawDataSize 8
+ totalSize 9
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: desc formatted p c1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p c1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
+
+c1 string from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+PREHOOK: query: desc formatted p c2
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p c2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
+
+c2 string from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alterColumnStatsPart.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alterColumnStatsPart.q.out b/ql/src/test/results/clientpositive/alterColumnStatsPart.q.out
new file mode 100644
index 0000000..672bd9f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alterColumnStatsPart.q.out
@@ -0,0 +1,507 @@
+PREHOOK: query: drop table p
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table p
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE p(a int, b int) partitioned by (c int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@p
+POSTHOOK: query: CREATE TABLE p(a int, b int) partitioned by (c int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@p
+PREHOOK: query: desc formatted p
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int
+b int
+
+# Partition Information
+# col_name data_type comment
+
+c int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles 0
+ numPartitions 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: insert into p partition (c=1) values (1,2)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@p@c=1
+POSTHOOK: query: insert into p partition (c=1) values (1,2)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@p@c=1
+POSTHOOK: Lineage: p PARTITION(c=1).a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: p PARTITION(c=1).b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: desc formatted p partition (c=1) a
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p partition (c=1) a
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int from deserializer
+PREHOOK: query: desc formatted p partition (c=1)
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p partition (c=1)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int
+b int
+
+# Partition Information
+# col_name data_type comment
+
+c int
+
+# Detailed Partition Information
+Partition Value: [1]
+Database: default
+Table: p
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles 1
+ numRows 1
+ rawDataSize 3
+ totalSize 4
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select max(a) from p where c=1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: p
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: a
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(a)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table p partition(c=1) compute statistics for columns a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@p
+PREHOOK: Input: default@p@c=1
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table p partition(c=1) compute statistics for columns a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@p
+POSTHOOK: Input: default@p@c=1
+#### A masked pattern was here ####
+PREHOOK: query: explain select max(a) from p where c=1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: insert into p partition (c) values (2,3,4)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@p
+POSTHOOK: query: insert into p partition (c) values (2,3,4)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@p@c=4
+POSTHOOK: Lineage: p PARTITION(c=4).a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: p PARTITION(c=4).b EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: insert into p partition (c) values (4,5,6)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@p
+POSTHOOK: query: insert into p partition (c) values (4,5,6)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@p@c=6
+POSTHOOK: Lineage: p PARTITION(c=6).a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: p PARTITION(c=6).b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: desc formatted p partition(c=4)
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p partition(c=4)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int
+b int
+
+# Partition Information
+# col_name data_type comment
+
+c int
+
+# Detailed Partition Information
+Partition Value: [4]
+Database: default
+Table: p
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles 1
+ numRows 1
+ rawDataSize 3
+ totalSize 4
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select max(a) from p where c=4
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=4
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: p
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: a
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(a)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: alter table p add partition (c=100)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@p
+POSTHOOK: query: alter table p add partition (c=100)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@p
+POSTHOOK: Output: default@p@c=100
+PREHOOK: query: desc formatted p partition (c=100)
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p partition (c=100)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int
+b int
+
+# Partition Information
+# col_name data_type comment
+
+c int
+
+# Detailed Partition Information
+Partition Value: [100]
+Database: default
+Table: p
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain select max(a) from p where c=100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: p
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: a
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: max(a)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: analyze table p partition(c=100) compute statistics for columns a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@p
+PREHOOK: Input: default@p@c=100
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@p
+POSTHOOK: Input: default@p@c=100
+#### A masked pattern was here ####
+PREHOOK: query: explain select max(a) from p where c=100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: p
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: a (type: int)
+ outputColumnNames: a
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: max(a)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: desc formatted p partition(c=100)
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@p
+POSTHOOK: query: desc formatted p partition(c=100)
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@p
+# col_name data_type comment
+
+a int
+b int
+
+# Partition Information
+# col_name data_type comment
+
+c int
+
+# Detailed Partition Information
+Partition Value: [100]
+Database: default
+Table: p
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: insert into p partition (c=100) values (1,2)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@p@c=100
+POSTHOOK: query: insert into p partition (c=100) values (1,2)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@p@c=100
+POSTHOOK: Lineage: p PARTITION(c=100).a EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: p PARTITION(c=100).b EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: analyze table p partition(c=100) compute statistics for columns a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@p
+PREHOOK: Input: default@p@c=100
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@p
+POSTHOOK: Input: default@p@c=100
+#### A masked pattern was here ####
+PREHOOK: query: explain select max(a) from p where c=100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(a) from p where c=100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alter_file_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_file_format.q.out b/ql/src/test/results/clientpositive/alter_file_format.q.out
index ca569be..e4be931 100644
--- a/ql/src/test/results/clientpositive/alter_file_format.q.out
+++ b/ql/src/test/results/clientpositive/alter_file_format.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -67,7 +67,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -111,7 +111,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -155,7 +155,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -199,7 +199,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -243,7 +243,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -287,7 +287,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -346,7 +346,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -394,7 +394,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -443,7 +443,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -492,7 +492,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -541,7 +541,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -590,7 +590,7 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out b/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
index 8bd9525..d67be56 100644
--- a/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
@@ -44,7 +44,7 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -93,7 +93,7 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -142,7 +142,7 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alter_skewed_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_skewed_table.q.out b/ql/src/test/results/clientpositive/alter_skewed_table.q.out
index fefef4c..e553267 100644
--- a/ql/src/test/results/clientpositive/alter_skewed_table.q.out
+++ b/ql/src/test/results/clientpositive/alter_skewed_table.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -67,7 +67,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -127,7 +127,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -170,7 +170,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -224,7 +224,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -269,7 +269,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alter_table_add_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_table_add_partition.q.out b/ql/src/test/results/clientpositive/alter_table_add_partition.q.out
index ae6624f..c96e5b4 100644
--- a/ql/src/test/results/clientpositive/alter_table_add_partition.q.out
+++ b/ql/src/test/results/clientpositive/alter_table_add_partition.q.out
@@ -114,7 +114,7 @@ Database: default
Table: mp
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/alter_table_not_sorted.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_table_not_sorted.q.out b/ql/src/test/results/clientpositive/alter_table_not_sorted.q.out
index 5afb7fa..5813207 100644
--- a/ql/src/test/results/clientpositive/alter_table_not_sorted.q.out
+++ b/ql/src/test/results/clientpositive/alter_table_not_sorted.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
SORTBUCKETCOLSPREFIX TRUE
numFiles 0
numRows 0
@@ -68,7 +68,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
SORTBUCKETCOLSPREFIX TRUE
#### A masked pattern was here ####
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/autoColumnStats_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_3.q.out b/ql/src/test/results/clientpositive/autoColumnStats_3.q.out
index b916eba..2f70095 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_3.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_3.q.out
@@ -36,7 +36,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index 101cc63..c3ad192 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -24,7 +24,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/binary_output_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/binary_output_format.q.out b/ql/src/test/results/clientpositive/binary_output_format.q.out
index 96afeb6..ddb6adf 100644
--- a/ql/src/test/results/clientpositive/binary_output_format.q.out
+++ b/ql/src/test/results/clientpositive/binary_output_format.q.out
@@ -95,7 +95,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -187,7 +187,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -225,7 +225,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -256,7 +256,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -278,7 +278,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -315,7 +315,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -346,7 +346,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
@@ -368,7 +368,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"mydata":"true"}}
bucket_count -1
column.name.delimiter ,
columns mydata
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucket1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucket1.q.out b/ql/src/test/results/clientpositive/bucket1.q.out
index 78571aa..1d20473 100644
--- a/ql/src/test/results/clientpositive/bucket1.q.out
+++ b/ql/src/test/results/clientpositive/bucket1.q.out
@@ -107,7 +107,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 100
bucket_field_name key
column.name.delimiter ,
@@ -139,7 +139,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 100
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucket2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucket2.q.out b/ql/src/test/results/clientpositive/bucket2.q.out
index a8e9f90..48ccafb 100644
--- a/ql/src/test/results/clientpositive/bucket2.q.out
+++ b/ql/src/test/results/clientpositive/bucket2.q.out
@@ -107,7 +107,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -139,7 +139,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out b/ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out
index 5478f12..70cd53c 100644
--- a/ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out
+++ b/ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out
@@ -236,7 +236,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -370,7 +370,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out b/ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out
index e335683..d0c3a1a 100644
--- a/ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out
+++ b/ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out
@@ -220,7 +220,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -354,7 +354,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out b/ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out
index bd5a438..eed4a5a 100644
--- a/ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out
+++ b/ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out
@@ -220,7 +220,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -354,7 +354,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
[13/36] hive git commit: HIVE-16769: Possible hive service startup
due to the existing file /tmp/stderr (Aihua Xu, reviewed by Naveen Gangam)
Posted by we...@apache.org.
HIVE-16769: Possible hive service startup due to the existing file /tmp/stderr (Aihua Xu, reviewed by Naveen Gangam)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f976fff
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f976fff
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f976fff
Branch: refs/heads/hive-14535
Commit: 9f976ffff343a891c9d3e8d5fa0e70b2c622ff1a
Parents: 3f5e652
Author: Aihua Xu <ai...@apache.org>
Authored: Wed May 31 17:08:29 2017 -0400
Committer: Aihua Xu <ai...@apache.org>
Committed: Fri Jun 2 09:21:39 2017 -0400
----------------------------------------------------------------------
bin/ext/version.sh | 2 +-
bin/hive | 20 ++------------------
2 files changed, 3 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f976fff/bin/ext/version.sh
----------------------------------------------------------------------
diff --git a/bin/ext/version.sh b/bin/ext/version.sh
index b6a237b..92fee3d 100644
--- a/bin/ext/version.sh
+++ b/bin/ext/version.sh
@@ -27,7 +27,7 @@ version () {
# hadoop 20 or newer - skip the aux_jars option and hiveconf
CLASS=org.apache.hive.common.util.HiveVersionInfo
- exec $HADOOP jar $JAR $CLASS 2>> ${STDERR}
+ exec $HADOOP jar $JAR $CLASS 2>&2
}
version_help () {
http://git-wip-us.apache.org/repos/asf/hive/blob/9f976fff/bin/hive
----------------------------------------------------------------------
diff --git a/bin/hive b/bin/hive
index a7671c3..4a908e1 100755
--- a/bin/hive
+++ b/bin/hive
@@ -25,8 +25,6 @@ bin=`cd "$bin"; pwd`
. "$bin"/hive-config.sh
-TMP_USER_DIR="/tmp/${USER}"
-STDERR="${TMP_USER_DIR}/stderr"
SERVICE=""
HELP=""
SKIP_HBASECP=false
@@ -227,24 +225,10 @@ if [ ! -f ${HADOOP} ]; then
exit 4;
fi
-if [ ! -d ${TMP_USER_DIR} ]; then
- mkdir -p ${TMP_USER_DIR} 2> /dev/null
- if [ $? -ne 0 ]; then
- STDERR="/dev/tty"
- fi
-fi
-
-if [ "${STDERR}" != "/dev/null" ] && [ ! -f ${STDERR} ]; then
- touch ${STDERR} 2> /dev/null
- if [ $? -ne 0 ]; then
- STDERR="/dev/tty"
- fi
-fi
-
if [ "$SKIP_HADOOPVERSION" = false ]; then
# Make sure we're using a compatible version of Hadoop
if [ "x$HADOOP_VERSION" == "x" ]; then
- HADOOP_VERSION=$($HADOOP version 2>> ${STDERR} | awk -F"\t" '/Hadoop/ {print $0}' | cut -d' ' -f 2);
+ HADOOP_VERSION=$($HADOOP version 2>&2 | awk -F"\t" '/Hadoop/ {print $0}' | cut -d' ' -f 2);
fi
# Save the regex to a var to workaround quoting incompatabilities
@@ -295,7 +279,7 @@ if [ "$SKIP_HBASECP" = false ]; then
if [[ -n $HBASE_BIN ]] ; then
# exclude ZK, PB, and Guava (See HIVE-2055)
# depends on HBASE-8438 (hbase-0.94.14+, hbase-0.96.1+) for `hbase mapredcp` command
- for x in $($HBASE_BIN mapredcp 2>> ${STDERR} | tr ':' '\n') ; do
+ for x in $($HBASE_BIN mapredcp 2>&2 | tr ':' '\n') ; do
if [[ $x == *zookeeper* || $x == *protobuf-java* || $x == *guava* ]] ; then
continue
fi
[12/36] hive git commit: HIVE-16801: Vectorization: throwExpandError
should be an immediate fatal (Gopal V, reviewed by Prasanth Jayachandran)
Posted by we...@apache.org.
HIVE-16801: Vectorization: throwExpandError should be an immediate fatal (Gopal V, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3f5e6528
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3f5e6528
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3f5e6528
Branch: refs/heads/hive-14535
Commit: 3f5e65288eacbff9d2c373d3bd4e1eaa74517f67
Parents: b560f49
Author: Gopal V <go...@apache.org>
Authored: Thu Jun 1 22:59:50 2017 -0700
Committer: Gopal V <go...@apache.org>
Committed: Thu Jun 1 22:59:50 2017 -0700
----------------------------------------------------------------------
.../ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3f5e6528/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
index 1f182ee..ce9a962 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTable;
public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTable {
@@ -47,7 +48,7 @@ public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTab
public static final int ONE_SIXTH_LIMIT = HIGHEST_INT_POWER_OF_2 / 6;
public void throwExpandError(int limit, String dataTypeName) {
- throw new RuntimeException(
+ throw new MapJoinMemoryExhaustionError(
"Vector MapJoin " + dataTypeName + " Hash Table cannot grow any more -- use a smaller container size. " +
"Current logical size is " + logicalHashBucketCount + " and " +
"the limit is " + limit + ". " +
[29/36] hive git commit: HIVE-16775 : Fix
HiveFilterAggregateTransposeRule when filter is always false (Pengcheng Xiong
via Jesus Camacho Rodriguez)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/results/clientpositive/perf/query74.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query74.q.out b/ql/src/test/results/clientpositive/perf/query74.q.out
new file mode 100644
index 0000000..bb4a71e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/query74.q.out
@@ -0,0 +1,529 @@
+PREHOOK: query: explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ss_net_paid) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ws_net_paid) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ )
+ select
+ t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.year = 2001
+ and t_s_secyear.year = 2001+1
+ and t_w_firstyear.year = 2001
+ and t_w_secyear.year = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ order by 2,1,3
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+with year_total as (
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ss_net_paid) year_total
+ ,'s' sale_type
+ from customer
+ ,store_sales
+ ,date_dim
+ where c_customer_sk = ss_customer_sk
+ and ss_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ union all
+ select c_customer_id customer_id
+ ,c_first_name customer_first_name
+ ,c_last_name customer_last_name
+ ,d_year as year
+ ,max(ws_net_paid) year_total
+ ,'w' sale_type
+ from customer
+ ,web_sales
+ ,date_dim
+ where c_customer_sk = ws_bill_customer_sk
+ and ws_sold_date_sk = d_date_sk
+ and d_year in (2001,2001+1)
+ group by c_customer_id
+ ,c_first_name
+ ,c_last_name
+ ,d_year
+ )
+ select
+ t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name
+ from year_total t_s_firstyear
+ ,year_total t_s_secyear
+ ,year_total t_w_firstyear
+ ,year_total t_w_secyear
+ where t_s_secyear.customer_id = t_s_firstyear.customer_id
+ and t_s_firstyear.customer_id = t_w_secyear.customer_id
+ and t_s_firstyear.customer_id = t_w_firstyear.customer_id
+ and t_s_firstyear.sale_type = 's'
+ and t_w_firstyear.sale_type = 'w'
+ and t_s_secyear.sale_type = 's'
+ and t_w_secyear.sale_type = 'w'
+ and t_s_firstyear.year = 2001
+ and t_s_secyear.year = 2001+1
+ and t_w_firstyear.year = 2001
+ and t_w_secyear.year = 2001+1
+ and t_s_firstyear.year_total > 0
+ and t_w_firstyear.year_total > 0
+ and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end
+ > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end
+ order by 2,1,3
+limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 12 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 13 <- Map 33 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Union 15 (CONTAINS)
+Reducer 16 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 17 <- Map 33 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Union 19 (CONTAINS)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 21 <- Map 20 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE)
+Reducer 22 <- Map 33 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
+Reducer 23 <- Reducer 22 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 24 <- Map 20 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE)
+Reducer 25 <- Map 33 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
+Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 27 <- Map 20 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE)
+Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE)
+Reducer 29 <- Reducer 28 (SIMPLE_EDGE), Union 15 (CONTAINS)
+Reducer 3 <- Map 33 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 30 <- Map 20 (SIMPLE_EDGE), Map 34 (SIMPLE_EDGE)
+Reducer 31 <- Map 33 (SIMPLE_EDGE), Reducer 30 (SIMPLE_EDGE)
+Reducer 32 <- Reducer 31 (SIMPLE_EDGE), Union 19 (CONTAINS)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 6 <- Union 11 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE), Union 19 (SIMPLE_EDGE), Union 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 9 <- Map 33 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:100
+ Stage-1
+ Reducer 7
+ File Output Operator [FS_184]
+ Limit [LIM_183] (rows=100 width=88)
+ Number of rows:100
+ Select Operator [SEL_182] (rows=574987681 width=88)
+ Output:["_col0","_col1","_col2"]
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_181]
+ Select Operator [SEL_180] (rows=574987681 width=88)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_179] (rows=574987681 width=88)
+ predicate:CASE WHEN ((_col1 > 0)) THEN (CASE WHEN ((_col3 > 0)) THEN (((_col5 / _col3) > (_col9 / _col1))) ELSE ((null > (_col9 / _col1))) END) ELSE (CASE WHEN ((_col3 > 0)) THEN (((_col5 / _col3) > null)) ELSE (null) END) END
+ Merge Join Operator [MERGEJOIN_339] (rows=1149975362 width=88)
+ Conds:Union 5._col0=Union 11._col0(Inner),Union 5._col0=Union 15._col0(Inner),Union 5._col0=Union 19._col0(Inner),Output:["_col1","_col3","_col5","_col6","_col7","_col8","_col9"]
+ <-Union 11 [SIMPLE_EDGE]
+ <-Reducer 10 [CONTAINS]
+ Reduce Output Operator [RS_175]
+ PartitionCols:_col0
+ Select Operator [SEL_63] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_62] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_61]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_60] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col4, _col7, _col8
+ Merge Join Operator [MERGEJOIN_328] (rows=1 width=105)
+ Conds:RS_56._col1=RS_57._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col0
+ Select Operator [SEL_52] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_305] (rows=1 width=860)
+ predicate:false
+ TableScan [TS_6] (rows=80000000 width=860)
+ default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name"]
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_327] (rows=1 width=96)
+ Conds:RS_53._col0=RS_54._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_54]
+ PartitionCols:_col0
+ Select Operator [SEL_49] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_304] (rows=1 width=1119)
+ predicate:false
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_53]
+ PartitionCols:_col0
+ Select Operator [SEL_46] (rows=1 width=88)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_303] (rows=1 width=88)
+ predicate:false
+ TableScan [TS_0] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_net_paid"]
+ <-Reducer 26 [CONTAINS]
+ Reduce Output Operator [RS_175]
+ PartitionCols:_col0
+ Select Operator [SEL_85] (rows=29040539 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_84] (rows=29040539 width=135)
+ predicate:(_col4 > 0)
+ Select Operator [SEL_321] (rows=87121617 width=135)
+ Output:["_col0","_col4"]
+ Group By Operator [GBY_83] (rows=87121617 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 25 [SIMPLE_EDGE]
+ SHUFFLE [RS_82]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_81] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4
+ Merge Join Operator [MERGEJOIN_330] (rows=174243235 width=135)
+ Conds:RS_77._col1=RS_78._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_78]
+ PartitionCols:_col0
+ Select Operator [SEL_73] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_308] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 24 [SIMPLE_EDGE]
+ SHUFFLE [RS_77]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_329] (rows=158402938 width=135)
+ Conds:RS_74._col0=RS_75._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_75]
+ PartitionCols:_col0
+ Select Operator [SEL_70] (rows=18262 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_307] (rows=18262 width=1119)
+ predicate:((d_year) IN (2001, 2002) and (d_year = 2001) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 34 [SIMPLE_EDGE]
+ SHUFFLE [RS_74]
+ PartitionCols:_col0
+ Select Operator [SEL_67] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_306] (rows=144002668 width=135)
+ predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
+ TableScan [TS_21] (rows=144002668 width=135)
+ default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk","ws_net_paid"]
+ <-Union 15 [SIMPLE_EDGE]
+ <-Reducer 14 [CONTAINS]
+ Reduce Output Operator [RS_176]
+ PartitionCols:_col0
+ Select Operator [SEL_107] (rows=1 width=105)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_106] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_105]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_104] (rows=1 width=105)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col4, _col7, _col8
+ Merge Join Operator [MERGEJOIN_332] (rows=1 width=105)
+ Conds:RS_100._col1=RS_101._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_101]
+ PartitionCols:_col0
+ Select Operator [SEL_96] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_311] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_100]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_331] (rows=1 width=96)
+ Conds:RS_97._col0=RS_98._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_98]
+ PartitionCols:_col0
+ Select Operator [SEL_93] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_310] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_97]
+ PartitionCols:_col0
+ Select Operator [SEL_90] (rows=1 width=88)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_309] (rows=1 width=88)
+ predicate:false
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 29 [CONTAINS]
+ Reduce Output Operator [RS_176]
+ PartitionCols:_col0
+ Select Operator [SEL_128] (rows=87121617 width=135)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_127] (rows=87121617 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 28 [SIMPLE_EDGE]
+ SHUFFLE [RS_126]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_125] (rows=174243235 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4
+ Merge Join Operator [MERGEJOIN_334] (rows=174243235 width=135)
+ Conds:RS_121._col1=RS_122._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_122]
+ PartitionCols:_col0
+ Select Operator [SEL_117] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_314] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 27 [SIMPLE_EDGE]
+ SHUFFLE [RS_121]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_333] (rows=158402938 width=135)
+ Conds:RS_118._col0=RS_119._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_119]
+ PartitionCols:_col0
+ Select Operator [SEL_114] (rows=18262 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_313] (rows=18262 width=1119)
+ predicate:((d_year) IN (2001, 2002) and (d_year = 2002) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 34 [SIMPLE_EDGE]
+ SHUFFLE [RS_118]
+ PartitionCols:_col0
+ Select Operator [SEL_111] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_312] (rows=144002668 width=135)
+ predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_21]
+ <-Union 19 [SIMPLE_EDGE]
+ <-Reducer 18 [CONTAINS]
+ Reduce Output Operator [RS_177]
+ PartitionCols:_col0
+ Select Operator [SEL_150] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2","_col3"]
+ Group By Operator [GBY_149] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_148]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_147] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4
+ Merge Join Operator [MERGEJOIN_336] (rows=696954748 width=88)
+ Conds:RS_143._col1=RS_144._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_144]
+ PartitionCols:_col0
+ Select Operator [SEL_139] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_317] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_143]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_335] (rows=633595212 width=88)
+ Conds:RS_140._col0=RS_141._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_141]
+ PartitionCols:_col0
+ Select Operator [SEL_136] (rows=18262 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_316] (rows=18262 width=1119)
+ predicate:((d_year) IN (2001, 2002) and (d_year = 2002) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_140]
+ PartitionCols:_col0
+ Select Operator [SEL_133] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_315] (rows=575995635 width=88)
+ predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Reducer 32 [CONTAINS]
+ Reduce Output Operator [RS_177]
+ PartitionCols:_col0
+ Select Operator [SEL_170] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3"]
+ Group By Operator [GBY_169] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_168]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_167] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col4, _col7, _col8
+ Merge Join Operator [MERGEJOIN_338] (rows=1 width=162)
+ Conds:RS_163._col1=RS_164._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_164]
+ PartitionCols:_col0
+ Select Operator [SEL_159] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_320] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 30 [SIMPLE_EDGE]
+ SHUFFLE [RS_163]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_337] (rows=1 width=148)
+ Conds:RS_160._col0=RS_161._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_161]
+ PartitionCols:_col0
+ Select Operator [SEL_156] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_319] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 34 [SIMPLE_EDGE]
+ SHUFFLE [RS_160]
+ PartitionCols:_col0
+ Select Operator [SEL_153] (rows=1 width=135)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_318] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_21]
+ <-Union 5 [SIMPLE_EDGE]
+ <-Reducer 23 [CONTAINS]
+ Reduce Output Operator [RS_174]
+ PartitionCols:_col0
+ Select Operator [SEL_40] (rows=1 width=162)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_39] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_38]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_37] (rows=1 width=162)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col4, _col7, _col8
+ Merge Join Operator [MERGEJOIN_326] (rows=1 width=162)
+ Conds:RS_33._col1=RS_34._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col0
+ Select Operator [SEL_29] (rows=1 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_302] (rows=1 width=860)
+ predicate:false
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_33]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_325] (rows=1 width=148)
+ Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_31]
+ PartitionCols:_col0
+ Select Operator [SEL_26] (rows=1 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_301] (rows=1 width=1119)
+ predicate:false
+ Please refer to the previous TableScan [TS_3]
+ <-Map 34 [SIMPLE_EDGE]
+ SHUFFLE [RS_30]
+ PartitionCols:_col0
+ Select Operator [SEL_23] (rows=1 width=135)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_300] (rows=1 width=135)
+ predicate:false
+ Please refer to the previous TableScan [TS_21]
+ <-Reducer 4 [CONTAINS]
+ Reduce Output Operator [RS_174]
+ PartitionCols:_col0
+ Select Operator [SEL_20] (rows=116159124 width=88)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_19] (rows=116159124 width=88)
+ predicate:(_col4 > 0)
+ Select Operator [SEL_322] (rows=348477374 width=88)
+ Output:["_col0","_col4"]
+ Group By Operator [GBY_18] (rows=348477374 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_17]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Group By Operator [GBY_16] (rows=696954748 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["max(_col2)"],keys:_col6, _col7, _col8, _col4
+ Merge Join Operator [MERGEJOIN_324] (rows=696954748 width=88)
+ Conds:RS_12._col1=RS_13._col0(Inner),Output:["_col2","_col4","_col6","_col7","_col8"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_299] (rows=80000000 width=860)
+ predicate:(c_customer_sk is not null and c_customer_id is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_323] (rows=633595212 width=88)
+ Conds:RS_9._col0=RS_10._col0(Inner),Output:["_col1","_col2","_col4"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_10]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=18262 width=1119)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_298] (rows=18262 width=1119)
+ predicate:((d_year) IN (2001, 2002) and (d_year = 2001) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_9]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_297] (rows=575995635 width=88)
+ predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+
[21/36] hive git commit: HIVE-16566: Set column stats default as true
when creating new tables/partitions (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
index 92babd7..9865af4 100644
--- a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
@@ -85,7 +85,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -180,7 +180,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -217,7 +217,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -247,7 +247,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -268,7 +268,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -304,7 +304,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -334,7 +334,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -355,7 +355,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -534,7 +534,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -565,7 +565,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1422,7 +1422,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1517,7 +1517,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1554,7 +1554,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1584,7 +1584,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1605,7 +1605,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1641,7 +1641,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1671,7 +1671,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1692,7 +1692,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1872,7 +1872,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -1903,7 +1903,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -4579,7 +4579,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4674,7 +4674,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4711,7 +4711,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4741,7 +4741,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4762,7 +4762,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4798,7 +4798,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4828,7 +4828,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -4849,7 +4849,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_sort_6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_sort_6.q.out b/ql/src/test/results/clientpositive/groupby_sort_6.q.out
index 80baa66..93e3f26 100644
--- a/ql/src/test/results/clientpositive/groupby_sort_6.q.out
+++ b/ql/src/test/results/clientpositive/groupby_sort_6.q.out
@@ -81,7 +81,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -112,7 +112,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
index 6e09753..16deb92 100644
--- a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
@@ -85,7 +85,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -180,7 +180,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -217,7 +217,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -247,7 +247,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -268,7 +268,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -304,7 +304,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -334,7 +334,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -355,7 +355,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -603,7 +603,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -634,7 +634,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1491,7 +1491,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1586,7 +1586,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1623,7 +1623,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1653,7 +1653,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1674,7 +1674,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1710,7 +1710,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1740,7 +1740,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1761,7 +1761,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -2010,7 +2010,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -2041,7 +2041,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -5062,7 +5062,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5157,7 +5157,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5194,7 +5194,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5224,7 +5224,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5245,7 +5245,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5281,7 +5281,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5311,7 +5311,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -5332,7 +5332,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/input_part1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_part1.q.out b/ql/src/test/results/clientpositive/input_part1.q.out
index 2f6f302..a685446 100644
--- a/ql/src/test/results/clientpositive/input_part1.q.out
+++ b/ql/src/test/results/clientpositive/input_part1.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -144,7 +144,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -181,7 +181,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -211,7 +211,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -232,7 +232,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -268,7 +268,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -298,7 +298,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -319,7 +319,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/input_part2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_part2.q.out b/ql/src/test/results/clientpositive/input_part2.q.out
index 95f36ab..d5524fe 100644
--- a/ql/src/test/results/clientpositive/input_part2.q.out
+++ b/ql/src/test/results/clientpositive/input_part2.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -108,7 +108,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -250,7 +250,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -287,7 +287,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -317,7 +317,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -338,7 +338,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -374,7 +374,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -404,7 +404,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -425,7 +425,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -471,7 +471,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -508,7 +508,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -538,7 +538,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -559,7 +559,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -595,7 +595,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -625,7 +625,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -646,7 +646,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out
index fa8417b..0d8e779 100644
--- a/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out
+++ b/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out
@@ -58,7 +58,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"cbigint\":\"true\",\"cboolean1\":\"true\",\"cboolean2\":\"true\",\"cdouble\":\"true\",\"cfloat\":\"true\",\"cint\":\"true\",\"csmallint\":\"true\",\"cstring1\":\"true\",\"cstring2\":\"true\",\"ctimestamp1\":\"true\",\"ctimestamp2\":\"true\",\"ctinyint\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -632,7 +632,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join17.q.out b/ql/src/test/results/clientpositive/join17.q.out
index 8bd67eb..f9edc79 100644
--- a/ql/src/test/results/clientpositive/join17.q.out
+++ b/ql/src/test/results/clientpositive/join17.q.out
@@ -141,7 +141,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","key2":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,value1,key2,value2
@@ -172,7 +172,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","key2":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,value1,key2,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join26.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join26.q.out b/ql/src/test/results/clientpositive/join26.q.out
index 5bc8245..b41fd8e 100644
--- a/ql/src/test/results/clientpositive/join26.q.out
+++ b/ql/src/test/results/clientpositive/join26.q.out
@@ -115,7 +115,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -293,7 +293,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join32.q.out b/ql/src/test/results/clientpositive/join32.q.out
index 0358968..a191284 100644
--- a/ql/src/test/results/clientpositive/join32.q.out
+++ b/ql/src/test/results/clientpositive/join32.q.out
@@ -120,7 +120,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -298,7 +298,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join33.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join33.q.out b/ql/src/test/results/clientpositive/join33.q.out
index 0358968..a191284 100644
--- a/ql/src/test/results/clientpositive/join33.q.out
+++ b/ql/src/test/results/clientpositive/join33.q.out
@@ -120,7 +120,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -298,7 +298,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join34.q.out b/ql/src/test/results/clientpositive/join34.q.out
index 117afac..67599bc 100644
--- a/ql/src/test/results/clientpositive/join34.q.out
+++ b/ql/src/test/results/clientpositive/join34.q.out
@@ -100,7 +100,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -159,7 +159,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -289,7 +289,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join35.q.out b/ql/src/test/results/clientpositive/join35.q.out
index 1f90df9..7ece4ef 100644
--- a/ql/src/test/results/clientpositive/join35.q.out
+++ b/ql/src/test/results/clientpositive/join35.q.out
@@ -200,7 +200,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -249,7 +249,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -377,7 +377,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join9.q.out b/ql/src/test/results/clientpositive/join9.q.out
index befc47a..a96f341 100644
--- a/ql/src/test/results/clientpositive/join9.q.out
+++ b/ql/src/test/results/clientpositive/join9.q.out
@@ -189,7 +189,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -220,7 +220,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/join_map_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_map_ppr.q.out b/ql/src/test/results/clientpositive/join_map_ppr.q.out
index dba176d..a4d4140 100644
--- a/ql/src/test/results/clientpositive/join_map_ppr.q.out
+++ b/ql/src/test/results/clientpositive/join_map_ppr.q.out
@@ -110,7 +110,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -205,7 +205,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -242,7 +242,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -272,7 +272,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -293,7 +293,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -329,7 +329,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -359,7 +359,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -380,7 +380,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out b/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
index ab30b7d..f827991 100644
--- a/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
@@ -44,7 +44,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -126,7 +126,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out
index e5e26ec..ec209b2 100644
--- a/ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out
@@ -516,7 +516,7 @@ Database: default
Table: alter5
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucket2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket2.q.out b/ql/src/test/results/clientpositive/llap/bucket2.q.out
index be886b0..e0c92ce 100644
--- a/ql/src/test/results/clientpositive/llap/bucket2.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket2.q.out
@@ -118,7 +118,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -153,7 +153,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucket4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket4.q.out b/ql/src/test/results/clientpositive/llap/bucket4.q.out
index b4f7ca1..5fbffc9 100644
--- a/ql/src/test/results/clientpositive/llap/bucket4.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket4.q.out
@@ -119,7 +119,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -155,7 +155,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucket5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket5.q.out b/ql/src/test/results/clientpositive/llap/bucket5.q.out
index ef5067b..0b5a14d 100644
--- a/ql/src/test/results/clientpositive/llap/bucket5.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket5.q.out
@@ -150,7 +150,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -192,7 +192,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -226,7 +226,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
@@ -260,7 +260,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -309,7 +309,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -339,7 +339,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -360,7 +360,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -399,7 +399,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -429,7 +429,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -450,7 +450,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucket_many.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_many.q.out b/ql/src/test/results/clientpositive/llap/bucket_many.q.out
index 72b0ec4..b78cbaa 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_many.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_many.q.out
@@ -118,7 +118,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 256
bucket_field_name key
column.name.delimiter ,
@@ -153,7 +153,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 256
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out
index 6338f66..feb3bbc 100644
--- a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out
@@ -584,7 +584,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -618,7 +618,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out
index 7e3d606..bda8070 100644
--- a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out
@@ -298,7 +298,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -332,7 +332,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out
index 00cf0f1..ae17c1b 100644
--- a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out
@@ -322,7 +322,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -356,7 +356,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
index 7b983f0..aab4343 100644
--- a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out
@@ -318,7 +318,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -352,7 +352,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out b/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out
index bc9258f..fb833bc 100644
--- a/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out
+++ b/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out
@@ -25,7 +25,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\" left\":\"true\",\" middle \":\"true\",\"right \":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -51,7 +51,7 @@ POSTHOOK: Input: default@space
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
left string from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\" left\":\"true\",\" middle \":\"true\",\"right \":\"true\"}}
PREHOOK: query: insert into space values ("1", "2", "3")
PREHOOK: type: QUERY
PREHOOK: Output: default@space
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out
index ca03652..4a2294f 100644
--- a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out
+++ b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out
@@ -118,7 +118,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -153,7 +153,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
index 35822f4..88c4a17 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out
@@ -1722,7 +1722,7 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","value1":"true"}}
bucket_count -1
#### A masked pattern was here ####
name default.srcpart_small
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out
index 18659b3..eb638d5 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out
@@ -986,7 +986,7 @@ STAGE PLANS:
partition values:
ds 2008-04-08
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","value1":"true"}}
bucket_count -1
#### A masked pattern was here ####
name default.srcpart_small
[35/36] hive git commit: HIVE-14671 : Merge branch 'master' into
hive-14535 (Wei Zheng)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index 6bed878,0000000..e08f1a8
mode 100644,000000..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2412 -1,0 +1,2443 @@@
+PREHOOK: query: drop table intermediate
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table intermediate
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@intermediate
+POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@intermediate
+PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=455
+POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=455
+POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=456
+POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=456
+POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@intermediate@p=457
+POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@intermediate@p=457
+POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: drop table part_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table part_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@part_mm
+POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@part_mm
+PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: intermediate
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.part_mm
+ Write Type: INSERT
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ key_mm 455
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.part_mm
+ micromanaged table: true
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-5
+ Merge File Operator
+ Map Operator Tree:
+ ORC File Merge Operator
+ merge level: stripe
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+0 455
+0 455
+0 456
+10 455
+10 455
+10 456
+97 455
+97 455
+97 456
+98 455
+98 455
+98 456
+100 455
+100 455
+100 456
+103 455
+103 455
+103 456
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+0 455
+0 455
+0 456
+10 455
+10 455
+10 456
+97 455
+97 455
+97 456
+98 455
+98 455
+98 456
+100 455
+100 455
+100 456
+103 455
+103 455
+103 456
+PREHOOK: query: truncate table part_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@part_mm@key_mm=455
+PREHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: query: truncate table part_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Output: default@part_mm@key_mm=456
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+PREHOOK: query: drop table part_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@part_mm
+PREHOOK: Output: default@part_mm
+POSTHOOK: query: drop table part_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Output: default@part_mm
+PREHOOK: query: drop table simple_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table simple_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: insert into table simple_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: insert into table simple_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@simple_mm
+POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from simple_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+0
+10
+97
+98
+100
+103
+PREHOOK: query: insert into table simple_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: insert into table simple_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@simple_mm
+POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from simple_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+0
+0
+10
+10
+97
+97
+98
+98
+100
+100
+103
+103
+PREHOOK: query: truncate table simple_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: truncate table simple_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: select * from simple_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+PREHOOK: query: drop table simple_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@simple_mm
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: drop table simple_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@simple_mm
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: drop table dp_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table dp_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
+ tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dp_mm
+POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
+ tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dp_mm
+PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@dp_mm@key1=123
+POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@dp_mm@key1=123/key2=0
+POSTHOOK: Output: default@dp_mm@key1=123/key2=10
+POSTHOOK: Output: default@dp_mm@key1=123/key2=100
+POSTHOOK: Output: default@dp_mm@key1=123/key2=103
+POSTHOOK: Output: default@dp_mm@key1=123/key2=97
+POSTHOOK: Output: default@dp_mm@key1=123/key2=98
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from dp_mm order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dp_mm
+PREHOOK: Input: default@dp_mm@key1=123/key2=0
+PREHOOK: Input: default@dp_mm@key1=123/key2=10
+PREHOOK: Input: default@dp_mm@key1=123/key2=100
+PREHOOK: Input: default@dp_mm@key1=123/key2=103
+PREHOOK: Input: default@dp_mm@key1=123/key2=97
+PREHOOK: Input: default@dp_mm@key1=123/key2=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from dp_mm order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dp_mm
+POSTHOOK: Input: default@dp_mm@key1=123/key2=0
+POSTHOOK: Input: default@dp_mm@key1=123/key2=10
+POSTHOOK: Input: default@dp_mm@key1=123/key2=100
+POSTHOOK: Input: default@dp_mm@key1=123/key2=103
+POSTHOOK: Input: default@dp_mm@key1=123/key2=97
+POSTHOOK: Input: default@dp_mm@key1=123/key2=98
+#### A masked pattern was here ####
+0 123 0
+10 123 10
+97 123 97
+98 123 98
+100 123 100
+103 123 103
+PREHOOK: query: drop table dp_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@dp_mm
+PREHOOK: Output: default@dp_mm
+POSTHOOK: query: drop table dp_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@dp_mm
+POSTHOOK: Output: default@dp_mm
+PREHOOK: query: create table union_mm(id int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: create table union_mm(id int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@union_mm
+PREHOOK: query: insert into table union_mm
+select temps.p from (
+select key as p from intermediate
+union all
+select key + 1 as p from intermediate ) temps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+select temps.p from (
+select key as p from intermediate
+union all
+select key + 1 as p from intermediate ) temps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+1
+10
+11
+97
+98
+98
+99
+100
+101
+103
+104
+PREHOOK: query: insert into table union_mm
+select p from
+(
+select key + 1 as p from intermediate
+union all
+select key from intermediate
+) tab group by p
+union all
+select key + 2 as p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+select p from
+(
+select key + 1 as p from intermediate
+union all
+select key from intermediate
+) tab group by p
+union all
+select key + 2 as p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+0
+1
+1
+2
+10
+10
+11
+11
+12
+97
+97
+98
+98
+98
+99
+99
+99
+100
+100
+100
+101
+101
+102
+103
+103
+104
+104
+105
+PREHOOK: query: insert into table union_mm
+SELECT p FROM
+(
+ SELECT key + 1 as p FROM intermediate
+ UNION ALL
+ SELECT key as p FROM (
+ SELECT distinct key FROM (
+ SELECT key FROM (
+ SELECT key + 2 as key FROM intermediate
+ UNION ALL
+ SELECT key FROM intermediate
+ )t1
+ group by key)t2
+ )t3
+)t4
+group by p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: insert into table union_mm
+SELECT p FROM
+(
+ SELECT key + 1 as p FROM intermediate
+ UNION ALL
+ SELECT key as p FROM (
+ SELECT distinct key FROM (
+ SELECT key FROM (
+ SELECT key + 2 as key FROM intermediate
+ UNION ALL
+ SELECT key FROM intermediate
+ )t1
+ group by key)t2
+ )t3
+)t4
+group by p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@union_mm
+POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from union_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from union_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@union_mm
+#### A masked pattern was here ####
+0
+0
+0
+1
+1
+1
+2
+2
+10
+10
+10
+11
+11
+11
+12
+12
+97
+97
+97
+98
+98
+98
+98
+99
+99
+99
+99
+100
+100
+100
+100
+101
+101
+101
+102
+102
+103
+103
+103
+104
+104
+104
+105
+105
+PREHOOK: query: drop table union_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@union_mm
+PREHOOK: Output: default@union_mm
+POSTHOOK: query: drop table union_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@union_mm
+POSTHOOK: Output: default@union_mm
+PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@partunion_mm
+PREHOOK: query: insert into table partunion_mm partition(key)
+select temps.* from (
+select key as p, key from intermediate
+union all
+select key + 1 as p, key + 1 from intermediate ) temps
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: insert into table partunion_mm partition(key)
+select temps.* from (
+select key as p, key from intermediate
+union all
+select key + 1 as p, key + 1 from intermediate ) temps
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@partunion_mm@key=0
+POSTHOOK: Output: default@partunion_mm@key=1
+POSTHOOK: Output: default@partunion_mm@key=10
+POSTHOOK: Output: default@partunion_mm@key=100
+POSTHOOK: Output: default@partunion_mm@key=101
+POSTHOOK: Output: default@partunion_mm@key=103
+POSTHOOK: Output: default@partunion_mm@key=104
+POSTHOOK: Output: default@partunion_mm@key=11
+POSTHOOK: Output: default@partunion_mm@key=97
+POSTHOOK: Output: default@partunion_mm@key=98
+POSTHOOK: Output: default@partunion_mm@key=99
+POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=100).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=101).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=103).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=104).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from partunion_mm order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@partunion_mm
+PREHOOK: Input: default@partunion_mm@key=0
+PREHOOK: Input: default@partunion_mm@key=1
+PREHOOK: Input: default@partunion_mm@key=10
+PREHOOK: Input: default@partunion_mm@key=100
+PREHOOK: Input: default@partunion_mm@key=101
+PREHOOK: Input: default@partunion_mm@key=103
+PREHOOK: Input: default@partunion_mm@key=104
+PREHOOK: Input: default@partunion_mm@key=11
+PREHOOK: Input: default@partunion_mm@key=97
+PREHOOK: Input: default@partunion_mm@key=98
+PREHOOK: Input: default@partunion_mm@key=99
+#### A masked pattern was here ####
+POSTHOOK: query: select * from partunion_mm order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@partunion_mm
+POSTHOOK: Input: default@partunion_mm@key=0
+POSTHOOK: Input: default@partunion_mm@key=1
+POSTHOOK: Input: default@partunion_mm@key=10
+POSTHOOK: Input: default@partunion_mm@key=100
+POSTHOOK: Input: default@partunion_mm@key=101
+POSTHOOK: Input: default@partunion_mm@key=103
+POSTHOOK: Input: default@partunion_mm@key=104
+POSTHOOK: Input: default@partunion_mm@key=11
+POSTHOOK: Input: default@partunion_mm@key=97
+POSTHOOK: Input: default@partunion_mm@key=98
+POSTHOOK: Input: default@partunion_mm@key=99
+#### A masked pattern was here ####
+0 0
+1 1
+10 10
+11 11
+97 97
+98 98
+98 98
+99 99
+100 100
+101 101
+103 103
+104 104
+PREHOOK: query: drop table partunion_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@partunion_mm
+PREHOOK: Output: default@partunion_mm
+POSTHOOK: query: drop table partunion_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@partunion_mm
+POSTHOOK: Output: default@partunion_mm
+PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
+ stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: insert into table skew_mm
+select key, key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@skew_mm
+POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_mm order by k2, k1, k4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_mm order by k2, k1, k4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_mm
+#### A masked pattern was here ####
+0 0 0
+10 10 10
+97 97 97
+98 98 98
+100 100 100
+103 103 103
+PREHOOK: query: drop table skew_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_mm
+PREHOOK: Output: default@skew_mm
+POSTHOOK: query: drop table skew_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_mm
+POSTHOOK: Output: default@skew_mm
+PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int)
+skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
+select key as i, key as j, key as k, key as l from intermediate
+union all
+select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@skew_dp_union_mm@k3=0
+POSTHOOK: Output: default@skew_dp_union_mm@k3=10
+POSTHOOK: Output: default@skew_dp_union_mm@k3=100
+POSTHOOK: Output: default@skew_dp_union_mm@k3=101
+POSTHOOK: Output: default@skew_dp_union_mm@k3=102
+POSTHOOK: Output: default@skew_dp_union_mm@k3=103
+POSTHOOK: Output: default@skew_dp_union_mm@k3=104
+POSTHOOK: Output: default@skew_dp_union_mm@k3=107
+POSTHOOK: Output: default@skew_dp_union_mm@k3=14
+POSTHOOK: Output: default@skew_dp_union_mm@k3=4
+POSTHOOK: Output: default@skew_dp_union_mm@k3=97
+POSTHOOK: Output: default@skew_dp_union_mm@k3=98
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Input: default@skew_dp_union_mm@k3=0
+PREHOOK: Input: default@skew_dp_union_mm@k3=10
+PREHOOK: Input: default@skew_dp_union_mm@k3=100
+PREHOOK: Input: default@skew_dp_union_mm@k3=101
+PREHOOK: Input: default@skew_dp_union_mm@k3=102
+PREHOOK: Input: default@skew_dp_union_mm@k3=103
+PREHOOK: Input: default@skew_dp_union_mm@k3=104
+PREHOOK: Input: default@skew_dp_union_mm@k3=107
+PREHOOK: Input: default@skew_dp_union_mm@k3=14
+PREHOOK: Input: default@skew_dp_union_mm@k3=4
+PREHOOK: Input: default@skew_dp_union_mm@k3=97
+PREHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Input: default@skew_dp_union_mm@k3=0
+POSTHOOK: Input: default@skew_dp_union_mm@k3=10
+POSTHOOK: Input: default@skew_dp_union_mm@k3=100
+POSTHOOK: Input: default@skew_dp_union_mm@k3=101
+POSTHOOK: Input: default@skew_dp_union_mm@k3=102
+POSTHOOK: Input: default@skew_dp_union_mm@k3=103
+POSTHOOK: Input: default@skew_dp_union_mm@k3=104
+POSTHOOK: Input: default@skew_dp_union_mm@k3=107
+POSTHOOK: Input: default@skew_dp_union_mm@k3=14
+POSTHOOK: Input: default@skew_dp_union_mm@k3=4
+POSTHOOK: Input: default@skew_dp_union_mm@k3=97
+POSTHOOK: Input: default@skew_dp_union_mm@k3=98
+#### A masked pattern was here ####
+0 0 0 0
+1 2 3 4
+10 10 10 10
+11 12 13 14
+97 97 97 97
+98 98 98 98
+98 99 100 101
+99 100 101 102
+100 100 100 100
+101 102 103 104
+103 103 103 103
+104 105 106 107
+PREHOOK: query: drop table skew_dp_union_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@skew_dp_union_mm
+PREHOOK: Output: default@skew_dp_union_mm
+POSTHOOK: query: drop table skew_dp_union_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@skew_dp_union_mm
+POSTHOOK: Output: default@skew_dp_union_mm
+PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge0_mm
+PREHOOK: query: insert into table merge0_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: insert into table merge0_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge0_mm
+POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+PREHOOK: query: insert into table merge0_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: insert into table merge0_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge0_mm
+POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge0_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+98
+97
+0
+10
+100
+103
+PREHOOK: query: drop table merge0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge0_mm
+PREHOOK: Output: default@merge0_mm
+POSTHOOK: query: drop table merge0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge0_mm
+POSTHOOK: Output: default@merge0_mm
+PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge2_mm
+PREHOOK: query: insert into table merge2_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: insert into table merge2_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge2_mm
+POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+PREHOOK: query: insert into table merge2_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: insert into table merge2_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge2_mm
+POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge2_mm
+#### A masked pattern was here ####
+98
+97
+0
+10
+100
+103
+98
+97
+0
+10
+100
+103
+PREHOOK: query: drop table merge2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge2_mm
+PREHOOK: Output: default@merge2_mm
+POSTHOOK: query: drop table merge2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge2_mm
+POSTHOOK: Output: default@merge2_mm
+PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@merge1_mm
+PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge1_mm@key=0
+POSTHOOK: Output: default@merge1_mm@key=10
+POSTHOOK: Output: default@merge1_mm@key=100
+POSTHOOK: Output: default@merge1_mm@key=103
+POSTHOOK: Output: default@merge1_mm@key=97
+POSTHOOK: Output: default@merge1_mm@key=98
+POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge1_mm order by id, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Input: default@merge1_mm@key=0
+PREHOOK: Input: default@merge1_mm@key=10
+PREHOOK: Input: default@merge1_mm@key=100
+PREHOOK: Input: default@merge1_mm@key=103
+PREHOOK: Input: default@merge1_mm@key=97
+PREHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge1_mm order by id, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Input: default@merge1_mm@key=0
+POSTHOOK: Input: default@merge1_mm@key=10
+POSTHOOK: Input: default@merge1_mm@key=100
+POSTHOOK: Input: default@merge1_mm@key=103
+POSTHOOK: Input: default@merge1_mm@key=97
+POSTHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+0 0
+10 10
+97 97
+98 98
+100 100
+103 103
+PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@merge1_mm@key=0
+POSTHOOK: Output: default@merge1_mm@key=10
+POSTHOOK: Output: default@merge1_mm@key=100
+POSTHOOK: Output: default@merge1_mm@key=103
+POSTHOOK: Output: default@merge1_mm@key=97
+POSTHOOK: Output: default@merge1_mm@key=98
+POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from merge1_mm order by id, key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Input: default@merge1_mm@key=0
+PREHOOK: Input: default@merge1_mm@key=10
+PREHOOK: Input: default@merge1_mm@key=100
+PREHOOK: Input: default@merge1_mm@key=103
+PREHOOK: Input: default@merge1_mm@key=97
+PREHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+POSTHOOK: query: select * from merge1_mm order by id, key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Input: default@merge1_mm@key=0
+POSTHOOK: Input: default@merge1_mm@key=10
+POSTHOOK: Input: default@merge1_mm@key=100
+POSTHOOK: Input: default@merge1_mm@key=103
+POSTHOOK: Input: default@merge1_mm@key=97
+POSTHOOK: Input: default@merge1_mm@key=98
+#### A masked pattern was here ####
+0 0
+0 0
+10 10
+10 10
+97 97
+97 97
+98 98
+98 98
+100 100
+100 100
+103 103
+103 103
+PREHOOK: query: drop table merge1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@merge1_mm
+PREHOOK: Output: default@merge1_mm
+POSTHOOK: query: drop table merge1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@merge1_mm
+POSTHOOK: Output: default@merge1_mm
+PREHOOK: query: drop table ctas0_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table ctas0_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas0_mm
+POSTHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas0_mm
+POSTHOOK: Lineage: ctas0_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: ctas0_mm.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from ctas0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas0_mm
+#### A masked pattern was here ####
+98 455
+97 455
+0 456
+10 456
+100 457
+103 457
+PREHOOK: query: drop table ctas0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas0_mm
+PREHOOK: Output: default@ctas0_mm
+POSTHOOK: query: drop table ctas0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas0_mm
+POSTHOOK: Output: default@ctas0_mm
+PREHOOK: query: drop table ctas1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table ctas1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
+ select * from intermediate union all select * from intermediate
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas1_mm
+POSTHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
+ select * from intermediate union all select * from intermediate
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas1_mm
+POSTHOOK: Lineage: ctas1_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: ctas1_mm.p EXPRESSION [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from ctas1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas1_mm
+#### A masked pattern was here ####
+98 455
+98 455
+97 455
+97 455
+0 456
+0 456
+10 456
+10 456
+100 457
+100 457
+103 457
+103 457
+PREHOOK: query: drop table ctas1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas1_mm
+PREHOOK: Output: default@ctas1_mm
+POSTHOOK: query: drop table ctas1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas1_mm
+POSTHOOK: Output: default@ctas1_mm
+PREHOOK: query: drop table load0_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load0_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: select count(1) from load0_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load0_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load0_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table load0_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load0_mm
+PREHOOK: Output: default@load0_mm
+POSTHOOK: query: drop table load0_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load0_mm
+POSTHOOK: Output: default@load0_mm
+PREHOOK: query: drop table intermediate2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table intermediate2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+1050
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: select count(1) from load1_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load1_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load1_mm
+#### A masked pattern was here ####
+500
+PREHOOK: query: drop table load1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load1_mm
+PREHOOK: Output: default@load1_mm
+POSTHOOK: query: drop table load1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load1_mm
+POSTHOOK: Output: default@load1_mm
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: create table load2_mm (key string, value string)
+ partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@intermediate2
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load2_mm
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load2_mm
+POSTHOOK: Output: default@load2_mm@k=5/l=5
+PREHOOK: query: select count(1) from load2_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load2_mm
+PREHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from load2_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Input: default@load2_mm@k=5/l=5
+#### A masked pattern was here ####
+1025
+PREHOOK: query: drop table load2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load2_mm
+PREHOOK: Output: default@load2_mm
+POSTHOOK: query: drop table load2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load2_mm
+POSTHOOK: Output: default@load2_mm
+PREHOOK: query: drop table intermediate2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@intermediate2
+PREHOOK: Output: default@intermediate2
+POSTHOOK: query: drop table intermediate2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@intermediate2
+POSTHOOK: Output: default@intermediate2
+PREHOOK: query: drop table multi0_1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi0_1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table multi0_2_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi0_2_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: create table multi0_1_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi0_1_mm
+PREHOOK: query: create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: create table multi0_2_mm (key int, key2 int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi0_2_mm
+PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_2_mm
+POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select * from multi0_1_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_1_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+PREHOOK: query: select * from multi0_2_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_2_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi0_2_mm
+POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi0_1_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_1_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_1_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: select * from multi0_2_mm order by key, key2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi0_2_mm order by key, key2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi0_2_mm
+#### A masked pattern was here ####
+0 456
+10 456
+97 455
+98 455
+100 457
+103 457
+455 97
+455 98
+456 0
+456 10
+457 100
+457 103
+PREHOOK: query: drop table multi0_1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi0_1_mm
+PREHOOK: Output: default@multi0_1_mm
+POSTHOOK: query: drop table multi0_1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi0_1_mm
+POSTHOOK: Output: default@multi0_1_mm
+PREHOOK: query: drop table multi0_2_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi0_2_mm
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: drop table multi0_2_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi0_2_mm
+POSTHOOK: Output: default@multi0_2_mm
+PREHOOK: query: drop table multi1_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table multi1_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@multi1_mm
+POSTHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@multi1_mm
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p=1) select p, key
+insert into table multi1_mm partition(p=2) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=1
+PREHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p=1) select p, key
+insert into table multi1_mm partition(p=2) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+0 456 2
+10 456 2
+97 455 2
+98 455 2
+100 457 2
+103 457 2
+455 97 1
+455 98 1
+456 0 1
+456 10 1
+457 100 1
+457 103 1
+PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+PREHOOK: query: select * from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
+#### A masked pattern was here ####
+0 456 1
+0 456 2
+10 456 1
+10 456 2
+97 455 1
+97 455 2
+98 455 1
+98 455 2
+100 457 1
+100 457 2
+103 457 1
+103 457 2
+455 97 1
+455 97 2
+455 98 1
+455 98 2
+456 0 1
+456 0 2
+456 10 1
+456 10 2
+457 100 1
+457 100 2
+457 103 1
+457 103 2
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, p
+insert into table multi1_mm partition(p=1) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, p
+insert into table multi1_mm partition(p=1) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
++POSTHOOK: Output: default@multi1_mm@p=455
++POSTHOOK: Output: default@multi1_mm@p=456
++POSTHOOK: Output: default@multi1_mm@p=457
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
++PREHOOK: Input: default@multi1_mm@p=455
++PREHOOK: Input: default@multi1_mm@p=456
++PREHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
++POSTHOOK: Input: default@multi1_mm@p=455
++POSTHOOK: Input: default@multi1_mm@p=456
++POSTHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+0 456 1
+0 456 1
+0 456 2
+10 456 1
+10 456 1
+10 456 2
+97 455 1
+97 455 1
+97 455 2
+98 455 1
+98 455 1
+98 455 2
+100 457 1
+100 457 1
+100 457 2
+103 457 1
+103 457 1
+103 457 2
+455 97 1
+455 97 2
++455 97 455
+455 98 1
+455 98 2
++455 98 455
+456 0 1
+456 0 2
++456 0 456
+456 10 1
+456 10 2
++456 10 456
+457 100 1
+457 100 2
++457 100 457
+457 103 1
+457 103 2
++457 103 457
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, 1
+insert into table multi1_mm partition(p=1) select key, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@multi1_mm
+PREHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p) select p, key, 1
+insert into table multi1_mm partition(p=1) select key, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+PREHOOK: type: QUERY
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Input: default@multi1_mm@p=1
+PREHOOK: Input: default@multi1_mm@p=2
++PREHOOK: Input: default@multi1_mm@p=455
++PREHOOK: Input: default@multi1_mm@p=456
++PREHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Input: default@multi1_mm@p=1
+POSTHOOK: Input: default@multi1_mm@p=2
++POSTHOOK: Input: default@multi1_mm@p=455
++POSTHOOK: Input: default@multi1_mm@p=456
++POSTHOOK: Input: default@multi1_mm@p=457
+#### A masked pattern was here ####
+0 456 1
+0 456 1
+0 456 1
+0 456 2
+10 456 1
+10 456 1
+10 456 1
+10 456 2
+97 455 1
+97 455 1
+97 455 1
+97 455 2
+98 455 1
+98 455 1
+98 455 1
+98 455 2
+100 457 1
+100 457 1
+100 457 1
+100 457 2
+103 457 1
+103 457 1
+103 457 1
+103 457 2
+455 97 1
+455 97 1
+455 97 2
++455 97 455
+455 98 1
+455 98 1
+455 98 2
++455 98 455
+456 0 1
+456 0 1
+456 0 2
++456 0 456
+456 10 1
+456 10 1
+456 10 2
++456 10 456
+457 100 1
+457 100 1
+457 100 2
++457 100 457
+457 103 1
+457 103 1
+457 103 2
++457 103 457
+PREHOOK: query: drop table multi1_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@multi1_mm
+PREHOOK: Output: default@multi1_mm
+POSTHOOK: query: drop table multi1_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@multi1_mm
+POSTHOOK: Output: default@multi1_mm
+PREHOOK: query: drop table stats_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table stats_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table stats_mm(key int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_mm
+POSTHOOK: query: create table stats_mm(key int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_mm
+PREHOOK: query: insert into table stats_mm select key from intermediate
+PREHOOK: type: QUERY
+PREHOOK: Input: default@intermediate
+PREHOOK: Input: default@intermediate@p=455
+PREHOOK: Input: default@intermediate@p=456
+PREHOOK: Input: default@intermediate@p=457
+PREHOOK: Output: default@stats_mm
+POSTHOOK: query: insert into table stats_mm select key from intermediate
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@intermediate
+POSTHOOK: Input: default@intermediate@p=455
+POSTHOOK: Input: default@intermediate@p=456
+POSTHOOK: Input: default@intermediate@p=457
+POSTHOOK: Output: default@stats_mm
+POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: desc formatted stats_mm
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@stats_mm
+POSTHOOK: query: desc formatted stats_mm
+POSTHOOK: type: DESCTABLE
+POSTHOOK: I
<TRUNCATED>
[08/36] hive git commit: HIVE-16774: Support position in ORDER BY
when using SELECT * (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16774: Support position in ORDER BY when using SELECT * (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d7ab32f2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d7ab32f2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d7ab32f2
Branch: refs/heads/hive-14535
Commit: d7ab32f2e657c5f11adf8d0d3143fc1b00223d88
Parents: 9e6c23d
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Wed May 31 17:44:49 2017 -0700
Committer: Pengcheng Xiong <px...@hortonworks.com>
Committed: Wed May 31 17:44:49 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/CalcitePlanner.java | 85 +++--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 36 +-
.../orderby_position_unsupported.q | 3 +-
.../test/queries/clientpositive/order_by_pos.q | 20 ++
.../test/queries/clientpositive/perf/query47.q | 53 +++
.../test/queries/clientpositive/perf/query57.q | 50 +++
.../orderby_position_unsupported.q.out | 4 +-
.../results/clientpositive/order_by_pos.q.out | 129 +++++++
.../results/clientpositive/perf/query47.q.out | 357 +++++++++++++++++++
.../results/clientpositive/perf/query57.q.out | 351 ++++++++++++++++++
10 files changed, 1020 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 721dac8..7f583ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -3104,38 +3104,63 @@ public class CalcitePlanner extends SemanticAnalyzer {
ASTNode ref = (ASTNode) nullObASTExpr.getChild(0);
Map<ASTNode, ExprNodeDesc> astToExprNDescMap = null;
ExprNodeDesc obExprNDesc = null;
- // first try to get it from select
- // in case of udtf, selectOutputRR may be null.
- if (selectOutputRR != null) {
- try {
- astToExprNDescMap = genAllExprNodeDesc(ref, selectOutputRR);
- obExprNDesc = astToExprNDescMap.get(ref);
- } catch (SemanticException ex) {
- // we can tolerate this as this is the previous behavior
- LOG.debug("Can not find column in " + ref.getText() + ". The error msg is "
- + ex.getMessage());
+
+ boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS);
+ boolean isObyByPos = isBothByPos
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ORDERBY_POSITION_ALIAS);
+ // replace each of the position alias in ORDERBY with the actual column
+ if (ref != null && ref.getToken().getType() == HiveParser.Number) {
+ if (isObyByPos) {
+ int pos = Integer.parseInt(ref.getText());
+ if (pos > 0 && pos <= selectOutputRR.getColumnInfos().size()) {
+ // fieldIndex becomes so simple
+ // Note that pos starts from 1 while fieldIndex starts from 0;
+ fieldIndex = pos - 1;
+ } else {
+ throw new SemanticException(
+ ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg("Position alias: " + pos
+ + " does not exist\n" + "The Select List is indexed from 1 to "
+ + selectOutputRR.getColumnInfos().size()));
+ }
+ } else { // if not using position alias and it is a number.
+ LOG.warn("Using constant number "
+ + ref.getText()
+ + " in order by. If you try to use position alias when hive.orderby.position.alias is false, the position alias will be ignored.");
}
- }
- // then try to get it from all
- if (obExprNDesc == null) {
- astToExprNDescMap = genAllExprNodeDesc(ref, inputRR);
- obExprNDesc = astToExprNDescMap.get(ref);
- }
- if (obExprNDesc == null) {
- throw new SemanticException("Invalid order by expression: " + obASTExpr.toString());
- }
- // 2.2 Convert ExprNode to RexNode
- rnd = converter.convert(obExprNDesc);
-
- // 2.3 Determine the index of ob expr in child schema
- // NOTE: Calcite can not take compound exprs in OB without it being
- // present in the child (& hence we add a child Project Rel)
- if (rnd instanceof RexInputRef) {
- fieldIndex = ((RexInputRef) rnd).getIndex();
} else {
- fieldIndex = srcRelRecordSz + newVCLst.size();
- newVCLst.add(rnd);
- vcASTTypePairs.add(new Pair<ASTNode, TypeInfo>(ref, obExprNDesc.getTypeInfo()));
+ // first try to get it from select
+ // in case of udtf, selectOutputRR may be null.
+ if (selectOutputRR != null) {
+ try {
+ astToExprNDescMap = genAllExprNodeDesc(ref, selectOutputRR);
+ obExprNDesc = astToExprNDescMap.get(ref);
+ } catch (SemanticException ex) {
+ // we can tolerate this as this is the previous behavior
+ LOG.debug("Can not find column in " + ref.getText() + ". The error msg is "
+ + ex.getMessage());
+ }
+ }
+ // then try to get it from all
+ if (obExprNDesc == null) {
+ astToExprNDescMap = genAllExprNodeDesc(ref, inputRR);
+ obExprNDesc = astToExprNDescMap.get(ref);
+ }
+ if (obExprNDesc == null) {
+ throw new SemanticException("Invalid order by expression: " + obASTExpr.toString());
+ }
+ // 2.2 Convert ExprNode to RexNode
+ rnd = converter.convert(obExprNDesc);
+
+ // 2.3 Determine the index of ob expr in child schema
+ // NOTE: Calcite can not take compound exprs in OB without it being
+ // present in the child (& hence we add a child Project Rel)
+ if (rnd instanceof RexInputRef) {
+ fieldIndex = ((RexInputRef) rnd).getIndex();
+ } else {
+ fieldIndex = srcRelRecordSz + newVCLst.size();
+ newVCLst.add(rnd);
+ vcASTTypePairs.add(new Pair<ASTNode, TypeInfo>(ref, obExprNDesc.getTypeInfo()));
+ }
}
// 2.4 Determine the Direction of order by
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 7f5051c..d514644 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12563,41 +12563,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
- // replace each of the position alias in ORDERBY with the actual column name
- if (orderbyNode != null) {
- isAllCol = false;
- for (int child_pos = 0; child_pos < selectNode.getChildCount(); ++child_pos) {
- ASTNode node = (ASTNode) selectNode.getChild(child_pos).getChild(0);
- if (node != null && node.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
- isAllCol = true;
- }
- }
- for (int child_pos = 0; child_pos < orderbyNode.getChildCount(); ++child_pos) {
- ASTNode colNode = (ASTNode) orderbyNode.getChild(child_pos).getChild(0);
- ASTNode node = (ASTNode) colNode.getChild(0);
- if (node != null && node.getToken().getType() == HiveParser.Number) {
- if (isObyByPos) {
- if (!isAllCol) {
- int pos = Integer.parseInt(node.getText());
- if (pos > 0 && pos <= selectExpCnt) {
- colNode.setChild(0, selectNode.getChild(pos - 1).getChild(0));
- } else {
- throw new SemanticException(
- ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg(
- "Position alias: " + pos + " does not exist\n" +
- "The Select List is indexed from 1 to " + selectExpCnt));
- }
- } else {
- throw new SemanticException(
- ErrorMsg.NO_SUPPORTED_ORDERBY_ALLCOLREF_POS.getMsg());
- }
- } else { //if not using position alias and it is a number.
- warn("Using constant number " + node.getText() +
- " in order by. If you try to use position alias when hive.orderby.position.alias is false, the position alias will be ignored.");
- }
- }
- }
- }
+ // orderby position will be processed in genPlan
}
for (int i = next.getChildren().size() - 1; i >= 0; i--) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/queries/clientnegative/orderby_position_unsupported.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/orderby_position_unsupported.q b/ql/src/test/queries/clientnegative/orderby_position_unsupported.q
index a490c23..01e897d 100644
--- a/ql/src/test/queries/clientnegative/orderby_position_unsupported.q
+++ b/ql/src/test/queries/clientnegative/orderby_position_unsupported.q
@@ -1,4 +1,3 @@
set hive.groupby.orderby.position.alias=true;
--- position alias is not supported when SELECT *
-SELECT src.* FROM src ORDER BY 1;
+SELECT src.* FROM src ORDER BY 3;
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/queries/clientpositive/order_by_pos.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/order_by_pos.q b/ql/src/test/queries/clientpositive/order_by_pos.q
new file mode 100644
index 0000000..744abe5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/order_by_pos.q
@@ -0,0 +1,20 @@
+set hive.fetch.task.conversion=none;
+
+create table t(a int, b int);
+
+insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9);
+
+select * from t order by 2;
+
+select * from t order by 1;
+
+select * from t union select * from t order by 1, 2;
+
+select * from t union select * from t order by 2;
+
+select * from t union select * from t order by 1;
+
+select * from (select a, count(a) from t group by a)subq order by 2, 1;
+
+select * from (select a,b, count(*) from t group by a, b)subq order by 3, 2 desc;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/queries/clientpositive/perf/query47.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/perf/query47.q b/ql/src/test/queries/clientpositive/perf/query47.q
new file mode 100644
index 0000000..5c26ba5
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/perf/query47.q
@@ -0,0 +1,53 @@
+set hive.mapred.mode=nonstrict;
+-- start query 1 in stream 0 using template query47.tpl and seed 2031708268
+explain
+with v1 as(
+ select i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy,
+ sum(ss_sales_price) sum_sales,
+ avg(sum(ss_sales_price)) over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name
+ order by d_year, d_moy) rn
+ from item, store_sales, date_dim, store
+ where ss_item_sk = i_item_sk and
+ ss_sold_date_sk = d_date_sk and
+ ss_store_sk = s_store_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy),
+ v2 as(
+ select v1.i_category
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1.s_store_name = v1_lag.s_store_name and
+ v1.s_store_name = v1_lead.s_store_name and
+ v1.s_company_name = v1_lag.s_company_name and
+ v1.s_company_name = v1_lead.s_company_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100;
+
+-- end query 1 in stream 0 using template query47.tpl
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/queries/clientpositive/perf/query57.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/perf/query57.q b/ql/src/test/queries/clientpositive/perf/query57.q
new file mode 100644
index 0000000..4dc6e63
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/perf/query57.q
@@ -0,0 +1,50 @@
+set hive.mapred.mode=nonstrict;
+-- start query 1 in stream 0 using template query57.tpl and seed 2031708268
+explain
+with v1 as(
+ select i_category, i_brand,
+ cc_name,
+ d_year, d_moy,
+ sum(cs_sales_price) sum_sales,
+ avg(sum(cs_sales_price)) over
+ (partition by i_category, i_brand,
+ cc_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ cc_name
+ order by d_year, d_moy) rn
+ from item, catalog_sales, date_dim, call_center
+ where cs_item_sk = i_item_sk and
+ cs_sold_date_sk = d_date_sk and
+ cc_call_center_sk= cs_call_center_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ cc_name , d_year, d_moy),
+ v2 as(
+ select v1.i_category, v1.i_brand
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1. cc_name = v1_lag. cc_name and
+ v1. cc_name = v1_lead. cc_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100;
+
+-- end query 1 in stream 0 using template query57.tpl
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/results/clientnegative/orderby_position_unsupported.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/orderby_position_unsupported.q.out b/ql/src/test/results/clientnegative/orderby_position_unsupported.q.out
index 56d3240..a275224 100644
--- a/ql/src/test/results/clientnegative/orderby_position_unsupported.q.out
+++ b/ql/src/test/results/clientnegative/orderby_position_unsupported.q.out
@@ -1 +1,3 @@
-FAILED: SemanticException [Error 10219]: Position in ORDER BY is not supported when using SELECT *
+FAILED: SemanticException [Error 10221]: Invalid position alias in Order By
+ Position alias: 3 does not exist
+The Select List is indexed from 1 to 2
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/results/clientpositive/order_by_pos.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/order_by_pos.q.out b/ql/src/test/results/clientpositive/order_by_pos.q.out
new file mode 100644
index 0000000..b7ea716
--- /dev/null
+++ b/ql/src/test/results/clientpositive/order_by_pos.q.out
@@ -0,0 +1,129 @@
+PREHOOK: query: create table t(a int, b int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: create table t(a int, b int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9)
+PREHOOK: type: QUERY
+PREHOOK: Output: default@t
+POSTHOOK: query: insert into t values (1,2),(1,2),(1,3),(2,4),(20,-100),(-1000,100),(4,5),(3,7),(8,9)
+POSTHOOK: type: QUERY
+POSTHOOK: Output: default@t
+POSTHOOK: Lineage: t.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: t.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: select * from t order by 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t order by 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+20 -100
+1 2
+1 2
+1 3
+2 4
+4 5
+3 7
+8 9
+-1000 100
+PREHOOK: query: select * from t order by 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t order by 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+-1000 100
+1 3
+1 2
+1 2
+2 4
+3 7
+4 5
+8 9
+20 -100
+PREHOOK: query: select * from t union select * from t order by 1, 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t union select * from t order by 1, 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+-1000 100
+1 2
+1 3
+2 4
+3 7
+4 5
+8 9
+20 -100
+PREHOOK: query: select * from t union select * from t order by 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t union select * from t order by 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+20 -100
+1 2
+1 3
+2 4
+4 5
+3 7
+8 9
+-1000 100
+PREHOOK: query: select * from t union select * from t order by 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from t union select * from t order by 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+-1000 100
+1 3
+1 2
+2 4
+3 7
+4 5
+8 9
+20 -100
+PREHOOK: query: select * from (select a, count(a) from t group by a)subq order by 2, 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select a, count(a) from t group by a)subq order by 2, 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+-1000 1
+2 1
+3 1
+4 1
+8 1
+20 1
+1 3
+PREHOOK: query: select * from (select a,b, count(*) from t group by a, b)subq order by 3, 2 desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select a,b, count(*) from t group by a, b)subq order by 3, 2 desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t
+#### A masked pattern was here ####
+-1000 100 1
+8 9 1
+3 7 1
+4 5 1
+2 4 1
+1 3 1
+20 -100 1
+1 2 2
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/results/clientpositive/perf/query47.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query47.q.out b/ql/src/test/results/clientpositive/perf/query47.q.out
new file mode 100644
index 0000000..d5e1922
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/query47.q.out
@@ -0,0 +1,357 @@
+PREHOOK: query: explain
+with v1 as(
+ select i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy,
+ sum(ss_sales_price) sum_sales,
+ avg(sum(ss_sales_price)) over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name
+ order by d_year, d_moy) rn
+ from item, store_sales, date_dim, store
+ where ss_item_sk = i_item_sk and
+ ss_sold_date_sk = d_date_sk and
+ ss_store_sk = s_store_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy),
+ v2 as(
+ select v1.i_category
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1.s_store_name = v1_lag.s_store_name and
+ v1.s_store_name = v1_lead.s_store_name and
+ v1.s_company_name = v1_lag.s_company_name and
+ v1.s_company_name = v1_lead.s_company_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+with v1 as(
+ select i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy,
+ sum(ss_sales_price) sum_sales,
+ avg(sum(ss_sales_price)) over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ s_store_name, s_company_name
+ order by d_year, d_moy) rn
+ from item, store_sales, date_dim, store
+ where ss_item_sk = i_item_sk and
+ ss_sold_date_sk = d_date_sk and
+ ss_store_sk = s_store_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ s_store_name, s_company_name,
+ d_year, d_moy),
+ v2 as(
+ select v1.i_category
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1.s_store_name = v1_lag.s_store_name and
+ v1.s_store_name = v1_lead.s_store_name and
+ v1.s_company_name = v1_lag.s_company_name and
+ v1.s_company_name = v1_lead.s_company_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
+Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
+Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
+Reducer 15 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 17 <- Map 22 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
+Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 8
+ File Output Operator [FS_112]
+ Limit [LIM_110] (rows=100 width=88)
+ Number of rows:100
+ Select Operator [SEL_109] (rows=843315280 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_108]
+ Select Operator [SEL_107] (rows=843315280 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Merge Join Operator [MERGEJOIN_189] (rows=843315280 width=88)
+ Conds:RS_103._col0, _col1, _col2, _col3, (_col7 + 1)=RS_104._col0, _col1, _col2, _col3, _col8(Inner),RS_104._col0, _col1, _col2, _col3, _col8=RS_105._col0, _col1, _col2, _col3, (_col7 - 1)(Inner),Output:["_col6","_col8","_col12","_col13","_col14","_col15","_col23"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
+ PartitionCols:_col0, _col1, _col2, _col3, _col8
+ Select Operator [SEL_67] (rows=31943759 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Filter Operator [FIL_169] (rows=31943759 width=88)
+ predicate:CASE WHEN ((_col0 > 0)) THEN (((abs((_col7 - _col0)) / _col0) > 0.1)) ELSE (null) END
+ Select Operator [SEL_66] (rows=63887519 width=88)
+ Output:["rank_window_1","_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_170] (rows=63887519 width=88)
+ predicate:((_col0 > 0) and rank_window_1 is not null and (_col5 = 2000))
+ PTF Operator [PTF_65] (rows=383325119 width=88)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST, _col6 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3, _col4"}]
+ Select Operator [SEL_64] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_63]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Select Operator [SEL_62] (rows=383325119 width=88)
+ Output:["avg_window_0","_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ PTF Operator [PTF_61] (rows=383325119 width=88)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3, _col4"}]
+ Select Operator [SEL_60] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4
+ Select Operator [SEL_58] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Group By Operator [GBY_57] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
+ <-Reducer 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
+ Group By Operator [GBY_55] (rows=766650239 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
+ Merge Join Operator [MERGEJOIN_185] (rows=766650239 width=88)
+ Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ PartitionCols:_col0
+ Select Operator [SEL_44] (rows=1704 width=1910)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_174] (rows=1704 width=1910)
+ predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
+ TableScan [TS_9] (rows=1704 width=1910)
+ default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_184] (rows=696954748 width=88)
+ Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_41] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_173] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ TableScan [TS_6] (rows=462000 width=1436)
+ default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_48]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_183] (rows=633595212 width=88)
+ Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col0
+ Select Operator [SEL_35] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_171] (rows=575995635 width=88)
+ predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
+ TableScan [TS_0] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_46]
+ PartitionCols:_col0
+ Select Operator [SEL_38] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_172] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_105]
+ PartitionCols:_col0, _col1, _col2, _col3, (_col7 - 1)
+ Select Operator [SEL_99] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col6","_col7"]
+ Filter Operator [FIL_175] (rows=383325119 width=88)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_98] (rows=383325119 width=88)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
+ Select Operator [SEL_97] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_96]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Select Operator [SEL_95] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Group By Operator [GBY_94] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_93]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
+ Group By Operator [GBY_92] (rows=766650239 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
+ Merge Join Operator [MERGEJOIN_188] (rows=766650239 width=88)
+ Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_89]
+ PartitionCols:_col0
+ Select Operator [SEL_81] (rows=1704 width=1910)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_179] (rows=1704 width=1910)
+ predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
+ Please refer to the previous TableScan [TS_9]
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_88]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_187] (rows=696954748 width=88)
+ Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_86]
+ PartitionCols:_col0
+ Select Operator [SEL_78] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_178] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_85]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_186] (rows=633595212 width=88)
+ Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_82]
+ PartitionCols:_col0
+ Select Operator [SEL_72] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_176] (rows=575995635 width=88)
+ predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_83]
+ PartitionCols:_col0
+ Select Operator [SEL_75] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_177] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
+ PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1)
+ Select Operator [SEL_29] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col6","_col7"]
+ Filter Operator [FIL_164] (rows=383325119 width=88)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_28] (rows=383325119 width=88)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
+ Select Operator [SEL_27] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_26]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Select Operator [SEL_25] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Group By Operator [GBY_24] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_23]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
+ Group By Operator [GBY_22] (rows=766650239 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
+ Merge Join Operator [MERGEJOIN_182] (rows=766650239 width=88)
+ Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
+ PartitionCols:_col0
+ Select Operator [SEL_11] (rows=1704 width=1910)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_168] (rows=1704 width=1910)
+ predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
+ Please refer to the previous TableScan [TS_9]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_181] (rows=696954748 width=88)
+ Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_167] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_15]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_180] (rows=633595212 width=88)
+ Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_165] (rows=575995635 width=88)
+ predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_166] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+
http://git-wip-us.apache.org/repos/asf/hive/blob/d7ab32f2/ql/src/test/results/clientpositive/perf/query57.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query57.q.out b/ql/src/test/results/clientpositive/perf/query57.q.out
new file mode 100644
index 0000000..6c237bf
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/query57.q.out
@@ -0,0 +1,351 @@
+PREHOOK: query: explain
+with v1 as(
+ select i_category, i_brand,
+ cc_name,
+ d_year, d_moy,
+ sum(cs_sales_price) sum_sales,
+ avg(sum(cs_sales_price)) over
+ (partition by i_category, i_brand,
+ cc_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ cc_name
+ order by d_year, d_moy) rn
+ from item, catalog_sales, date_dim, call_center
+ where cs_item_sk = i_item_sk and
+ cs_sold_date_sk = d_date_sk and
+ cc_call_center_sk= cs_call_center_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ cc_name , d_year, d_moy),
+ v2 as(
+ select v1.i_category, v1.i_brand
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1. cc_name = v1_lag. cc_name and
+ v1. cc_name = v1_lead. cc_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+with v1 as(
+ select i_category, i_brand,
+ cc_name,
+ d_year, d_moy,
+ sum(cs_sales_price) sum_sales,
+ avg(sum(cs_sales_price)) over
+ (partition by i_category, i_brand,
+ cc_name, d_year)
+ avg_monthly_sales,
+ rank() over
+ (partition by i_category, i_brand,
+ cc_name
+ order by d_year, d_moy) rn
+ from item, catalog_sales, date_dim, call_center
+ where cs_item_sk = i_item_sk and
+ cs_sold_date_sk = d_date_sk and
+ cc_call_center_sk= cs_call_center_sk and
+ (
+ d_year = 2000 or
+ ( d_year = 2000-1 and d_moy =12) or
+ ( d_year = 2000+1 and d_moy =1)
+ )
+ group by i_category, i_brand,
+ cc_name , d_year, d_moy),
+ v2 as(
+ select v1.i_category, v1.i_brand
+ ,v1.d_year, v1.d_moy
+ ,v1.avg_monthly_sales
+ ,v1.sum_sales, v1_lag.sum_sales psum, v1_lead.sum_sales nsum
+ from v1, v1 v1_lag, v1 v1_lead
+ where v1.i_category = v1_lag.i_category and
+ v1.i_category = v1_lead.i_category and
+ v1.i_brand = v1_lag.i_brand and
+ v1.i_brand = v1_lead.i_brand and
+ v1. cc_name = v1_lag. cc_name and
+ v1. cc_name = v1_lead. cc_name and
+ v1.rn = v1_lag.rn + 1 and
+ v1.rn = v1_lead.rn - 1)
+ select *
+ from v2
+ where d_year = 2000 and
+ avg_monthly_sales > 0 and
+ case when avg_monthly_sales > 0 then abs(sum_sales - avg_monthly_sales) / avg_monthly_sales else null end > 0.1
+ order by sum_sales - avg_monthly_sales, 3
+ limit 100
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
+Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
+Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
+Reducer 15 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 16 <- Map 21 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 17 <- Map 22 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
+Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 14 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 8
+ File Output Operator [FS_112]
+ Limit [LIM_110] (rows=100 width=135)
+ Number of rows:100
+ Select Operator [SEL_109] (rows=421645952 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ <-Reducer 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_108]
+ Select Operator [SEL_107] (rows=421645952 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
+ Merge Join Operator [MERGEJOIN_189] (rows=421645952 width=135)
+ Conds:RS_103._col0, _col2, _col1, (_col6 + 1)=RS_104._col0, _col2, _col1, _col7(Inner),RS_104._col0, _col2, _col1, _col7=RS_105._col0, _col2, _col1, (_col6 - 1)(Inner),Output:["_col5","_col7","_col8","_col10","_col11","_col12","_col13","_col20"]
+ <-Reducer 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
+ PartitionCols:_col0, _col2, _col1, _col7
+ Select Operator [SEL_67] (rows=15971437 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ Filter Operator [FIL_169] (rows=15971437 width=135)
+ predicate:CASE WHEN ((_col0 > 0)) THEN (((abs((_col6 - _col0)) / _col0) > 0.1)) ELSE (null) END
+ Select Operator [SEL_66] (rows=31942874 width=135)
+ Output:["rank_window_1","_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Filter Operator [FIL_170] (rows=31942874 width=135)
+ predicate:((_col0 > 0) and rank_window_1 is not null and (_col4 = 2000))
+ PTF Operator [PTF_65] (rows=191657247 width=135)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3"}]
+ Select Operator [SEL_64] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_63]
+ PartitionCols:_col0, _col1, _col2
+ Select Operator [SEL_62] (rows=191657247 width=135)
+ Output:["avg_window_0","_col0","_col1","_col2","_col3","_col4","_col5"]
+ PTF Operator [PTF_61] (rows=191657247 width=135)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
+ Select Operator [SEL_60] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Select Operator [SEL_58] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Group By Operator [GBY_57] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
+ <-Reducer 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4
+ Group By Operator [GBY_55] (rows=383314495 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
+ Merge Join Operator [MERGEJOIN_185] (rows=383314495 width=135)
+ Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_52]
+ PartitionCols:_col0
+ Select Operator [SEL_44] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_174] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ TableScan [TS_9] (rows=462000 width=1436)
+ default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_184] (rows=348467716 width=135)
+ Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_49]
+ PartitionCols:_col0
+ Select Operator [SEL_41] (rows=60 width=2045)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_173] (rows=60 width=2045)
+ predicate:(cc_call_center_sk is not null and cc_name is not null)
+ TableScan [TS_6] (rows=60 width=2045)
+ default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"]
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_48]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_183] (rows=316788826 width=135)
+ Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ PartitionCols:_col0
+ Select Operator [SEL_35] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_171] (rows=287989836 width=135)
+ predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
+ TableScan [TS_0] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_46]
+ PartitionCols:_col0
+ Select Operator [SEL_38] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_172] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_105]
+ PartitionCols:_col0, _col2, _col1, (_col6 - 1)
+ Select Operator [SEL_99] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col5","_col6"]
+ Filter Operator [FIL_175] (rows=191657247 width=135)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_98] (rows=191657247 width=135)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}]
+ Select Operator [SEL_97] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_96]
+ PartitionCols:_col0, _col1, _col2
+ Select Operator [SEL_95] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Group By Operator [GBY_94] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_93]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4
+ Group By Operator [GBY_92] (rows=383314495 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
+ Merge Join Operator [MERGEJOIN_188] (rows=383314495 width=135)
+ Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_89]
+ PartitionCols:_col0
+ Select Operator [SEL_81] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_179] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ Please refer to the previous TableScan [TS_9]
+ <-Reducer 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_88]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_187] (rows=348467716 width=135)
+ Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_86]
+ PartitionCols:_col0
+ Select Operator [SEL_78] (rows=60 width=2045)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_178] (rows=60 width=2045)
+ predicate:(cc_call_center_sk is not null and cc_name is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_85]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_186] (rows=316788826 width=135)
+ Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_82]
+ PartitionCols:_col0
+ Select Operator [SEL_72] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_176] (rows=287989836 width=135)
+ predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_83]
+ PartitionCols:_col0
+ Select Operator [SEL_75] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_177] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
+ PartitionCols:_col0, _col2, _col1, (_col6 + 1)
+ Select Operator [SEL_29] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col5","_col6"]
+ Filter Operator [FIL_164] (rows=191657247 width=135)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_28] (rows=191657247 width=135)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}]
+ Select Operator [SEL_27] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_26]
+ PartitionCols:_col0, _col1, _col2
+ Select Operator [SEL_25] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Group By Operator [GBY_24] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_23]
+ PartitionCols:_col0, _col1, _col2, _col3, _col4
+ Group By Operator [GBY_22] (rows=383314495 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
+ Merge Join Operator [MERGEJOIN_182] (rows=383314495 width=135)
+ Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
+ PartitionCols:_col0
+ Select Operator [SEL_11] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_168] (rows=462000 width=1436)
+ predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
+ Please refer to the previous TableScan [TS_9]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_181] (rows=348467716 width=135)
+ Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=60 width=2045)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_167] (rows=60 width=2045)
+ predicate:(cc_call_center_sk is not null and cc_name is not null)
+ Please refer to the previous TableScan [TS_6]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_15]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_180] (rows=316788826 width=135)
+ Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_165] (rows=287989836 width=135)
+ predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
+ Please refer to the previous TableScan [TS_0]
+ <-Map 20 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_166] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ Please refer to the previous TableScan [TS_3]
+
[17/36] hive git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/hive
Posted by we...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/788d486e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/788d486e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/788d486e
Branch: refs/heads/hive-14535
Commit: 788d486e8fbf58919d04f15e965050f1e885093f
Parents: 363ffe0 690a9f8
Author: Xuefu Zhang <xu...@uber.com>
Authored: Fri Jun 2 11:27:38 2017 -0700
Committer: Xuefu Zhang <xu...@uber.com>
Committed: Fri Jun 2 11:27:38 2017 -0700
----------------------------------------------------------------------
bin/ext/version.sh | 2 +-
bin/hive | 20 ++------------------
.../hive/http/Log4j2ConfiguratorServlet.java | 18 +++++++++++-------
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 ++--
.../hadoop/hive/metastore/MetaStoreUtils.java | 14 +++++++++++---
.../hive/metastore/TestMetaStoreUtils.java | 16 +++++++++-------
.../fast/VectorMapJoinFastHashTable.java | 3 ++-
7 files changed, 38 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
[06/36] hive git commit: HIVE-16771: Schematool should use
MetastoreSchemaInfo to get the metastore schema version from database (Vihang
Karajgaonkar, reviewed by Naveen Gangam, Sergio Pena)
Posted by we...@apache.org.
HIVE-16771: Schematool should use MetastoreSchemaInfo to get the metastore schema version from database (Vihang Karajgaonkar, reviewed by Naveen Gangam, Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb815dcf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb815dcf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb815dcf
Branch: refs/heads/hive-14535
Commit: cb815dcf96030cb48b4d8ab89895ff3141f7dbca
Parents: 396cba4
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Wed May 31 13:50:14 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed May 31 13:50:14 2017 -0500
----------------------------------------------------------------------
.../apache/hive/beeline/HiveSchemaHelper.java | 581 -----------------
.../org/apache/hive/beeline/HiveSchemaTool.java | 79 +--
.../apache/hive/beeline/TestHiveSchemaTool.java | 1 +
.../org/apache/hive/beeline/TestSchemaTool.java | 6 +-
.../hive/metastore/IMetaStoreSchemaInfo.java | 15 +
.../hive/metastore/MetaStoreSchemaInfo.java | 33 +
.../hive/metastore/tools/HiveSchemaHelper.java | 640 +++++++++++++++++++
7 files changed, 719 insertions(+), 636 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
deleted file mode 100644
index a4ecc08..0000000
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaHelper.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.beeline;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.IllegalFormatException;
-import java.util.List;
-
-public class HiveSchemaHelper {
- public static final String DB_DERBY = "derby";
- public static final String DB_HIVE = "hive";
- public static final String DB_MSSQL = "mssql";
- public static final String DB_MYSQL = "mysql";
- public static final String DB_POSTGRACE = "postgres";
- public static final String DB_ORACLE = "oracle";
-
- /***
- * Get JDBC connection to metastore db
- *
- * @param userName metastore connection username
- * @param password metastore connection password
- * @param printInfo print connection parameters
- * @param hiveConf hive config object
- * @return metastore connection object
- * @throws org.apache.hadoop.hive.metastore.api.MetaException
- */
- public static Connection getConnectionToMetastore(String userName,
- String password, String url, String driver, boolean printInfo,
- HiveConf hiveConf)
- throws HiveMetaException {
- try {
- url = url == null ? getValidConfVar(
- HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url;
- driver = driver == null ? getValidConfVar(
- HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver;
- if (printInfo) {
- System.out.println("Metastore connection URL:\t " + url);
- System.out.println("Metastore Connection Driver :\t " + driver);
- System.out.println("Metastore connection User:\t " + userName);
- }
- if ((userName == null) || userName.isEmpty()) {
- throw new HiveMetaException("UserName empty ");
- }
-
- // load required JDBC driver
- Class.forName(driver);
-
- // Connect using the JDBC URL and user/pass from conf
- return DriverManager.getConnection(url, userName, password);
- } catch (IOException e) {
- throw new HiveMetaException("Failed to get schema version.", e);
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get schema version.", e);
- } catch (ClassNotFoundException e) {
- throw new HiveMetaException("Failed to load driver", e);
- }
- }
-
- public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf)
- throws IOException {
- String confVarStr = hiveConf.get(confVar.varname);
- if (confVarStr == null || confVarStr.isEmpty()) {
- throw new IOException("Empty " + confVar.varname);
- }
- return confVarStr.trim();
- }
-
- public interface NestedScriptParser {
-
- public enum CommandType {
- PARTIAL_STATEMENT,
- TERMINATED_STATEMENT,
- COMMENT
- }
-
- static final String DEFAULT_DELIMITER = ";";
- static final String DEFAULT_QUOTE = "\"";
-
- /**
- * Find the type of given command
- *
- * @param dbCommand
- * @return
- */
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Parse the DB specific nesting format and extract the inner script name if any
- *
- * @param dbCommand command from parent script
- * @return
- * @throws IllegalFormatException
- */
- public String getScriptName(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Find if the given command is a nested script execution
- *
- * @param dbCommand
- * @return
- */
- public boolean isNestedScript(String dbCommand);
-
- /**
- * Find if the given command should not be passed to DB
- *
- * @param dbCommand
- * @return
- */
- public boolean isNonExecCommand(String dbCommand);
-
- /**
- * Get the SQL statement delimiter
- *
- * @return
- */
- public String getDelimiter();
-
- /**
- * Get the SQL indentifier quotation character
- *
- * @return
- */
- public String getQuoteCharacter();
-
- /**
- * Clear any client specific tags
- *
- * @return
- */
- public String cleanseCommand(String dbCommand);
-
- /**
- * Does the DB required table/column names quoted
- *
- * @return
- */
- public boolean needsQuotedIdentifier();
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @return string of sql commands
- */
- public String buildCommand(String scriptDir, String scriptFile)
- throws IllegalFormatException, IOException;
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @param fixQuotes whether to replace quote characters
- * @return string of sql commands
- */
- public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
- throws IllegalFormatException, IOException;
- }
-
- /***
- * Base implementation of NestedScriptParser
- * abstractCommandParser.
- *
- */
- private static abstract class AbstractCommandParser implements NestedScriptParser {
- private List<String> dbOpts;
- private String msUsername;
- private String msPassword;
- private HiveConf hiveConf;
-
- public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- setDbOpts(dbOpts);
- this.msUsername = msUsername;
- this.msPassword = msPassword;
- this.hiveConf = hiveConf;
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- if (dbCommand == null || dbCommand.isEmpty()) {
- throw new IllegalArgumentException("invalid command line " + dbCommand);
- }
- dbCommand = dbCommand.trim();
- if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
- }
-
- @Override
- public String getDelimiter() {
- return DEFAULT_DELIMITER;
- }
-
- @Override
- public String getQuoteCharacter() {
- return DEFAULT_QUOTE;
- }
-
-
- @Override
- public String cleanseCommand(String dbCommand) {
- // strip off the delimiter
- if (dbCommand.endsWith(getDelimiter())) {
- dbCommand = dbCommand.substring(0,
- dbCommand.length() - getDelimiter().length());
- }
- return dbCommand;
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return false;
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
- return buildCommand(scriptDir, scriptFile, false);
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
- BufferedReader bfReader =
- new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
- String currLine;
- StringBuilder sb = new StringBuilder();
- String currentCommand = null;
- while ((currLine = bfReader.readLine()) != null) {
- currLine = currLine.trim();
-
- if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
- currLine = currLine.replace("\\\"", getQuoteCharacter());
- }
-
- if (currLine.isEmpty()) {
- continue; // skip empty lines
- }
-
- if (currentCommand == null) {
- currentCommand = currLine;
- } else {
- currentCommand = currentCommand + " " + currLine;
- }
- if (isPartialCommand(currLine)) {
- // if its a partial line, continue collecting the pieces
- continue;
- }
-
- // if this is a valid executable command then add it to the buffer
- if (!isNonExecCommand(currentCommand)) {
- currentCommand = cleanseCommand(currentCommand);
- if (isNestedScript(currentCommand)) {
- // if this is a nested sql script then flatten it
- String currScript = getScriptName(currentCommand);
- sb.append(buildCommand(scriptDir, currScript));
- } else {
- // Now we have a complete statement, process it
- // write the line to buffer
- sb.append(currentCommand);
- sb.append(System.getProperty("line.separator"));
- }
- }
- currentCommand = null;
- }
- bfReader.close();
- return sb.toString();
- }
-
- private void setDbOpts(String dbOpts) {
- if (dbOpts != null) {
- this.dbOpts = Lists.newArrayList(dbOpts.split(","));
- } else {
- this.dbOpts = Lists.newArrayList();
- }
- }
-
- protected List<String> getDbOpts() {
- return dbOpts;
- }
-
- protected String getMsUsername() {
- return msUsername;
- }
-
- protected String getMsPassword() {
- return msPassword;
- }
-
- protected HiveConf getHiveConf() {
- return hiveConf;
- }
- }
-
- // Derby commandline parser
- public static class DerbyCommandParser extends AbstractCommandParser {
- private static final String DERBY_NESTING_TOKEN = "RUN";
-
- public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "").replaceAll("'", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- // Derby script format is RUN '<file>'
- return dbCommand.startsWith(DERBY_NESTING_TOKEN);
- }
- }
-
- // Derby commandline parser
- public static class HiveCommandParser extends AbstractCommandParser {
- private static String HIVE_NESTING_TOKEN = "SOURCE";
- private final NestedScriptParser nestedDbCommandParser;
-
- public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf, String metaDbType) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- nestedDbCommandParser = getDbCommandParser(metaDbType);
- }
-
- @Override
- public String getQuoteCharacter() {
- return nestedDbCommandParser.getQuoteCharacter();
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(HIVE_NESTING_TOKEN);
- }
- }
-
- // MySQL parser
- public static class MySqlCommandParser extends AbstractCommandParser {
- private static final String MYSQL_NESTING_TOKEN = "SOURCE";
- private static final String DELIMITER_TOKEN = "DELIMITER";
- private String delimiter = DEFAULT_DELIMITER;
-
- public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- boolean isPartial = super.isPartialCommand(dbCommand);
- // if this is a delimiter directive, reset our delimiter
- if (dbCommand.startsWith(DELIMITER_TOKEN)) {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- delimiter = tokens[1];
- }
- return isPartial;
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
- }
-
- @Override
- public String getDelimiter() {
- return delimiter;
- }
-
- @Override
- public String getQuoteCharacter() {
- return "`";
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return super.isNonExecCommand(dbCommand) ||
- (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
- dbCommand.startsWith(DELIMITER_TOKEN);
- }
-
- @Override
- public String cleanseCommand(String dbCommand) {
- return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
- }
-
- }
-
- // Postgres specific parser
- public static class PostgresCommandParser extends AbstractCommandParser {
- private static final String POSTGRES_NESTING_TOKEN = "\\i";
- @VisibleForTesting
- public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
- @VisibleForTesting
- public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
-
- public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return true;
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- // Skip "standard_conforming_strings" command which is read-only in older
- // Postgres versions like 8.1
- // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
- if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
- if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
- return true;
- }
- }
- return super.isNonExecCommand(dbCommand);
- }
- }
-
- //Oracle specific parser
- public static class OracleCommandParser extends AbstractCommandParser {
- private static final String ORACLE_NESTING_TOKEN = "@";
-
- public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a nested script format " + dbCommand);
- }
- // remove ending ';' and starting '@'
- return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
- }
- }
-
- //MSSQL specific parser
- public static class MSSQLCommandParser extends AbstractCommandParser {
- private static final String MSSQL_NESTING_TOKEN = ":r";
-
- public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1];
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
- }
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName) {
- return getDbCommandParser(dbName, null);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
- return getDbCommandParser(dbName, null, null, null, null, metaDbName);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName,
- String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf, String metaDbType) {
- if (dbName.equalsIgnoreCase(DB_DERBY)) {
- return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
- return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType);
- } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
- return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
- return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
- return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
- return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else {
- throw new IllegalArgumentException("Unknown dbType " + dbName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index f312e46..4ff4beb 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -38,8 +38,10 @@ import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,7 @@ public class HiveSchemaTool {
private final String dbType;
private final String metaDbType;
private final IMetaStoreSchemaInfo metaStoreSchemaInfo;
+ private boolean needsQuotedIdentifier;
static final private Logger LOG = LoggerFactory.getLogger(HiveSchemaTool.class.getName());
@@ -95,6 +98,7 @@ public class HiveSchemaTool {
this.hiveConf = hiveConf;
this.dbType = dbType;
this.metaDbType = metaDbType;
+ this.needsQuotedIdentifier = getDbCommandParser(dbType).needsQuotedIdentifier();
this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(hiveConf, hiveHome, dbType);
}
@@ -167,42 +171,11 @@ public class HiveSchemaTool {
* @throws MetaException
*/
public void showInfo() throws HiveMetaException {
- Connection metastoreConn = getConnectionToMetastore(true);
String hiveVersion = metaStoreSchemaInfo.getHiveSchemaVersion();
- String dbVersion = getMetaStoreSchemaVersion(metastoreConn);
+ String dbVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(true));
System.out.println("Hive distribution version:\t " + hiveVersion);
System.out.println("Metastore schema version:\t " + dbVersion);
assertCompatibleVersion(hiveVersion, dbVersion);
-
- }
-
- private String getMetaStoreSchemaVersion(Connection metastoreConn)
- throws HiveMetaException {
- return getMetaStoreSchemaVersion(metastoreConn, false);
- }
-
- // read schema version from metastore
- private String getMetaStoreSchemaVersion(Connection metastoreConn,
- boolean checkDuplicatedVersion) throws HiveMetaException {
- String versionQuery;
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
- versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
- } else {
- versionQuery = "select t.SCHEMA_VERSION from VERSION t";
- }
- try(Statement stmt = metastoreConn.createStatement();
- ResultSet res = stmt.executeQuery(versionQuery)) {
- if (!res.next()) {
- throw new HiveMetaException("Could not find version info in metastore VERSION table.");
- }
- String currentSchemaVersion = res.getString(1);
- if (checkDuplicatedVersion && res.next()) {
- throw new HiveMetaException("Multiple versions were found in metastore.");
- }
- return currentSchemaVersion;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
- }
}
boolean validateLocations(Connection conn, URI[] defaultServers) throws HiveMetaException {
@@ -226,7 +199,7 @@ public class HiveSchemaTool {
String dbLoc;
boolean isValid = true;
int numOfInvalid = 0;
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
dbLoc = "select dbt.\"DB_ID\", dbt.\"NAME\", dbt.\"DB_LOCATION_URI\" from \"DBS\" dbt";
} else {
dbLoc = "select dbt.DB_ID, dbt.NAME, dbt.DB_LOCATION_URI from DBS dbt";
@@ -255,13 +228,13 @@ public class HiveSchemaTool {
String tabLoc, tabIDRange;
boolean isValid = true;
int numOfInvalid = 0;
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
tabIDRange = "select max(\"TBL_ID\"), min(\"TBL_ID\") from \"TBLS\" ";
} else {
tabIDRange = "select max(TBL_ID), min(TBL_ID) from TBLS";
}
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
tabLoc = "select tbl.\"TBL_ID\", tbl.\"TBL_NAME\", sd.\"LOCATION\", dbt.\"DB_ID\", dbt.\"NAME\" from \"TBLS\" tbl inner join " +
"\"SDS\" sd on tbl.\"SD_ID\" = sd.\"SD_ID\" and tbl.\"TBL_TYPE\" != '" + TableType.VIRTUAL_VIEW +
"' and tbl.\"TBL_ID\" >= ? and tbl.\"TBL_ID\"<= ? " + "inner join \"DBS\" dbt on tbl.\"DB_ID\" = dbt.\"DB_ID\" ";
@@ -315,13 +288,13 @@ public class HiveSchemaTool {
String partLoc, partIDRange;
boolean isValid = true;
int numOfInvalid = 0;
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
partIDRange = "select max(\"PART_ID\"), min(\"PART_ID\") from \"PARTITIONS\" ";
} else {
partIDRange = "select max(PART_ID), min(PART_ID) from PARTITIONS";
}
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
partLoc = "select pt.\"PART_ID\", pt.\"PART_NAME\", sd.\"LOCATION\", tbl.\"TBL_ID\", tbl.\"TBL_NAME\",dbt.\"DB_ID\", dbt.\"NAME\" from \"PARTITIONS\" pt "
+ "inner join \"SDS\" sd on pt.\"SD_ID\" = sd.\"SD_ID\" and pt.\"PART_ID\" >= ? and pt.\"PART_ID\"<= ? "
+ " inner join \"TBLS\" tbl on pt.\"TBL_ID\" = tbl.\"TBL_ID\" inner join "
@@ -376,13 +349,13 @@ public class HiveSchemaTool {
String skewedColLoc, skewedColIDRange;
boolean isValid = true;
int numOfInvalid = 0;
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
skewedColIDRange = "select max(\"STRING_LIST_ID_KID\"), min(\"STRING_LIST_ID_KID\") from \"SKEWED_COL_VALUE_LOC_MAP\" ";
} else {
skewedColIDRange = "select max(STRING_LIST_ID_KID), min(STRING_LIST_ID_KID) from SKEWED_COL_VALUE_LOC_MAP";
}
- if (getDbCommandParser(dbType).needsQuotedIdentifier()) {
+ if (needsQuotedIdentifier) {
skewedColLoc = "select t.\"TBL_NAME\", t.\"TBL_ID\", sk.\"STRING_LIST_ID_KID\", sk.\"LOCATION\", db.\"NAME\", db.\"DB_ID\" from \"TBLS\" t, \"SDS\" s, \"DBS\" db, \"SKEWED_COL_VALUE_LOC_MAP\" sk "
+ "where sk.\"SD_ID\" = s.\"SD_ID\" and s.\"SD_ID\" = t.\"SD_ID\" and t.\"DB_ID\" = db.\"DB_ID\" and sk.\"STRING_LIST_ID_KID\" >= ? and sk.\"STRING_LIST_ID_KID\" <= ? ";
} else {
@@ -497,11 +470,9 @@ public class HiveSchemaTool {
if (dryRun) {
return;
}
- String newSchemaVersion = getMetaStoreSchemaVersion(
- getConnectionToMetastore(false));
+ String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
// verify that the new version is added to schema
assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
-
}
private void assertCompatibleVersion(String hiveSchemaVersion, String dbSchemaVersion)
@@ -517,8 +488,8 @@ public class HiveSchemaTool {
* @throws MetaException
*/
public void doUpgrade() throws HiveMetaException {
- String fromVersion = getMetaStoreSchemaVersion(
- getConnectionToMetastore(false));
+ String fromVersion =
+ metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
if (fromVersion == null || fromVersion.isEmpty()) {
throw new HiveMetaException("Schema version not stored in the metastore. " +
"Metastore schema is too old or corrupt. Try specifying the version manually");
@@ -526,6 +497,10 @@ public class HiveSchemaTool {
doUpgrade(fromVersion);
}
+ private MetaStoreConnectionInfo getConnectionInfo(boolean printInfo) {
+ return new MetaStoreConnectionInfo(userName, passWord, url, driver, printInfo, hiveConf,
+ dbType);
+ }
/**
* Perform metastore schema upgrade
*
@@ -682,10 +657,10 @@ public class HiveSchemaTool {
for (String seqName : seqNameToTable.keySet()) {
String tableName = seqNameToTable.get(seqName).getLeft();
String tableKey = seqNameToTable.get(seqName).getRight();
- String seqQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+ String seqQuery = needsQuotedIdentifier ?
("select t.\"NEXT_VAL\" from \"SEQUENCE_TABLE\" t WHERE t.\"SEQUENCE_NAME\"='org.apache.hadoop.hive.metastore.model." + seqName + "'")
: ("select t.NEXT_VAL from SEQUENCE_TABLE t WHERE t.SEQUENCE_NAME='org.apache.hadoop.hive.metastore.model." + seqName + "'");
- String maxIdQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+ String maxIdQuery = needsQuotedIdentifier ?
("select max(\"" + tableKey + "\") from \"" + tableName + "\"")
: ("select max(" + tableKey + ") from " + tableName);
@@ -715,7 +690,7 @@ public class HiveSchemaTool {
boolean validateSchemaVersions(Connection conn) throws HiveMetaException {
System.out.println("Validating schema version");
try {
- String newSchemaVersion = getMetaStoreSchemaVersion(conn, true);
+ String newSchemaVersion = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
assertCompatibleVersion(metaStoreSchemaInfo.getHiveSchemaVersion(), newSchemaVersion);
} catch (HiveMetaException hme) {
if (hme.getMessage().contains("Metastore schema version is not compatible")
@@ -743,7 +718,7 @@ public class HiveSchemaTool {
System.out.println("Validating metastore schema tables");
try {
- version = getMetaStoreSchemaVersion(hmsConn);
+ version = metaStoreSchemaInfo.getMetaStoreSchemaVersion(getConnectionInfo(false));
} catch (HiveMetaException he) {
System.err.println("Failed to determine schema version from Hive Metastore DB. " + he.getMessage());
System.out.println("Failed in schema version validation.");
@@ -781,8 +756,8 @@ public class HiveSchemaTool {
// parse the schema file to determine the tables that are expected to exist
// we are using oracle schema because it is simpler to parse, no quotes or backticks etc
String baseDir = new File(metaStoreSchemaInfo.getMetaStoreScriptDir()).getParent();
- String schemaFile = baseDir + "/" + dbType + "/hive-schema-" + version + "." + dbType + ".sql";
-
+ String schemaFile = new File(metaStoreSchemaInfo.getMetaStoreScriptDir(),
+ metaStoreSchemaInfo.generateInitFileName(version)).getPath();
try {
LOG.debug("Parsing schema script " + schemaFile);
subScripts.addAll(findCreateTable(schemaFile, schemaTables));
@@ -896,7 +871,7 @@ public class HiveSchemaTool {
boolean isValid = true;
try {
Statement stmt = conn.createStatement();
- String tblQuery = getDbCommandParser(dbType).needsQuotedIdentifier() ?
+ String tblQuery = needsQuotedIdentifier ?
("select t.* from \"TBLS\" t WHERE t.\"SD_ID\" IS NULL and (t.\"TBL_TYPE\"='" + TableType.EXTERNAL_TABLE + "' or t.\"TBL_TYPE\"='" + TableType.MANAGED_TABLE + "')")
: ("select t.* from TBLS t WHERE t.SD_ID IS NULL and (t.TBL_TYPE='" + TableType.EXTERNAL_TABLE + "' or t.TBL_TYPE='" + TableType.MANAGED_TABLE + "')");
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
index 716bce7..d9b8f42 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
@@ -18,6 +18,7 @@
package org.apache.hive.beeline;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
index d1658f7..45172c4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestSchemaTool.java
@@ -36,11 +36,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.PostgresCommandParser;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.beeline.HiveSchemaHelper.NestedScriptParser;
-import org.apache.hive.beeline.HiveSchemaHelper.PostgresCommandParser;
public class TestSchemaTool extends TestCase {
private HiveSchemaTool schemaTool;
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
index d662743..296d04b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
@@ -18,7 +18,10 @@
package org.apache.hadoop.hive.metastore;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+
+import java.sql.Connection;
import java.util.List;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
/**
* Defines the method which must be implemented to be used using schema tool to support metastore
@@ -78,6 +81,18 @@ public interface IMetaStoreSchemaInfo {
String getHiveSchemaVersion();
/**
+ * Get the schema version from the backend database. This version is used by SchemaTool to to
+ * compare the version returned by getHiveSchemaVersion and determine the upgrade order and
+ * scripts needed to upgrade the metastore schema
+ *
+ * @param metastoreDbConnectionInfo Connection information needed to connect to the backend
+ * database
+ * @return
+ * @throws HiveMetaException when unable to fetch the schema version
+ */
+ String getMetaStoreSchemaVersion(
+ HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException;
+ /**
* A dbVersion is compatible with hive version if it is greater or equal to the hive version. This
* is result of the db schema upgrade design principles followed in hive project. The state where
* db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
index 6bddb8e..d6d83e0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -22,10 +22,16 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
import org.apache.hive.common.util.HiveVersionInfo;
import com.google.common.collect.ImmutableMap;
@@ -198,4 +204,31 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
return true;
}
+ @Override
+ public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo)
+ throws HiveMetaException {
+ String versionQuery;
+ boolean needsQuotedIdentifier =
+ HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType()).needsQuotedIdentifier();
+ if (needsQuotedIdentifier) {
+ versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
+ } else {
+ versionQuery = "select t.SCHEMA_VERSION from VERSION t";
+ }
+ try (Connection metastoreDbConnection =
+ HiveSchemaHelper.getConnectionToMetastore(connectionInfo)) {
+ Statement stmt = metastoreDbConnection.createStatement();
+ ResultSet res = stmt.executeQuery(versionQuery);
+ if (!res.next()) {
+ throw new HiveMetaException("Could not find version info in metastore VERSION table.");
+ }
+ String currentSchemaVersion = res.getString(1);
+ if (res.next()) {
+ throw new HiveMetaException("Multiple versions were found in metastore.");
+ }
+ return currentSchemaVersion;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cb815dcf/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
new file mode 100644
index 0000000..0127bdd
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -0,0 +1,640 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.IllegalFormatException;
+import java.util.List;
+
+public class HiveSchemaHelper {
+ public static final String DB_DERBY = "derby";
+ public static final String DB_HIVE = "hive";
+ public static final String DB_MSSQL = "mssql";
+ public static final String DB_MYSQL = "mysql";
+ public static final String DB_POSTGRACE = "postgres";
+ public static final String DB_ORACLE = "oracle";
+
+ /***
+ * Get JDBC connection to metastore db
+ *
+ * @param userName metastore connection username
+ * @param password metastore connection password
+ * @param printInfo print connection parameters
+ * @param hiveConf hive config object
+ * @return metastore connection object
+ * @throws org.apache.hadoop.hive.metastore.api.MetaException
+ */
+ public static Connection getConnectionToMetastore(String userName,
+ String password, String url, String driver, boolean printInfo,
+ HiveConf hiveConf)
+ throws HiveMetaException {
+ try {
+ url = url == null ? getValidConfVar(
+ HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url;
+ driver = driver == null ? getValidConfVar(
+ HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver;
+ if (printInfo) {
+ System.out.println("Metastore connection URL:\t " + url);
+ System.out.println("Metastore Connection Driver :\t " + driver);
+ System.out.println("Metastore connection User:\t " + userName);
+ }
+ if ((userName == null) || userName.isEmpty()) {
+ throw new HiveMetaException("UserName empty ");
+ }
+
+ // load required JDBC driver
+ Class.forName(driver);
+
+ // Connect using the JDBC URL and user/pass from conf
+ return DriverManager.getConnection(url, userName, password);
+ } catch (IOException e) {
+ throw new HiveMetaException("Failed to get schema version.", e);
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get schema version.", e);
+ } catch (ClassNotFoundException e) {
+ throw new HiveMetaException("Failed to load driver", e);
+ }
+ }
+
+ public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException {
+ return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(),
+ info.getDriver(), info.getPrintInfo(), info.getHiveConf());
+ }
+
+ public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf)
+ throws IOException {
+ String confVarStr = hiveConf.get(confVar.varname);
+ if (confVarStr == null || confVarStr.isEmpty()) {
+ throw new IOException("Empty " + confVar.varname);
+ }
+ return confVarStr.trim();
+ }
+
+ public interface NestedScriptParser {
+
+ public enum CommandType {
+ PARTIAL_STATEMENT,
+ TERMINATED_STATEMENT,
+ COMMENT
+ }
+
+ static final String DEFAULT_DELIMITER = ";";
+ static final String DEFAULT_QUOTE = "\"";
+
+ /**
+ * Find the type of given command
+ *
+ * @param dbCommand
+ * @return
+ */
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
+
+ /**
+ * Parse the DB specific nesting format and extract the inner script name if any
+ *
+ * @param dbCommand command from parent script
+ * @return
+ * @throws IllegalFormatException
+ */
+ public String getScriptName(String dbCommand) throws IllegalArgumentException;
+
+ /**
+ * Find if the given command is a nested script execution
+ *
+ * @param dbCommand
+ * @return
+ */
+ public boolean isNestedScript(String dbCommand);
+
+ /**
+ * Find if the given command should not be passed to DB
+ *
+ * @param dbCommand
+ * @return
+ */
+ public boolean isNonExecCommand(String dbCommand);
+
+ /**
+ * Get the SQL statement delimiter
+ *
+ * @return
+ */
+ public String getDelimiter();
+
+ /**
+ * Get the SQL indentifier quotation character
+ *
+ * @return
+ */
+ public String getQuoteCharacter();
+
+ /**
+ * Clear any client specific tags
+ *
+ * @return
+ */
+ public String cleanseCommand(String dbCommand);
+
+ /**
+ * Does the DB required table/column names quoted
+ *
+ * @return
+ */
+ public boolean needsQuotedIdentifier();
+
+ /**
+ * Flatten the nested upgrade script into a buffer
+ *
+ * @param scriptDir upgrade script directory
+ * @param scriptFile upgrade script file
+ * @return string of sql commands
+ */
+ public String buildCommand(String scriptDir, String scriptFile)
+ throws IllegalFormatException, IOException;
+
+ /**
+ * Flatten the nested upgrade script into a buffer
+ *
+ * @param scriptDir upgrade script directory
+ * @param scriptFile upgrade script file
+ * @param fixQuotes whether to replace quote characters
+ * @return string of sql commands
+ */
+ public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
+ throws IllegalFormatException, IOException;
+ }
+
+ /***
+ * Base implementation of NestedScriptParser
+ * abstractCommandParser.
+ *
+ */
+ private static abstract class AbstractCommandParser implements NestedScriptParser {
+ private List<String> dbOpts;
+ private String msUsername;
+ private String msPassword;
+ private HiveConf hiveConf;
+
+ public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ setDbOpts(dbOpts);
+ this.msUsername = msUsername;
+ this.msPassword = msPassword;
+ this.hiveConf = hiveConf;
+ }
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ if (dbCommand == null || dbCommand.isEmpty()) {
+ throw new IllegalArgumentException("invalid command line " + dbCommand);
+ }
+ dbCommand = dbCommand.trim();
+ if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
+ }
+
+ @Override
+ public String getDelimiter() {
+ return DEFAULT_DELIMITER;
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return DEFAULT_QUOTE;
+ }
+
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ // strip off the delimiter
+ if (dbCommand.endsWith(getDelimiter())) {
+ dbCommand = dbCommand.substring(0,
+ dbCommand.length() - getDelimiter().length());
+ }
+ return dbCommand;
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return false;
+ }
+
+ @Override
+ public String buildCommand(
+ String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
+ return buildCommand(scriptDir, scriptFile, false);
+ }
+
+ @Override
+ public String buildCommand(
+ String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
+ BufferedReader bfReader =
+ new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
+ String currLine;
+ StringBuilder sb = new StringBuilder();
+ String currentCommand = null;
+ while ((currLine = bfReader.readLine()) != null) {
+ currLine = currLine.trim();
+
+ if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
+ currLine = currLine.replace("\\\"", getQuoteCharacter());
+ }
+
+ if (currLine.isEmpty()) {
+ continue; // skip empty lines
+ }
+
+ if (currentCommand == null) {
+ currentCommand = currLine;
+ } else {
+ currentCommand = currentCommand + " " + currLine;
+ }
+ if (isPartialCommand(currLine)) {
+ // if its a partial line, continue collecting the pieces
+ continue;
+ }
+
+ // if this is a valid executable command then add it to the buffer
+ if (!isNonExecCommand(currentCommand)) {
+ currentCommand = cleanseCommand(currentCommand);
+ if (isNestedScript(currentCommand)) {
+ // if this is a nested sql script then flatten it
+ String currScript = getScriptName(currentCommand);
+ sb.append(buildCommand(scriptDir, currScript));
+ } else {
+ // Now we have a complete statement, process it
+ // write the line to buffer
+ sb.append(currentCommand);
+ sb.append(System.getProperty("line.separator"));
+ }
+ }
+ currentCommand = null;
+ }
+ bfReader.close();
+ return sb.toString();
+ }
+
+ private void setDbOpts(String dbOpts) {
+ if (dbOpts != null) {
+ this.dbOpts = Lists.newArrayList(dbOpts.split(","));
+ } else {
+ this.dbOpts = Lists.newArrayList();
+ }
+ }
+
+ protected List<String> getDbOpts() {
+ return dbOpts;
+ }
+
+ protected String getMsUsername() {
+ return msUsername;
+ }
+
+ protected String getMsPassword() {
+ return msPassword;
+ }
+
+ protected HiveConf getHiveConf() {
+ return hiveConf;
+ }
+ }
+
+ // Derby commandline parser
+ public static class DerbyCommandParser extends AbstractCommandParser {
+ private static final String DERBY_NESTING_TOKEN = "RUN";
+
+ public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a script format " + dbCommand);
+ }
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1].replace(";", "").replaceAll("'", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ // Derby script format is RUN '<file>'
+ return dbCommand.startsWith(DERBY_NESTING_TOKEN);
+ }
+ }
+
+ // Derby commandline parser
+ public static class HiveCommandParser extends AbstractCommandParser {
+ private static String HIVE_NESTING_TOKEN = "SOURCE";
+ private final NestedScriptParser nestedDbCommandParser;
+
+ public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf, String metaDbType) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ nestedDbCommandParser = getDbCommandParser(metaDbType);
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return nestedDbCommandParser.getQuoteCharacter();
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a script format " + dbCommand);
+ }
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(HIVE_NESTING_TOKEN);
+ }
+ }
+
+ // MySQL parser
+ public static class MySqlCommandParser extends AbstractCommandParser {
+ private static final String MYSQL_NESTING_TOKEN = "SOURCE";
+ private static final String DELIMITER_TOKEN = "DELIMITER";
+ private String delimiter = DEFAULT_DELIMITER;
+
+ public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ }
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ boolean isPartial = super.isPartialCommand(dbCommand);
+ // if this is a delimiter directive, reset our delimiter
+ if (dbCommand.startsWith(DELIMITER_TOKEN)) {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ delimiter = tokens[1];
+ }
+ return isPartial;
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
+ }
+
+ @Override
+ public String getDelimiter() {
+ return delimiter;
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return "`";
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return super.isNonExecCommand(dbCommand) ||
+ (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
+ dbCommand.startsWith(DELIMITER_TOKEN);
+ }
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
+ }
+
+ }
+
+ // Postgres specific parser
+ public static class PostgresCommandParser extends AbstractCommandParser {
+ private static final String POSTGRES_NESTING_TOKEN = "\\i";
+ @VisibleForTesting
+ public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
+ @VisibleForTesting
+ public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
+
+ public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return true;
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ // Skip "standard_conforming_strings" command which is read-only in older
+ // Postgres versions like 8.1
+ // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
+ if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
+ if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
+ return true;
+ }
+ }
+ return super.isNonExecCommand(dbCommand);
+ }
+ }
+
+ //Oracle specific parser
+ public static class OracleCommandParser extends AbstractCommandParser {
+ private static final String ORACLE_NESTING_TOKEN = "@";
+
+ public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a nested script format " + dbCommand);
+ }
+ // remove ending ';' and starting '@'
+ return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
+ }
+ }
+
+ //MSSQL specific parser
+ public static class MSSQLCommandParser extends AbstractCommandParser {
+ private static final String MSSQL_NESTING_TOKEN = ":r";
+
+ public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf) {
+ super(dbOpts, msUsername, msPassword, hiveConf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1];
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
+ }
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName) {
+ return getDbCommandParser(dbName, null);
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
+ return getDbCommandParser(dbName, null, null, null, null, metaDbName);
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName,
+ String dbOpts, String msUsername, String msPassword,
+ HiveConf hiveConf, String metaDbType) {
+ if (dbName.equalsIgnoreCase(DB_DERBY)) {
+ return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+ } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
+ return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType);
+ } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
+ return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+ } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
+ return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+ } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+ return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+ } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
+ return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf);
+ } else {
+ throw new IllegalArgumentException("Unknown dbType " + dbName);
+ }
+ }
+
+ public static class MetaStoreConnectionInfo {
+ private final String userName;
+ private final String password;
+ private final String url;
+ private final String driver;
+ private final boolean printInfo;
+ private final HiveConf hiveConf;
+ private final String dbType;
+
+ public MetaStoreConnectionInfo(String userName, String password, String url, String driver,
+ boolean printInfo, HiveConf hiveConf, String dbType) {
+ super();
+ this.userName = userName;
+ this.password = password;
+ this.url = url;
+ this.driver = driver;
+ this.printInfo = printInfo;
+ this.hiveConf = hiveConf;
+ this.dbType = dbType;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public String getDriver() {
+ return driver;
+ }
+
+ public boolean isPrintInfo() {
+ return printInfo;
+ }
+
+ public HiveConf getHiveConf() {
+ return hiveConf;
+ }
+
+ public String getUsername() {
+ return userName;
+ }
+
+ public boolean getPrintInfo() {
+ return printInfo;
+ }
+
+ public String getDbType() {
+ return dbType;
+ }
+ }
+}
[04/36] hive git commit: HIVE-16591 : DR for function Binaries on
HDFS (Anishek Agarwal, reviewed by Sankar Hariappan, Thejas Nair)
Posted by we...@apache.org.
HIVE-16591 : DR for function Binaries on HDFS (Anishek Agarwal, reviewed by Sankar Hariappan, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f5b22502
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f5b22502
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f5b22502
Branch: refs/heads/hive-14535
Commit: f5b2250219a254c54886ebe9aa70b7dd5bc00b5f
Parents: 727a3df
Author: Anishek Agarwal <an...@gmail.com>
Authored: Wed May 31 09:20:14 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed May 31 09:20:23 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
itests/hive-unit/pom.xml | 6 +
.../hive/metastore/TestReplChangeManager.java | 89 ++++++----
...TestReplicationScenariosAcrossInstances.java | 166 +++++++++++++----
.../hadoop/hive/ql/parse/WarehouseInstance.java | 176 +++++++++++++------
.../hadoop/hive/metastore/HiveMetaStore.java | 14 +-
.../hive/metastore/ReplChangeManager.java | 51 ++++--
.../apache/hadoop/hive/metastore/Warehouse.java | 4 +
pom.xml | 4 +-
ql/pom.xml | 13 ++
.../hadoop/hive/ql/exec/ReplCopyTask.java | 22 ++-
.../ql/parse/ReplicationSemanticAnalyzer.java | 50 +++---
.../hadoop/hive/ql/parse/repl/PathBuilder.java | 66 +++++++
.../repl/dump/events/CreateFunctionHandler.java | 9 +-
.../parse/repl/dump/io/FunctionSerializer.java | 40 ++++-
.../hive/ql/parse/repl/load/MetaData.java | 2 +-
.../load/message/CreateFunctionHandler.java | 156 ++++++++++++++--
.../PrimaryToReplicaResourceFunctionTest.java | 87 +++++++++
18 files changed, 756 insertions(+), 201 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 2dfc8b6..5344f36 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -449,6 +449,8 @@ public class HiveConf extends Configuration {
REPLCMINTERVAL("hive.repl.cm.interval","3600s",
new TimeValidator(TimeUnit.SECONDS),
"Inteval for cmroot cleanup thread."),
+ REPL_FUNCTIONS_ROOT_DIR("hive.repl.replica.functions.root.dir","/user/hive/repl/functions/",
+ "Root directory on the replica warehouse where the repl sub-system will store jars from the primary warehouse"),
LOCALSCRATCHDIR("hive.exec.local.scratchdir",
"${system:java.io.tmpdir}" + File.separator + "${system:user.name}",
"Local scratch space for Hive jobs"),
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/itests/hive-unit/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index 8adf309..ba9d7b9 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -393,6 +393,12 @@
<version>${curator.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
+ <version>${hamcrest.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
index 3f9eec3..1495c1a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java
@@ -47,6 +47,9 @@ import org.junit.Test;
import com.google.common.collect.ImmutableMap;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
public class TestReplChangeManager {
private static HiveMetaStoreClient client;
private static HiveConf hiveConf;
@@ -163,28 +166,28 @@ public class TestReplChangeManager {
createFile(part3Path, "p3");
String path3Chksum = ReplChangeManager.getChksumString(part3Path, fs);
- Assert.assertTrue(part1Path.getFileSystem(hiveConf).exists(part1Path));
- Assert.assertTrue(part2Path.getFileSystem(hiveConf).exists(part2Path));
- Assert.assertTrue(part3Path.getFileSystem(hiveConf).exists(part3Path));
+ assertTrue(part1Path.getFileSystem(hiveConf).exists(part1Path));
+ assertTrue(part2Path.getFileSystem(hiveConf).exists(part2Path));
+ assertTrue(part3Path.getFileSystem(hiveConf).exists(part3Path));
ReplChangeManager cm = ReplChangeManager.getInstance(hiveConf);
// verify cm.recycle(db, table, part) api moves file to cmroot dir
int ret = cm.recycle(part1Path, false);
Assert.assertEquals(ret, 1);
- Path cmPart1Path = ReplChangeManager.getCMPath(part1Path, hiveConf, path1Chksum);
- Assert.assertTrue(cmPart1Path.getFileSystem(hiveConf).exists(cmPart1Path));
+ Path cmPart1Path = ReplChangeManager.getCMPath(hiveConf, path1Chksum);
+ assertTrue(cmPart1Path.getFileSystem(hiveConf).exists(cmPart1Path));
// Verify dropPartition recycle part files
client.dropPartition(dbName, tblName, Arrays.asList("20160102"));
- Assert.assertFalse(part2Path.getFileSystem(hiveConf).exists(part2Path));
- Path cmPart2Path = ReplChangeManager.getCMPath(part2Path, hiveConf, path2Chksum);
- Assert.assertTrue(cmPart2Path.getFileSystem(hiveConf).exists(cmPart2Path));
+ assertFalse(part2Path.getFileSystem(hiveConf).exists(part2Path));
+ Path cmPart2Path = ReplChangeManager.getCMPath(hiveConf, path2Chksum);
+ assertTrue(cmPart2Path.getFileSystem(hiveConf).exists(cmPart2Path));
// Verify dropTable recycle partition files
client.dropTable(dbName, tblName);
- Assert.assertFalse(part3Path.getFileSystem(hiveConf).exists(part3Path));
- Path cmPart3Path = ReplChangeManager.getCMPath(part3Path, hiveConf, path3Chksum);
- Assert.assertTrue(cmPart3Path.getFileSystem(hiveConf).exists(cmPart3Path));
+ assertFalse(part3Path.getFileSystem(hiveConf).exists(part3Path));
+ Path cmPart3Path = ReplChangeManager.getCMPath(hiveConf, path3Chksum);
+ assertTrue(cmPart3Path.getFileSystem(hiveConf).exists(cmPart3Path));
client.dropDatabase(dbName, true, true);
}
@@ -233,28 +236,28 @@ public class TestReplChangeManager {
createFile(filePath3, "f3");
String fileChksum3 = ReplChangeManager.getChksumString(filePath3, fs);
- Assert.assertTrue(filePath1.getFileSystem(hiveConf).exists(filePath1));
- Assert.assertTrue(filePath2.getFileSystem(hiveConf).exists(filePath2));
- Assert.assertTrue(filePath3.getFileSystem(hiveConf).exists(filePath3));
+ assertTrue(filePath1.getFileSystem(hiveConf).exists(filePath1));
+ assertTrue(filePath2.getFileSystem(hiveConf).exists(filePath2));
+ assertTrue(filePath3.getFileSystem(hiveConf).exists(filePath3));
ReplChangeManager cm = ReplChangeManager.getInstance(hiveConf);
// verify cm.recycle(Path) api moves file to cmroot dir
cm.recycle(filePath1, false);
- Assert.assertFalse(filePath1.getFileSystem(hiveConf).exists(filePath1));
+ assertFalse(filePath1.getFileSystem(hiveConf).exists(filePath1));
- Path cmPath1 = ReplChangeManager.getCMPath(filePath1, hiveConf, fileChksum1);
- Assert.assertTrue(cmPath1.getFileSystem(hiveConf).exists(cmPath1));
+ Path cmPath1 = ReplChangeManager.getCMPath(hiveConf, fileChksum1);
+ assertTrue(cmPath1.getFileSystem(hiveConf).exists(cmPath1));
// Verify dropTable recycle table files
client.dropTable(dbName, tblName);
- Path cmPath2 = ReplChangeManager.getCMPath(filePath2, hiveConf, fileChksum2);
- Assert.assertFalse(filePath2.getFileSystem(hiveConf).exists(filePath2));
- Assert.assertTrue(cmPath2.getFileSystem(hiveConf).exists(cmPath2));
+ Path cmPath2 = ReplChangeManager.getCMPath(hiveConf, fileChksum2);
+ assertFalse(filePath2.getFileSystem(hiveConf).exists(filePath2));
+ assertTrue(cmPath2.getFileSystem(hiveConf).exists(cmPath2));
- Path cmPath3 = ReplChangeManager.getCMPath(filePath3, hiveConf, fileChksum3);
- Assert.assertFalse(filePath3.getFileSystem(hiveConf).exists(filePath3));
- Assert.assertTrue(cmPath3.getFileSystem(hiveConf).exists(cmPath3));
+ Path cmPath3 = ReplChangeManager.getCMPath(hiveConf, fileChksum3);
+ assertFalse(filePath3.getFileSystem(hiveConf).exists(filePath3));
+ assertTrue(cmPath3.getFileSystem(hiveConf).exists(cmPath3));
client.dropDatabase(dbName, true, true);
}
@@ -294,17 +297,17 @@ public class TestReplChangeManager {
ReplChangeManager.getInstance(hiveConf).recycle(dirTbl2, false);
ReplChangeManager.getInstance(hiveConf).recycle(dirTbl3, true);
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part11, hiveConf, fileChksum11)));
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part12, hiveConf, fileChksum12)));
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part21, hiveConf, fileChksum21)));
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part22, hiveConf, fileChksum22)));
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part31, hiveConf, fileChksum31)));
- Assert.assertTrue(fs.exists(ReplChangeManager.getCMPath(part32, hiveConf, fileChksum32)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31)));
+ assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum32)));
- fs.setTimes(ReplChangeManager.getCMPath(part11, hiveConf, fileChksum11), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(part21, hiveConf, fileChksum21), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(part31, hiveConf, fileChksum31), now - 86400*1000*2, now - 86400*1000*2);
- fs.setTimes(ReplChangeManager.getCMPath(part32, hiveConf, fileChksum32), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum11), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum21), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum31), now - 86400*1000*2, now - 86400*1000*2);
+ fs.setTimes(ReplChangeManager.getCMPath(hiveConf, fileChksum32), now - 86400*1000*2, now - 86400*1000*2);
ReplChangeManager.scheduleCMClearer(hiveConf);
@@ -317,14 +320,22 @@ public class TestReplChangeManager {
if (end - start > 5000) {
Assert.fail("timeout, cmroot has not been cleared");
}
- if (!fs.exists(ReplChangeManager.getCMPath(part11, hiveConf, fileChksum11)) &&
- fs.exists(ReplChangeManager.getCMPath(part12, hiveConf, fileChksum12)) &&
- !fs.exists(ReplChangeManager.getCMPath(part21, hiveConf, fileChksum21)) &&
- fs.exists(ReplChangeManager.getCMPath(part22, hiveConf, fileChksum22)) &&
- !fs.exists(ReplChangeManager.getCMPath(part31, hiveConf, fileChksum31)) &&
- !fs.exists(ReplChangeManager.getCMPath(part31, hiveConf, fileChksum31))) {
+ if (!fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum11)) &&
+ fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum12)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum21)) &&
+ fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum22)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31)) &&
+ !fs.exists(ReplChangeManager.getCMPath(hiveConf, fileChksum31))) {
cleared = true;
}
} while (!cleared);
}
+
+ @Test
+ public void shouldIdentifyCMURIs() {
+ assertTrue(ReplChangeManager
+ .isCMFileUri(new Path("hdfs://localhost:90000/somepath/adir/", "ab.jar#e239s2233"), fs));
+ assertFalse(ReplChangeManager
+ .isCMFileUri(new Path("/somepath/adir/", "ab.jar"), fs));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 3c1ef08..6713dff 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -1,23 +1,30 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ <p>
+ http://www.apache.org/licenses/LICENSE-2.0
+ <p>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
*/
package org.apache.hadoop.hive.ql.parse;
-import org.junit.After;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
+import org.apache.hadoop.hive.ql.util.DependencyResolver;
+import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -27,44 +34,56 @@ import org.junit.rules.TestRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
public class TestReplicationScenariosAcrossInstances {
@Rule
public final TestName testName = new TestName();
@Rule
- public TestRule replV1BackwardCompat = primary.getReplivationV1CompatRule(new ArrayList<String>());
+ public TestRule replV1BackwardCompat;
protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationScenarios.class);
private static WarehouseInstance primary, replica;
- @BeforeClass
- public static void classLevelSetup() throws Exception {
- primary = new WarehouseInstance();
- replica = new WarehouseInstance();
- }
+ @BeforeClass
+ public static void classLevelSetup() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("dfs.client.use.datanode.hostname", "true");
+ MiniDFSCluster miniDFSCluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
+ primary = new WarehouseInstance(LOG, miniDFSCluster);
+ replica = new WarehouseInstance(LOG, miniDFSCluster);
+ }
+
+ @AfterClass
+ public static void classLevelTearDown() throws IOException {
+ primary.close();
+ replica.close();
+ }
private String primaryDbName, replicatedDbName;
@Before
public void setup() throws Throwable {
+ replV1BackwardCompat = primary.getReplivationV1CompatRule(new ArrayList<>());
primaryDbName = testName.getMethodName() + "_" + +System.currentTimeMillis();
replicatedDbName = "replicated_" + primaryDbName;
primary.run("create database " + primaryDbName);
}
- @After
- public void tearDown() throws Throwable {
- primary.run(dropCommand(primaryDbName));
- replica.run(dropCommand(replicatedDbName));
- }
-
- private String dropCommand(String dbName) {
- return "drop database if exists " + dbName + " cascade ";
- }
-
@Test
public void testCreateFunctionIncrementalReplication() throws Throwable {
WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName, null);
@@ -73,10 +92,11 @@ public class TestReplicationScenariosAcrossInstances {
.verify(bootStrapDump.lastReplicationId);
primary.run("CREATE FUNCTION " + primaryDbName
- + ".testFunction as 'com.yahoo.sketches.hive.theta.DataToSketchUDAF' "
- + "using jar 'ivy://com.yahoo.datasketches:sketches-hive:0.8.2'");
+ + ".testFunction as 'hivemall.tools.string.StopwordUDF' "
+ + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'");
- WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, bootStrapDump.lastReplicationId);
+ WarehouseInstance.Tuple incrementalDump =
+ primary.dump(primaryDbName, bootStrapDump.lastReplicationId);
replica.load(replicatedDbName, incrementalDump.dumpLocation)
.run("REPL STATUS " + replicatedDbName)
.verify(incrementalDump.lastReplicationId)
@@ -96,7 +116,8 @@ public class TestReplicationScenariosAcrossInstances {
primary.run("Drop FUNCTION " + primaryDbName + ".testFunction ");
- WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, bootStrapDump.lastReplicationId);
+ WarehouseInstance.Tuple incrementalDump =
+ primary.dump(primaryDbName, bootStrapDump.lastReplicationId);
replica.load(replicatedDbName, incrementalDump.dumpLocation)
.run("REPL STATUS " + replicatedDbName)
.verify(incrementalDump.lastReplicationId)
@@ -107,8 +128,8 @@ public class TestReplicationScenariosAcrossInstances {
@Test
public void testBootstrapFunctionReplication() throws Throwable {
primary.run("CREATE FUNCTION " + primaryDbName
- + ".testFunction as 'com.yahoo.sketches.hive.theta.DataToSketchUDAF' "
- + "using jar 'ivy://com.yahoo.datasketches:sketches-hive:0.8.2'");
+ + ".testFunction as 'hivemall.tools.string.StopwordUDF' "
+ + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'");
WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName, null);
replica.load(replicatedDbName, bootStrapDump.dumpLocation)
@@ -116,4 +137,71 @@ public class TestReplicationScenariosAcrossInstances {
.verify(replicatedDbName + ".testFunction");
}
+ @Test
+ public void testCreateFunctionWithFunctionBinaryJarsOnHDFS() throws Throwable {
+ Dependencies dependencies = dependencies("ivy://io.github.myui:hivemall:0.4.0-2", primary);
+ String jarSubString = dependencies.toJarSubSql();
+
+ primary.run("CREATE FUNCTION " + primaryDbName
+ + ".anotherFunction as 'hivemall.tools.string.StopwordUDF' "
+ + "using " + jarSubString);
+
+ WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null);
+
+ replica.load(replicatedDbName, tuple.dumpLocation)
+ .run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "*'")
+ .verify(replicatedDbName + ".anotherFunction");
+
+ FileStatus[] fileStatuses = replica.miniDFSCluster.getFileSystem().globStatus(
+ new Path(
+ replica.functionsRoot + "/" + replicatedDbName.toLowerCase() + "/anotherfunction/*/*")
+ , path -> path.toString().endsWith("jar"));
+ List<String> expectedDependenciesNames = dependencies.jarNames();
+ assertThat(fileStatuses.length, is(equalTo(expectedDependenciesNames.size())));
+ List<String> jars = Arrays.stream(fileStatuses).map(f -> {
+ String[] splits = f.getPath().toString().split("/");
+ return splits[splits.length - 1];
+ }).collect(Collectors.toList());
+
+ assertThat(jars, containsInAnyOrder(expectedDependenciesNames.toArray()));
+ }
+
+ static class Dependencies {
+ private final List<Path> fullQualifiedJarPaths;
+
+ Dependencies(List<Path> fullQualifiedJarPaths) {
+ this.fullQualifiedJarPaths = fullQualifiedJarPaths;
+ }
+
+ private String toJarSubSql() {
+ return StringUtils.join(
+ fullQualifiedJarPaths.stream().map(p -> "jar '" + p + "'").collect(Collectors.toList()),
+ ","
+ );
+ }
+
+ private List<String> jarNames() {
+ return fullQualifiedJarPaths.stream().map(p -> {
+ String[] splits = p.toString().split("/");
+ return splits[splits.length - 1];
+ }).collect(Collectors.toList());
+ }
+ }
+
+ private Dependencies dependencies(String ivyPath, WarehouseInstance onWarehouse)
+ throws IOException, URISyntaxException, SemanticException {
+ List<URI> localUris = new DependencyResolver().downloadDependencies(new URI(ivyPath));
+ List<Path> remotePaths = onWarehouse.copyToHDFS(localUris);
+ List<Path> collect =
+ remotePaths.stream().map(r -> {
+ try {
+ return PathBuilder
+ .fullyQualifiedHDFSUri(r, onWarehouse.miniDFSCluster.getFileSystem());
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toList());
+ return new Dependencies(collect);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index 7271eae..a35f7b2 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -1,102 +1,131 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ <p>
+ http://www.apache.org/licenses/LICENSE-2.0
+ <p>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
*/
package org.apache.hadoop.hive.ql.parse;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule;
import org.apache.hive.hcatalog.listener.DbNotificationListener;
-import org.junit.rules.TestRule;
+import org.codehaus.plexus.util.ExceptionUtils;
+import org.slf4j.Logger;
+import java.io.Closeable;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
-class WarehouseInstance {
+class WarehouseInstance implements Closeable {
+ final String functionsRoot;
+ private Logger logger;
private Driver driver;
+ private HiveConf hiveConf;
+ MiniDFSCluster miniDFSCluster;
private HiveMetaStoreClient client;
- private HiveConf hconf;
- private static int schemaNameCounter = 0;
+ private static int uniqueIdentifier = 0;
+
private final static String LISTENER_CLASS = DbNotificationListener.class.getCanonicalName();
- /**
- * This will be used to allow the primary and replica warehouse to be the same instance of
- * hive server
- */
- WarehouseInstance(WarehouseInstance other){
- this.driver = other.driver;
- this.client = other.client;
- this.hconf = other.hconf;
+ WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception {
+ this.logger = logger;
+ this.miniDFSCluster = cluster;
+ assert miniDFSCluster.isClusterUp();
+ assert miniDFSCluster.isDataNodeUp();
+ DistributedFileSystem fs = miniDFSCluster.getFileSystem();
+
+ Path cmRootPath = mkDir(fs, "/cmroot" + uniqueIdentifier);
+ this.functionsRoot = mkDir(fs, "/functions" + uniqueIdentifier).toString();
+ initialize(cmRootPath.toString());
}
- WarehouseInstance() throws Exception {
- hconf = new HiveConf(TestReplicationScenarios.class);
+ private void initialize(String cmRoot) throws Exception {
+ hiveConf = new HiveConf(miniDFSCluster.getConfiguration(0), TestReplicationScenarios.class);
String metaStoreUri = System.getProperty("test." + HiveConf.ConfVars.METASTOREURIS.varname);
String hiveWarehouseLocation = System.getProperty("test.warehouse.dir", "/tmp")
+ Path.SEPARATOR
+ TestReplicationScenarios.class.getCanonicalName().replace('.', '_')
+ "_"
+ System.nanoTime();
-
if (metaStoreUri != null) {
- hconf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri);
- // useExternalMS = true;
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri);
return;
}
// turn on db notification listener on meta store
- hconf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, LISTENER_CLASS);
- hconf.setBoolVar(HiveConf.ConfVars.REPLCMENABLED, true);
- hconf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true);
- hconf.setVar(HiveConf.ConfVars.REPLCMDIR, hiveWarehouseLocation + "/cmroot/");
- String schemaName = "APP" + schemaNameCounter++;
+ hiveConf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, LISTENER_CLASS);
+ hiveConf.setBoolVar(HiveConf.ConfVars.REPLCMENABLED, true);
+ hiveConf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true);
+ hiveConf.setVar(HiveConf.ConfVars.REPLCMDIR, cmRoot);
+ hiveConf.setVar(HiveConf.ConfVars.REPL_FUNCTIONS_ROOT_DIR, functionsRoot);
+ String schemaName = "APP" + uniqueIdentifier;
System.setProperty("datanucleus.mapping.Schema", schemaName);
- hconf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
+ hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
"jdbc:derby:memory:${test.tmp.dir}/" + schemaName + ";create=true");
- int metaStorePort = MetaStoreUtils.startMetaStore(hconf);
- hconf.setVar(HiveConf.ConfVars.REPLDIR, hiveWarehouseLocation + "/hrepl/");
- hconf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
- hconf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
- hconf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
- hconf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
- hconf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+
+ int metaStorePort = MetaStoreUtils.startMetaStore(hiveConf);
+ hiveConf.setVar(HiveConf.ConfVars.REPLDIR,
+ hiveWarehouseLocation + "/hrepl" + uniqueIdentifier + "/");
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort);
+ hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+ hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
Path testPath = new Path(hiveWarehouseLocation);
- FileSystem fs = FileSystem.get(testPath.toUri(), hconf);
- fs.mkdirs(testPath);
+ FileSystem testPathFileSystem = FileSystem.get(testPath.toUri(), hiveConf);
+ testPathFileSystem.mkdirs(testPath);
+
+ driver = new Driver(hiveConf);
+ SessionState.start(new CliSessionState(hiveConf));
+ client = new HiveMetaStoreClient(hiveConf);
+ // change the value for the next instance.
+ ++uniqueIdentifier;
+ }
- driver = new Driver(hconf);
- SessionState.start(new CliSessionState(hconf));
- client = new HiveMetaStoreClient(hconf);
+ private Path mkDir(DistributedFileSystem fs, String pathString)
+ throws IOException, SemanticException {
+ Path path = new Path(pathString);
+ fs.mkdir(path, new FsPermission("777"));
+ return PathBuilder.fullyQualifiedHDFSUri(path, fs);
}
private int next = 0;
@@ -161,8 +190,8 @@ class WarehouseInstance {
*/
private void verifyResults(String[] data) throws IOException {
List<String> results = getOutput();
- TestReplicationScenariosAcrossInstances.LOG.info("Expecting {}", data);
- TestReplicationScenariosAcrossInstances.LOG.info("Got {}", results);
+ logger.info("Expecting {}", StringUtils.join(data, ","));
+ logger.info("Got {}", results);
assertEquals(data.length, results.size());
for (int i = 0; i < data.length; i++) {
assertEquals(data[i].toLowerCase(), results.get(i).toLowerCase());
@@ -174,7 +203,7 @@ class WarehouseInstance {
try {
driver.getResults(results);
} catch (CommandNeedRetryException e) {
- TestReplicationScenariosAcrossInstances.LOG.warn(e.getMessage(), e);
+ logger.warn(e.getMessage(), e);
throw new RuntimeException(e);
}
return results;
@@ -182,12 +211,45 @@ class WarehouseInstance {
private void printOutput() throws IOException {
for (String s : getOutput()) {
- TestReplicationScenariosAcrossInstances.LOG.info(s);
+ logger.info(s);
}
}
- public ReplicationV1CompatRule getReplivationV1CompatRule(List<String> testsToSkip){
- return new ReplicationV1CompatRule(client,hconf,testsToSkip);
+ ReplicationV1CompatRule getReplivationV1CompatRule(List<String> testsToSkip) {
+ return new ReplicationV1CompatRule(client, hiveConf, testsToSkip);
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (miniDFSCluster != null && miniDFSCluster.isClusterUp()) {
+ miniDFSCluster.shutdown();
+ }
+ }
+
+ List<Path> copyToHDFS(List<URI> localUris) throws IOException, SemanticException {
+ DistributedFileSystem fs = miniDFSCluster.getFileSystem();
+ Path destinationBasePath = new Path("/", String.valueOf(System.nanoTime()));
+ mkDir(fs, destinationBasePath.toString());
+ localUris.forEach(uri -> {
+ Path localPath = new Path(uri);
+ try {
+ FileSystem localFs = localPath.getFileSystem(hiveConf);
+ boolean success = FileUtils
+ .copy(localFs, localPath, fs, destinationBasePath, false, false, hiveConf);
+ if (!success) {
+ fail("FileUtils could not copy local uri " + localPath.toString() + " to hdfs");
+ }
+ } catch (IOException e) {
+ String message = "error on copy of local uri " + localPath.toString() + " to hdfs";
+ logger.error(message, e);
+ fail(message + ExceptionUtils.getFullStackTrace(e));
+ }
+ });
+
+ List<FileStatus> fileStatuses =
+ Arrays.asList(fs.globStatus(new Path(destinationBasePath, "*")));
+ return fileStatuses.stream().map(FileStatus::getPath).collect(Collectors.toList());
}
static class Tuple {
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 52bfb26..e886540 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -6377,7 +6377,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (func == null) {
throw new NoSuchObjectException("Function " + funcName + " does not exist");
}
+ // if copy of jar to change management fails we fail the metastore transaction, since the
+ // user might delete the jars on HDFS externally after dropping the function, hence having
+ // a copy is required to allow incremental replication to work correctly.
+ if (func.getResourceUris() != null && !func.getResourceUris().isEmpty()) {
+ for (ResourceUri uri : func.getResourceUris()) {
+ if (uri.getUri().toLowerCase().startsWith("hdfs:")) {
+ wh.addToChangeManagement(new Path(uri.getUri()));
+ }
+ }
+ }
+ // if the operation on metastore fails, we don't do anything in change management, but fail
+ // the metastore transaction, as having a copy of the jar in change management is not going
+ // to cause any problem, the cleaner thread will remove this when this jar expires.
ms.dropFunction(dbName, funcName);
if (transactionalListeners.size() > 0) {
transactionalListenerResponses =
@@ -6385,7 +6398,6 @@ public class HiveMetaStore extends ThriftHiveMetastore {
EventType.DROP_FUNCTION,
new DropFunctionEvent(func, true, this));
}
-
success = ms.commitTransaction();
} finally {
if (!success) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
index 51e4627..c955470 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -100,6 +101,34 @@ public class ReplChangeManager {
}
};
+ void addFile(Path path) throws MetaException {
+ if (!enabled) {
+ return;
+ }
+ try {
+ if (fs.isDirectory(path)) {
+ throw new IllegalArgumentException(path + " cannot be a directory");
+ }
+ Path cmPath = getCMPath(hiveConf, getChksumString(path, fs));
+ boolean copySuccessful = FileUtils
+ .copy(path.getFileSystem(hiveConf), path, cmPath.getFileSystem(hiveConf), cmPath, false,
+ false, hiveConf);
+ if (!copySuccessful) {
+ LOG.debug("A file with the same content of " + path.toString() + " already exists, ignore");
+ } else {
+ fs.setOwner(cmPath, msUser, msGroup);
+ try {
+ fs.setXAttr(cmPath, ORIG_LOC_TAG, path.toString().getBytes());
+ } catch (UnsupportedOperationException e) {
+ LOG.warn("Error setting xattr for " + path.toString());
+ }
+ }
+ } catch (Exception exception) {
+ throw new MetaException(StringUtils.stringifyException(exception));
+ }
+ }
+
+
/***
* Move a path into cmroot. If the path is a directory (of a partition, or table if nonpartitioned),
* recursively move files inside directory to cmroot. Note the table must be managed table
@@ -122,7 +151,7 @@ public class ReplChangeManager {
count += recycle(file.getPath(), ifPurge);
}
} else {
- Path cmPath = getCMPath(path, hiveConf, getChksumString(path, fs));
+ Path cmPath = getCMPath(hiveConf, getChksumString(path, fs));
if (LOG.isDebugEnabled()) {
LOG.debug("Moving " + path.toString() + " to " + cmPath.toString());
@@ -198,16 +227,15 @@ public class ReplChangeManager {
* Convert a path of file inside a partition or table (if non-partitioned)
* to a deterministic location of cmroot. So user can retrieve the file back
* with the original location plus checksum.
- * @param path original path inside partition or table
* @param conf
- * @param chksum checksum of the file, can be retrieved by {@link getCksumString}
+ * @param checkSum checksum of the file, can be retrieved by {@link getCksumString}
* @return
* @throws IOException
* @throws MetaException
*/
- static public Path getCMPath(Path path, Configuration conf, String chksum)
+ static Path getCMPath(Configuration conf, String checkSum)
throws IOException, MetaException {
- String newFileName = chksum;
+ String newFileName = checkSum;
int maxLength = conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
@@ -215,9 +243,7 @@ public class ReplChangeManager {
newFileName = newFileName.substring(0, maxLength-1);
}
- Path cmPath = new Path(cmroot, newFileName);
-
- return cmPath;
+ return new Path(cmroot, newFileName);
}
/***
@@ -238,14 +264,14 @@ public class ReplChangeManager {
}
if (!srcFs.exists(src)) {
- return srcFs.getFileStatus(getCMPath(src, conf, chksumString));
+ return srcFs.getFileStatus(getCMPath(conf, chksumString));
}
String currentChksumString = getChksumString(src, srcFs);
if (currentChksumString == null || chksumString.equals(currentChksumString)) {
return srcFs.getFileStatus(src);
} else {
- return srcFs.getFileStatus(getCMPath(src, conf, chksumString));
+ return srcFs.getFileStatus(getCMPath(conf, chksumString));
}
} catch (IOException e) {
throw new MetaException(StringUtils.stringifyException(e));
@@ -283,6 +309,11 @@ public class ReplChangeManager {
return result;
}
+ public static boolean isCMFileUri(Path fromPath, FileSystem srcFs) {
+ String[] result = getFileWithChksumFromURI(fromPath.toString());
+ return result[1] != null;
+ }
+
/**
* Thread to clear old files of cmroot recursively
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 8134ab2..053a0de 100755
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -206,6 +206,10 @@ public class Warehouse {
return false;
}
+ void addToChangeManagement(Path file) throws MetaException {
+ cm.addFile(file);
+ }
+
public boolean deleteDir(Path f, boolean recursive) throws MetaException {
return deleteDir(f, recursive, false);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e3ff84f..70518d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,7 +143,7 @@
<h2database.version>1.3.166</h2database.version>
<hadoop.version>2.8.0</hadoop.version>
<hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop</hadoop.bin.path>
- <hamcrest.version>1.1</hamcrest.version>
+ <hamcrest.version>1.3</hamcrest.version>
<hbase.version>1.1.1</hbase.version>
<!-- required for logging test to avoid including hbase which pulls disruptor transitively -->
<disruptor.version>3.3.0</disruptor.version>
@@ -178,7 +178,7 @@
<log4j2.version>2.6.2</log4j2.version>
<opencsv.version>2.3</opencsv.version>
<orc.version>1.3.3</orc.version>
- <mockito-all.version>1.9.5</mockito-all.version>
+ <mockito-all.version>1.10.19</mockito-all.version>
<mina.version>2.0.0-M5</mina.version>
<netty.version>4.0.29.Final</netty.version>
<parquet.version>1.8.1</parquet.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index 40a216b..9c17695 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -29,6 +29,7 @@
<properties>
<hive.path.to.root>..</hive.path.to.root>
+ <powermock.version>1.6.6</powermock.version>
</properties>
<dependencies>
@@ -735,6 +736,18 @@
<version>${hamcrest.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index f277284..71db332 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -75,7 +75,27 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
FileSystem srcFs = fromPath.getFileSystem(conf);
dstFs = toPath.getFileSystem(conf);
- List<FileStatus> srcFiles = new ArrayList<FileStatus>();
+ // This should only be true for copy tasks created from functions, otherwise there should never
+ // be a CM uri in the from path.
+ if (ReplChangeManager.isCMFileUri(fromPath, srcFs)) {
+ String[] result = ReplChangeManager.getFileWithChksumFromURI(fromPath.toString());
+ Path sourcePath = ReplChangeManager
+ .getFileStatus(new Path(result[0]), result[1], conf)
+ .getPath();
+ if (FileUtils.copy(
+ sourcePath.getFileSystem(conf), sourcePath,
+ dstFs, toPath
+ , false, false, conf
+ )) {
+ return 0;
+ } else {
+ console.printError("Failed to copy: '" + fromPath.toString() + "to: '" + toPath.toString()
+ + "'");
+ return 1;
+ }
+ }
+
+ List<FileStatus> srcFiles = new ArrayList<>();
FileStatus[] srcs = LoadSemanticAnalyzer.matchFilesOrDir(srcFs, fromPath);
LOG.debug("ReplCopyTasks srcs=" + (srcs == null ? "null" : srcs.length));
if (! rwork.getReadListFromInput()){
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 1e6b192..adcdc12 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -47,21 +47,20 @@ import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import org.apache.hadoop.hive.ql.parse.repl.dump.HiveWrapper;
+import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandler;
import org.apache.hadoop.hive.ql.parse.repl.dump.events.EventHandlerFactory;
-import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
-import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
+import org.apache.hadoop.hive.ql.parse.repl.load.message.CreateFunctionHandler;
import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler;
import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
-import org.apache.hadoop.hive.ql.plan.FunctionWork;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +76,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Consumer;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT;
@@ -377,11 +377,13 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
continue;
}
- Path functionMetadataRoot =
- new Path(new Path(functionsRoot, functionName), FUNCTION_METADATA_DIR_NAME);
+ Path functionRoot = new Path(functionsRoot, functionName);
+ Path functionMetadataRoot = new Path(functionRoot, FUNCTION_METADATA_DIR_NAME);
try (JsonWriter jsonWriter = new JsonWriter(functionMetadataRoot.getFileSystem(conf),
functionMetadataRoot)) {
- new FunctionSerializer(tuple.object).writeTo(jsonWriter, tuple.replicationSpec);
+ FunctionSerializer serializer =
+ new FunctionSerializer(tuple.object, conf);
+ serializer.writeTo(jsonWriter, tuple.replicationSpec);
}
REPL_STATE_LOG.info("Repl Dump: Dumped metadata for function: {}", functionName);
}
@@ -839,31 +841,23 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
.getValidatedURI(conf, stripQuotes(functionDir.getPath().toUri().toString()));
Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath());
- FileSystem fs = FileSystem.get(fromURI, conf);
- inputs.add(toReadEntity(fromPath, conf));
-
try {
- MetaData metaData = EximUtil.readMetaData(fs, new Path(fromPath, EximUtil.METADATA_NAME));
- ReplicationSpec replicationSpec = metaData.getReplicationSpec();
- if (replicationSpec.isNoop()) {
- // nothing to do here, silently return.
- return;
- }
- CreateFunctionDesc desc = new CreateFunctionDesc(
- dbName + "." + metaData.function.getFunctionName(),
- false,
- metaData.function.getClassName(),
- metaData.function.getResourceUris()
+ CreateFunctionHandler handler = new CreateFunctionHandler();
+ List<Task<? extends Serializable>> tasksList = handler.handle(
+ new MessageHandler.Context(
+ dbName, null, fromPath.toString(), createDbTask, null, conf, db,
+ null, LOG)
);
- Task<FunctionWork> currentTask = TaskFactory.get(new FunctionWork(desc), conf);
- if (createDbTask != null) {
- createDbTask.addDependentTask(currentTask);
+ tasksList.forEach(task -> {
+ createDbTask.addDependentTask(task);
LOG.debug("Added {}:{} as a precursor of {}:{}",
- createDbTask.getClass(), createDbTask.getId(), currentTask.getClass(),
- currentTask.getId());
- }
- } catch (IOException e) {
+ createDbTask.getClass(), createDbTask.getId(), task.getClass(),
+ task.getId());
+
+ });
+ inputs.addAll(handler.readEntities());
+ } catch (Exception e) {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/PathBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/PathBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/PathBuilder.java
new file mode 100644
index 0000000..05b9821
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/PathBuilder.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+
+/**
+ * Path builder to stitch together paths with different components might be useful as utils across
+ * replication semantic analyzer atleast.
+ */
+public class PathBuilder {
+ private ArrayList<String> descendants = new ArrayList<>();
+ private String basePath;
+
+ public PathBuilder(String basePath) {
+ this.basePath = basePath;
+ }
+
+ public PathBuilder addDescendant(String path) {
+ descendants.add(path);
+ return this;
+ }
+
+ public Path build() {
+ Path result = new Path(this.basePath);
+ for (String descendant : descendants) {
+ result = new Path(result, descendant);
+ }
+ return result;
+ }
+
+ public static Path fullyQualifiedHDFSUri(Path input, FileSystem hdfsFileSystem)
+ throws SemanticException {
+ URI uri = input.toUri();
+ String scheme = hdfsFileSystem.getScheme();
+ String authority = hdfsFileSystem.getUri().getAuthority();
+ String path = uri.getPath();
+ try {
+ return new Path(new URI(scheme, authority, path, null, null));
+ } catch (URISyntaxException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java
index d04c7b5..ee3432c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.hadoop.hive.ql.parse.repl.dump.events;
import org.apache.hadoop.fs.FileSystem;
@@ -23,11 +23,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
import org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter;
-import org.apache.hadoop.hive.ql.parse.repl.DumpType;
-
class CreateFunctionHandler extends AbstractEventHandler {
CreateFunctionHandler(NotificationEvent event) {
super(event);
@@ -42,7 +41,7 @@ class CreateFunctionHandler extends AbstractEventHandler {
FileSystem fileSystem = metadataPath.getFileSystem(withinContext.hiveConf);
try (JsonWriter jsonWriter = new JsonWriter(fileSystem, metadataPath)) {
- new FunctionSerializer(createFunctionMessage.getFunctionObj())
+ new FunctionSerializer(createFunctionMessage.getFunctionObj(), withinContext.hiveConf)
.writeTo(jsonWriter, withinContext.replicationSpec);
}
withinContext.createDmd(this).write();
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
index 5dc7023..6c2a402 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java
@@ -17,21 +17,31 @@
*/
package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ReplChangeManager;
import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TJSONProtocol;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
public class FunctionSerializer implements JsonWriter.Serializer {
- public static final String FIELD_NAME="function";
+ public static final String FIELD_NAME = "function";
private Function function;
+ private HiveConf hiveConf;
- public FunctionSerializer(Function function) {
+ public FunctionSerializer(Function function, HiveConf hiveConf) {
+ this.hiveConf = hiveConf;
this.function = function;
}
@@ -39,9 +49,33 @@ public class FunctionSerializer implements JsonWriter.Serializer {
public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
throws SemanticException, IOException {
TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+ List<ResourceUri> resourceUris = new ArrayList<>();
+ for (ResourceUri uri : function.getResourceUris()) {
+ Path inputPath = new Path(uri.getUri());
+ if ("hdfs".equals(inputPath.toUri().getScheme())) {
+ FileSystem fileSystem = inputPath.getFileSystem(hiveConf);
+ Path qualifiedUri = PathBuilder.fullyQualifiedHDFSUri(inputPath, fileSystem);
+ String checkSum = ReplChangeManager.getChksumString(qualifiedUri, fileSystem);
+ String newFileUri = ReplChangeManager.encodeFileUri(qualifiedUri.toString(), checkSum);
+ resourceUris.add(new ResourceUri(uri.getResourceType(), newFileUri));
+ } else {
+ resourceUris.add(uri);
+ }
+ }
+ Function copyObj = new Function(this.function);
+ if (!resourceUris.isEmpty()) {
+ assert resourceUris.size() == this.function.getResourceUris().size();
+ copyObj.setResourceUris(resourceUris);
+ }
+
try {
+ //This is required otherwise correct work object on repl load wont be created.
+ writer.jsonGenerator.writeStringField(ReplicationSpec.KEY.REPL_SCOPE.toString(),
+ "all");
+ writer.jsonGenerator.writeStringField(ReplicationSpec.KEY.CURR_STATE_ID.toString(),
+ additionalPropertiesProvider.getCurrentReplicationState());
writer.jsonGenerator
- .writeStringField(FIELD_NAME, serializer.toString(function, UTF_8));
+ .writeStringField(FIELD_NAME, serializer.toString(copyObj, UTF_8));
} catch (TException e) {
throw new SemanticException(ErrorMsg.ERROR_SERIALIZE_METASTORE.getMsg(), e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
index fc02dfd..17d8ab2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
@@ -37,7 +37,7 @@ public class MetaData {
this(null, null, null, new ReplicationSpec(), null);
}
- MetaData(Database db, Table table, Iterable<Partition> partitions,
+ public MetaData(Database db, Table table, Iterable<Partition> partitions,
ReplicationSpec replicationSpec, Function function) {
this.db = db;
this.table = table;
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index 8b6179b..452f506 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -18,51 +18,177 @@
*/
package org.apache.hadoop.hive.ql.parse.repl.load.message;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ReplChangeManager;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.FunctionUtils;
+import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.PathBuilder;
import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
+import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
import org.apache.hadoop.hive.ql.plan.FunctionWork;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.toReadEntity;
+
public class CreateFunctionHandler extends AbstractMessageHandler {
@Override
public List<Task<? extends Serializable>> handle(Context context)
throws SemanticException {
try {
- FileSystem fs = FileSystem.get(new Path(context.location).toUri(), context.hiveConf);
- MetaData metadata;
+ FunctionDescBuilder builder = new FunctionDescBuilder(context);
+ CreateFunctionDesc descToLoad = builder.build();
+ context.log.debug("Loading function desc : {}", descToLoad.toString());
+ Task<FunctionWork> createTask = TaskFactory.get(
+ new FunctionWork(descToLoad), context.hiveConf
+ );
+ context.log.debug("Added create function task : {}:{},{}", createTask.getId(),
+ descToLoad.getFunctionName(), descToLoad.getClassName());
+ // This null check is specifically done as the same class is used to handle both incremental and
+ // bootstrap replication scenarios for create function. When doing bootstrap we do not have
+ // event id for this event but rather when bootstrap started and hence we pass in null dmd for
+ // bootstrap.There should be a better way to do this but might required a lot of changes across
+ // different handlers, unless this is a common pattern that is seen, leaving this here.
+ if (context.dmd != null) {
+ databasesUpdated.put(builder.destinationDbName, context.dmd.getEventTo());
+ }
+ readEntitySet.add(toReadEntity(new Path(context.location), context.hiveConf));
+ if (builder.replCopyTasks.isEmpty()) {
+ // reply copy only happens for jars on hdfs not otherwise.
+ return Collections.singletonList(createTask);
+ } else {
+ /**
+ * This is to understand how task dependencies work.
+ * All root tasks are executed in parallel. For bootstrap replication there should be only one root task of creating db. Incremental can be multiple ( have to verify ).
+ * Task has children, which are put in queue for execution after the parent has finished execution.
+ * One -to- One dependency can be satisfied by adding children to a given task, do this recursively where the relation holds.
+ * for many to one , create a barrier task that is the child of every item in 'many' dependencies, make the 'one' dependency as child of barrier task.
+ * add the 'many' to parent/root tasks. The execution environment will make sure that the child barrier task will not get executed unless all parents of the barrier task are complete,
+ * which should only happen when the last task is finished, at which point the child of the barrier task is picked up.
+ */
+ Task<? extends Serializable> barrierTask =
+ TaskFactory.get(new DependencyCollectionWork(), context.hiveConf);
+ builder.replCopyTasks.forEach(t -> t.addDependentTask(barrierTask));
+ barrierTask.addDependentTask(createTask);
+ return builder.replCopyTasks;
+ }
+ } catch (Exception e) {
+ throw (e instanceof SemanticException)
+ ? (SemanticException) e
+ : new SemanticException("Error reading message members", e);
+ }
+ }
+
+ private static class FunctionDescBuilder {
+ private final Context context;
+ private final MetaData metadata;
+ private final String destinationDbName;
+ private final List<Task<?>> replCopyTasks = new ArrayList<>();
+
+ private FunctionDescBuilder(Context context) throws SemanticException {
+ this.context = context;
try {
+ FileSystem fs = FileSystem.get(new Path(context.location).toUri(), context.hiveConf);
metadata = EximUtil.readMetaData(fs, new Path(context.location, EximUtil.METADATA_NAME));
} catch (IOException e) {
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
}
+ destinationDbName = context.isDbNameEmpty() ? metadata.function.getDbName() : context.dbName;
+ }
- String dbName = context.isDbNameEmpty() ? metadata.function.getDbName() : context.dbName;
- CreateFunctionDesc desc = new CreateFunctionDesc(
- FunctionUtils.qualifyFunctionName(metadata.function.getFunctionName(), dbName), false,
- metadata.function.getClassName(), metadata.function.getResourceUris()
+ private CreateFunctionDesc build() {
+ replCopyTasks.clear();
+ PrimaryToReplicaResourceFunction conversionFunction =
+ new PrimaryToReplicaResourceFunction(context, metadata, destinationDbName);
+ // We explicitly create immutable lists here as it forces the guava lib to run the transformations
+ // and not do them lazily. The reason being the function class used for transformations additionally
+ // also creates the corresponding replCopyTasks, which cannot be evaluated lazily. since the query
+ // plan needs to be complete before we execute and not modify it while execution in the driver.
+ List<ResourceUri> transformedUris = ImmutableList.copyOf(
+ Lists.transform(metadata.function.getResourceUris(), conversionFunction)
+ );
+ replCopyTasks.addAll(conversionFunction.replCopyTasks);
+ String fullQualifiedFunctionName = FunctionUtils.qualifyFunctionName(
+ metadata.function.getFunctionName(), destinationDbName
);
+ return new CreateFunctionDesc(
+ fullQualifiedFunctionName, false, metadata.function.getClassName(), transformedUris
+ );
+ }
+ }
- Task<FunctionWork> task = TaskFactory.get(new FunctionWork(desc), context.hiveConf);
- context.log.debug("Added create function task : {}:{},{}", task.getId(),
- metadata.function.getFunctionName(), metadata.function.getClassName());
- databasesUpdated.put(dbName, context.dmd.getEventTo());
- return Collections.singletonList(task);
- } catch (Exception e) {
- throw (e instanceof SemanticException)
- ? (SemanticException) e
- : new SemanticException("Error reading message members", e);
+ static class PrimaryToReplicaResourceFunction
+ implements Function<ResourceUri, ResourceUri> {
+ private final Context context;
+ private final MetaData metadata;
+ private final List<Task<?>> replCopyTasks = new ArrayList<>();
+ private final String functionsRootDir;
+ private String destinationDbName;
+
+ PrimaryToReplicaResourceFunction(Context context, MetaData metadata,
+ String destinationDbName) {
+ this.context = context;
+ this.metadata = metadata;
+ this.destinationDbName = destinationDbName;
+ this.functionsRootDir = context.hiveConf.getVar(HiveConf.ConfVars.REPL_FUNCTIONS_ROOT_DIR);
+ }
+
+ @Override
+ public ResourceUri apply(ResourceUri resourceUri) {
+ try {
+ return resourceUri.getUri().toLowerCase().startsWith("hdfs:")
+ ? destinationResourceUri(resourceUri)
+ : resourceUri;
+ } catch (IOException | SemanticException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * the destination also includes the current timestamp to randomise the placement of the jar at a given location for a function .
+ * this is done to allow the CREATE / DROP / CREATE of the same function with same name and jar's but updated
+ * binaries across the two creates.
+ */
+ ResourceUri destinationResourceUri(ResourceUri resourceUri)
+ throws IOException, SemanticException {
+ String sourceUri = resourceUri.getUri();
+ String[] split = sourceUri.split(Path.SEPARATOR);
+ PathBuilder pathBuilder = new PathBuilder(functionsRootDir);
+ Path qualifiedDestinationPath = PathBuilder.fullyQualifiedHDFSUri(
+ pathBuilder
+ .addDescendant(destinationDbName.toLowerCase())
+ .addDescendant(metadata.function.getFunctionName().toLowerCase())
+ .addDescendant(String.valueOf(System.nanoTime()))
+ .addDescendant(ReplChangeManager.getFileWithChksumFromURI(split[split.length - 1])[0])
+ .build(),
+ FileSystem.get(context.hiveConf)
+ );
+
+ Task<?> copyTask = ReplCopyTask.getLoadCopyTask(
+ metadata.getReplicationSpec(), new Path(sourceUri), qualifiedDestinationPath,
+ context.hiveConf
+ );
+ replCopyTasks.add(copyTask);
+ ResourceUri destinationUri =
+ new ResourceUri(resourceUri.getResourceType(), qualifiedDestinationPath.toString());
+ context.log.debug("copy source uri : {} to destination uri: {}", sourceUri, destinationUri);
+ return destinationUri;
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/f5b22502/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/load/message/PrimaryToReplicaResourceFunctionTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/load/message/PrimaryToReplicaResourceFunctionTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/load/message/PrimaryToReplicaResourceFunctionTest.java
new file mode 100644
index 0000000..7a87701
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/load/message/PrimaryToReplicaResourceFunctionTest.java
@@ -0,0 +1,87 @@
+package org.apache.hadoop.hive.ql.parse.repl.load.message;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.ResourceType;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.mockito.Mock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.hive.ql.parse.repl.load.message.CreateFunctionHandler.PrimaryToReplicaResourceFunction;
+import static org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler.Context;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+import static org.mockito.Matchers.any;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ PrimaryToReplicaResourceFunction.class, FileSystem.class, ReplCopyTask.class,
+ System.class })
+public class PrimaryToReplicaResourceFunctionTest {
+
+ private PrimaryToReplicaResourceFunction function;
+ @Mock
+ private HiveConf hiveConf;
+ @Mock
+
+ private Function functionObj;
+ @Mock
+ private FileSystem mockFs;
+ private static Logger logger =
+ LoggerFactory.getLogger(PrimaryToReplicaResourceFunctionTest.class);
+
+ @Before
+ public void setup() {
+ MetaData metadata = new MetaData(null, null, null, null, functionObj);
+ Context context =
+ new Context("primaryDb", null, null, null, null, hiveConf, null, null, logger);
+ when(hiveConf.getVar(HiveConf.ConfVars.REPL_FUNCTIONS_ROOT_DIR))
+ .thenReturn("/someBasePath/withADir/");
+ function = new PrimaryToReplicaResourceFunction(context, metadata, "replicaDbName");
+ }
+
+ @Test
+ public void createDestinationPath() throws IOException, SemanticException, URISyntaxException {
+ mockStatic(FileSystem.class);
+ when(FileSystem.get(any(Configuration.class))).thenReturn(mockFs);
+ when(mockFs.getScheme()).thenReturn("hdfs");
+ when(mockFs.getUri()).thenReturn(new URI("hdfs", "somehost:9000", null, null, null));
+ mockStatic(System.class);
+ when(System.currentTimeMillis()).thenReturn(Long.MAX_VALUE);
+ when(functionObj.getFunctionName()).thenReturn("someFunctionName");
+ mockStatic(ReplCopyTask.class);
+ Task mock = mock(Task.class);
+ when(ReplCopyTask.getLoadCopyTask(any(ReplicationSpec.class), any(Path.class), any(Path.class),
+ any(HiveConf.class))).thenReturn(mock);
+
+ ResourceUri resourceUri = function.destinationResourceUri(new ResourceUri(ResourceType.JAR,
+ "hdfs://localhost:9000/user/someplace/ab.jar#e094828883"));
+
+ assertThat(resourceUri.getUri(),
+ is(equalTo(
+ "hdfs://somehost:9000/someBasePath/withADir/replicaDbName/somefunctionname/" + String
+ .valueOf(Long.MAX_VALUE) + "/ab.jar")));
+ }
+}
\ No newline at end of file
[14/36] hive git commit: HIVE-16618: Clean up javadoc from errors in
module hive-common [addendum] (Janos Gub via Zoltan Haindrich)
Posted by we...@apache.org.
HIVE-16618: Clean up javadoc from errors in module hive-common [addendum] (Janos Gub via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0155a34c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0155a34c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0155a34c
Branch: refs/heads/hive-14535
Commit: 0155a34cd0413b82b9f82aea2bf8ef9198377be3
Parents: 9f976ff
Author: Janos Gub <gu...@gmail.com>
Authored: Fri Jun 2 16:28:58 2017 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Fri Jun 2 16:29:05 2017 +0200
----------------------------------------------------------------------
.../hive/http/Log4j2ConfiguratorServlet.java | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0155a34c/common/src/java/org/apache/hive/http/Log4j2ConfiguratorServlet.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/Log4j2ConfiguratorServlet.java b/common/src/java/org/apache/hive/http/Log4j2ConfiguratorServlet.java
index 8042f21..6d07fb4 100644
--- a/common/src/java/org/apache/hive/http/Log4j2ConfiguratorServlet.java
+++ b/common/src/java/org/apache/hive/http/Log4j2ConfiguratorServlet.java
@@ -38,13 +38,13 @@ import org.slf4j.LoggerFactory;
/**
* A servlet to configure log4j2.
- * <br/>
+ * <br>
* HTTP GET returns all loggers and it's log level in JSON formatted response.
- * <br/>
+ * <br>
* HTTP POST is used for configuring the loggers. POST data should be in the same format as GET's response.
* To configure (add/update existing loggers), use HTTP POST with logger names and level in the following JSON format.
*
- * <br/>
+ * <br>
* <pre>
* <code>{
* "loggers": [ {
@@ -57,11 +57,12 @@ import org.slf4j.LoggerFactory;
* "logger" : "org.apache.zookeeper.server.NIOServerCnxn",
* "level" : "WARN"
* }]
- * }<code>
+ * }</code>
* </pre>
*
- * <br/>
+ * <br>
* Example usage:
+ * <ul>
* <li>
* Returns all loggers with levels in JSON format:
* <pre>
@@ -98,14 +99,17 @@ import org.slf4j.LoggerFactory;
* { "logger" : "org.apache.orc", "level" : "INFO" } ] }' http://hostame:port/conflog
* </pre>
* </li>
- * <br/>
+ * </ul>
+ * <br>
* Response Status Codes:
- * <br/>
+ * <br>
+ * <ul>
* <li>200 - OK : If the POST data is valid and if the request succeeds or if GET request succeeds.</li>
* <li>401 - UNAUTHORIZED : If the user does not have privileges to access instrumentation servlets.
* Refer <code>hadoop.security.instrumentation.requires.admin</code> config for more info.</li>
* <li>400 - BAD_REQUEST : If the POST data is not a valid JSON.</li>
* <li>500 - INTERNAL_SERVER_ERROR : If GET requests throws any IOException during JSON output generation.</li>
+ * </ul>
*/
public class Log4j2ConfiguratorServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
[32/36] hive git commit: HIVE-16809 : Improve filter condition for
correlated subqueries (Vineet Garg via Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16809 : Improve filter condition for correlated subqueries (Vineet Garg via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/04bb1ca2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/04bb1ca2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/04bb1ca2
Branch: refs/heads/hive-14535
Commit: 04bb1ca27c3bf3733466134d285d507013689f06
Parents: b754cb7
Author: Vineet Garg <vg...@apache.com>
Authored: Sat Jun 3 13:41:15 2017 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Jun 3 13:41:15 2017 -0700
----------------------------------------------------------------------
.../calcite/rules/HiveRelDecorrelator.java | 46 +++++++++++++++++---
1 file changed, 40 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/04bb1ca2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
index 05d2b28..4c99932 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
@@ -1232,7 +1232,8 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
// Replace the filter expression to reference output of the join
// Map filter to the new filter over join
- relBuilder.push(frame.r).filter(decorrelateExpr(rel.getCondition()));
+ relBuilder.push(frame.r).filter(
+ simplifyComparison(decorrelateExpr(rel.getCondition())));
// Filter does not change the input ordering.
// Filter rel does not permute the input.
@@ -1243,11 +1244,44 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
}
}
- /**
- * Rewrite LogicalFilter.
- *
- * @param rel the filter rel to rewrite
- */
+ private RexNode simplifyComparison(RexNode op) {
+ switch(op.getKind()) {
+ case EQUALS:
+ case GREATER_THAN:
+ case GREATER_THAN_OR_EQUAL:
+ case LESS_THAN:
+ case LESS_THAN_OR_EQUAL:
+ case NOT_EQUALS:
+ RexCall e = (RexCall) op;
+ final List<RexNode> operands = new ArrayList<>(e.operands);
+
+ // Simplify "x <op> x"
+ final RexNode o0 = operands.get(0);
+ final RexNode o1 = operands.get(1);
+ // this should only be called when we are creating filter (decorrelate filter)
+ // since in that case null/unknown is treated as false we don't care about
+ // nullability of operands and will always rewrite op=op to op is not null
+ if (RexUtil.eq(o0, o1) )
+ switch (e.getKind()) {
+ case EQUALS:
+ case GREATER_THAN_OR_EQUAL:
+ case LESS_THAN_OR_EQUAL:
+ // "x = x" simplifies to "x is not null" (similarly <= and >=)
+ return rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, o0);
+ default:
+ // "x != x" simplifies to "false" (similarly < and >)
+ return rexBuilder.makeLiteral(false);
+ }
+ }
+ return op;
+ }
+
+
+ /**
+ * Rewrite LogicalFilter.
+ *
+ * @param rel the filter rel to rewrite
+ */
public Frame decorrelateRel(LogicalFilter rel) {
//
// Rewrite logic:
[28/36] hive git commit: HIVE-16744 : LLAP index update may be broken
after ORC switch (Sergey Shelukhin,
reviewed by Prasanth Jayachandran) ADDENDUM
Posted by we...@apache.org.
HIVE-16744 : LLAP index update may be broken after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran) ADDENDUM
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a691696
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a691696
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a691696
Branch: refs/heads/hive-14535
Commit: 9a6916960cf38a3e85068f82d4fc370020e43314
Parents: 7916965
Author: sergey <se...@apache.org>
Authored: Fri Jun 2 16:01:09 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Jun 2 16:01:09 2017 -0700
----------------------------------------------------------------------
.../clientpositive/llap/orc_llap_counters.q.out | 2 +-
.../clientpositive/llap/orc_ppd_basic.q.out | 230 -------------------
.../llap/orc_ppd_schema_evol_3a.q.out | 6 +-
3 files changed, 4 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9a691696/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
index 1b7f7e3..8af84dc 100644
--- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out
@@ -697,7 +697,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 5980
+ HDFS_BYTES_READ: 6786
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
http://git-wip-us.apache.org/repos/asf/hive/blob/9a691696/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out
index a5ef454..5382c42 100644
--- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out
@@ -203,11 +203,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 16673
- HDFS_BYTES_WRITTEN: 104
- HDFS_READ_OPS: 5
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -226,11 +221,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
RECORDS_OUT_0: 1
@@ -240,11 +230,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 1344
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 4
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -267,11 +252,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -292,11 +272,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -317,11 +292,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -342,11 +312,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -367,11 +332,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -392,11 +352,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -417,11 +372,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 104
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -442,11 +392,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -467,11 +412,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 104
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -492,11 +432,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -517,11 +452,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -542,11 +472,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 103
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -567,11 +492,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
RECORDS_OUT_0: 1
@@ -581,11 +501,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
RECORDS_OUT_0: 1
@@ -595,11 +510,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -615,11 +525,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -635,11 +540,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
RECORDS_OUT_0: 1
@@ -649,11 +549,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 5980
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 4
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -677,11 +572,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -702,11 +592,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -727,11 +612,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
RECORDS_OUT_0: 1
@@ -741,11 +621,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 104
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -766,11 +641,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -786,11 +656,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -811,11 +676,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -836,11 +696,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -861,11 +716,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -886,11 +736,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -911,11 +756,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -936,11 +776,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 102
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -961,11 +796,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -986,11 +816,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1011,11 +836,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1031,11 +851,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1051,11 +866,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1071,11 +881,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1096,11 +901,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1121,11 +921,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1146,11 +941,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1171,11 +961,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 5201
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 4
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1199,11 +984,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1224,11 +1004,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
@@ -1249,11 +1024,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 0
- HDFS_BYTES_WRITTEN: 101
- HDFS_READ_OPS: 2
- HDFS_LARGE_READ_OPS: 0
- HDFS_WRITE_OPS: 2
Stage-1 HIVE COUNTERS:
CREATED_FILES: 1
DESERIALIZE_ERRORS: 0
http://git-wip-us.apache.org/repos/asf/hive/blob/9a691696/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
index 7a73c0b..b799527 100644
--- a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
@@ -890,7 +890,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 5201
+ HDFS_BYTES_READ: 5287
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
@@ -1027,7 +1027,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 4388
+ HDFS_BYTES_READ: 4577
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
@@ -1276,7 +1276,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@orc_ppd
#### A masked pattern was here ####
Stage-1 FILE SYSTEM COUNTERS:
- HDFS_BYTES_READ: 2472
+ HDFS_BYTES_READ: 2780
HDFS_BYTES_WRITTEN: 101
HDFS_READ_OPS: 4
HDFS_LARGE_READ_OPS: 0
[11/36] hive git commit: HIVE-16654: Optimize a combination of avg(),
sum(), count(distinct) etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16654: Optimize a combination of avg(), sum(), count(distinct) etc (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b560f492
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b560f492
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b560f492
Branch: refs/heads/hive-14535
Commit: b560f492ea08a9a83005f166e8ed5ef0fda6592d
Parents: d7ab32f
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Wed May 31 18:17:53 2017 -0700
Committer: Pengcheng Xiong <px...@hortonworks.com>
Committed: Wed May 31 18:17:53 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../test/resources/testconfiguration.properties | 1 +
.../apache/hadoop/hive/ql/exec/Operator.java | 10 +-
.../ql/optimizer/CountDistinctRewriteProc.java | 504 ++++++++
.../hadoop/hive/ql/optimizer/Optimizer.java | 4 +
.../apache/hadoop/hive/ql/plan/GroupByDesc.java | 15 +
.../queries/clientpositive/count_dist_rewrite.q | 65 +
.../clientpositive/count_dist_rewrite.q.out | 1151 +++++++++++++++++
.../clientpositive/groupby_sort_11.q.out | 39 +-
.../llap/count_dist_rewrite.q.out | 1169 ++++++++++++++++++
.../results/clientpositive/nullgroup4.q.out | 41 +-
.../results/clientpositive/perf/query16.q.out | 318 ++---
.../results/clientpositive/perf/query28.q.out | 58 +-
.../results/clientpositive/perf/query94.q.out | 318 ++---
.../results/clientpositive/perf/query95.q.out | 302 ++---
.../clientpositive/spark/nullgroup4.q.out | 31 +-
.../test/results/clientpositive/udf_count.q.out | 39 +-
.../clientpositive/vector_empty_where.q.out | 300 ++++-
18 files changed, 3820 insertions(+), 547 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5344f36..176d36f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1569,6 +1569,8 @@ public class HiveConf extends Configuration {
"Whether to transform OR clauses in Filter operators into IN clauses"),
HIVEPOINTLOOKUPOPTIMIZERMIN("hive.optimize.point.lookup.min", 31,
"Minimum number of OR clauses needed to transform into IN clauses"),
+ HIVECOUNTDISTINCTOPTIMIZER("hive.optimize.countdistinct", true,
+ "Whether to transform count distinct into two stages"),
HIVEPARTITIONCOLUMNSEPARATOR("hive.optimize.partition.columns.separate", true,
"Extract partition columns from IN clauses"),
// Constant propagation optimizer
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f4a53df..e613374 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -130,6 +130,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
constprog_semijoin.q,\
correlationoptimizer1.q,\
count.q,\
+ count_dist_rewrite.q,\
create_merge_compressed.q,\
cross_join.q,\
cross_product_check_1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index ffa5f41..3656842 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -1164,8 +1164,16 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
@SuppressWarnings("unchecked")
T descClone = (T)conf.clone();
// also clone the colExprMap by default
+ // we need a deep copy
+ ArrayList<ColumnInfo> colInfos = new ArrayList<>();
+ colInfos.addAll(getSchema().getSignature());
+ Map<String, ExprNodeDesc> map = null;
+ if (getColumnExprMap() != null) {
+ map = new HashMap<>();
+ map.putAll(getColumnExprMap());
+ }
Operator<? extends OperatorDesc> ret = OperatorFactory.getAndMakeChild(
- cContext, descClone, getSchema(), getColumnExprMap(), parentClones);
+ cContext, descClone, new RowSchema(colInfos), map, parentClones);
return ret;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/java/org/apache/hadoop/hive/ql/optimizer/CountDistinctRewriteProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/CountDistinctRewriteProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/CountDistinctRewriteProc.java
new file mode 100644
index 0000000..6450cb3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/CountDistinctRewriteProc.java
@@ -0,0 +1,504 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.GenericUDAFInfo;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+
+/**
+ * Queries of form : select max(c), count(distinct c) from T; generates a plan
+ * of form TS->mGBy->RS->rGBy->FS This plan suffers from a problem that vertex
+ * containing rGBy->FS necessarily need to have 1 task. This limitation results
+ * in slow execution because that task gets all the data. This optimization if
+ * successful will rewrite above plan to mGby1-rs1-mGby2-mGby3-rs2-rGby1 This
+ * introduces extra vertex of mGby2-mGby3-rs2. Note this vertex can have
+ * multiple tasks and since we are doing aggregation, output of this must
+ * necessarily be smaller than its input, which results in much less data going
+ * in to original rGby->FS vertex, which continues to have single task. Also
+ * note on calcite tree we have HiveExpandDistinctAggregatesRule rule which does
+ * similar plan transformation but has different conditions which needs to be
+ * satisfied. Additionally, we don't do any costing here but this is possibly
+ * that this transformation may slow down query a bit since if data is small
+ * enough to fit in a single task of last reducer, injecting additional vertex
+ * in pipeline may make query slower. If this happens, users can use the
+ * configuration hive.optimize.countdistinct to turn it off.
+ */
+public class CountDistinctRewriteProc extends Transform {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CountDistinctRewriteProc.class
+ .getName());
+
+ public CountDistinctRewriteProc() {
+ }
+
+ @Override
+ public ParseContext transform(ParseContext pctx) throws SemanticException {
+
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ // process group-by pattern
+ opRules
+ .put(
+ new RuleRegExp("R1", GroupByOperator.getOperatorName() + "%"
+ + ReduceSinkOperator.getOperatorName() + "%" + GroupByOperator.getOperatorName()
+ + "%"), getCountDistinctProc(pctx));
+
+ // The dispatcher fires the processor corresponding to the closest matching
+ // rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, null);
+ GraphWalker ogw = new DefaultGraphWalker(disp);
+
+ // Create a list of topop nodes
+ List<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(pctx.getTopOps().values());
+ ogw.startWalking(topNodes, null);
+
+ return pctx;
+ }
+
+ private NodeProcessor getDefaultProc() {
+ return new NodeProcessor() {
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ return null;
+ }
+ };
+ }
+
+ private NodeProcessor getCountDistinctProc(ParseContext pctx) {
+ return new CountDistinctProcessor(pctx);
+ }
+
+ /**
+ * CountDistinctProcessor.
+ *
+ */
+ public class CountDistinctProcessor implements NodeProcessor {
+
+ protected ParseContext pGraphContext;
+
+ public CountDistinctProcessor(ParseContext pGraphContext) {
+ this.pGraphContext = pGraphContext;
+ }
+
+ // Position of distinct column in aggregator list of map Gby before rewrite.
+ int indexOfDist = -1;
+
+ // Check if we can process it or not
+ protected boolean checkCountDistinct(GroupByOperator mGby, ReduceSinkOperator rs,
+ GroupByOperator rGby) {
+ ArrayList<ExprNodeDesc> keys = mGby.getConf().getKeys();
+ if (!(mGby.getConf().getMode() == GroupByDesc.Mode.HASH
+ && !mGby.getConf().isGroupingSetsPresent() && rs.getConf().getKeyCols().size() == 1
+ && rs.getConf().getPartitionCols().size() == 0
+ && rs.getConf().getDistinctColumnIndices().size() == 1
+ && rGby.getConf().getMode() == GroupByDesc.Mode.MERGEPARTIAL && keys.size() == 1
+ && rGby.getConf().getKeys().size() == 0 && mGby.getConf().getOutputColumnNames().size() == mGby
+ .getConf().getAggregators().size() + 1)) {
+ return false;
+ }
+ for (int pos = 0; pos < mGby.getConf().getAggregators().size(); pos++) {
+ AggregationDesc aggr = mGby.getConf().getAggregators().get(pos);
+ if (aggr.getDistinct()) {
+ if (indexOfDist != -1 || !aggr.getGenericUDAFName().equalsIgnoreCase("count")) {
+ // there are 2 or more distincts, or distinct is not on count
+ // TODO: may be the same count(distinct key), count(distinct key)
+ // TODO: deal with duplicate count distinct key
+ return false;
+ }
+ indexOfDist = pos;
+ if (!(aggr.getParameters().size() == 1
+ && aggr.getParameters().get(0) instanceof ExprNodeColumnDesc && mGby.getConf()
+ .getKeys().get(0) instanceof ExprNodeColumnDesc)) {
+ return false;
+ } else {
+ ExprNodeColumnDesc agg = (ExprNodeColumnDesc) aggr.getParameters().get(0);
+ ExprNodeColumnDesc key = (ExprNodeColumnDesc) mGby.getConf().getKeys().get(0);
+ if (!agg.isSame(key)) {
+ return false;
+ }
+ }
+ }
+ }
+ if (indexOfDist == -1) {
+ return false;
+ }
+ // check if it is potential to trigger nullscan
+ if (pGraphContext.getConf().getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
+ for (TableScanOperator tsOp : pGraphContext.getTopOps().values()) {
+ List<Integer> colIDs = tsOp.getNeededColumnIDs();
+ TableScanDesc desc = tsOp.getConf();
+ boolean noColNeeded = (colIDs == null) || (colIDs.isEmpty());
+ // VC is still here and it will be pruned by column pruner
+ // boolean noVCneeded = (desc == null) || (desc.getVirtualCols() == null)
+ // || (desc.getVirtualCols().isEmpty());
+ boolean isSkipHF = desc.isNeedSkipHeaderFooters();
+ if (noColNeeded && !isSkipHF) {
+ // it is possible that nullscan can fire, we skip this rule.
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /*
+ * We will transform GB-RS-GBY to mGby1-rs1-mGby2-mGby3-rs2-rGby1
+ */
+ @SuppressWarnings("unchecked")
+ protected void processGroupBy(GroupByOperator mGby, ReduceSinkOperator rs, GroupByOperator rGby)
+ throws SemanticException, CloneNotSupportedException {
+ // remove count(distinct) in map-side gby
+ List<Operator<? extends OperatorDesc>> parents = mGby.getParentOperators();
+ List<Operator<? extends OperatorDesc>> children = rGby.getChildOperators();
+ mGby.removeParents();
+ rs.removeParents();
+ rGby.removeParents();
+
+ GroupByOperator mGby1 = genMapGroupby1(mGby, indexOfDist);
+ ReduceSinkOperator rs1 = genReducesink1(mGby1, rs, indexOfDist);
+ GroupByOperator mGby2 = genMapGroupby2(rs1, mGby);
+ GroupByOperator mGby3 = genMapGroupby3(mGby2, mGby);
+ ReduceSinkOperator rs2 = genReducesink2(mGby3, rs);
+ GroupByOperator rGby1 = genReduceGroupby(rs2, rGby, indexOfDist);
+ for (Operator<? extends OperatorDesc> parent : parents) {
+ OperatorFactory.makeChild(parent, mGby1);
+ }
+ OperatorFactory.makeChild(mGby1, rs1);
+ OperatorFactory.makeChild(rs1, mGby2);
+ OperatorFactory.makeChild(mGby2, mGby3);
+ OperatorFactory.makeChild(mGby3, rs2);
+ OperatorFactory.makeChild(rs2, rGby1);
+ for (Operator<? extends OperatorDesc> child : children) {
+ child.removeParents();
+ OperatorFactory.makeChild(rGby1, child);
+ }
+ }
+
+ // mGby1 ---already contains group by key, we need to remove distinct column
+ private GroupByOperator genMapGroupby1(Operator<? extends OperatorDesc> mGby, int indexOfDist)
+ throws CloneNotSupportedException {
+ GroupByOperator mGby1 = (GroupByOperator) mGby.clone();
+ // distinct is at lost position.
+ String fieldString = mGby1.getConf().getOutputColumnNames().get(indexOfDist + 1);
+ mGby1.getColumnExprMap().remove(fieldString);
+ mGby1.getConf().getOutputColumnNames().remove(indexOfDist + 1);
+ mGby1.getConf().getAggregators().remove(indexOfDist);
+ mGby1.getConf().setDistinct(false);
+ mGby1.getSchema().getColumnNames().remove(indexOfDist + 1);
+ mGby1.getSchema().getSignature().remove(indexOfDist + 1);
+ return mGby1;
+ }
+
+ // rs1 --- remove distinctColIndices, set #reducer as -1, reset keys,
+ // values, colexpmap and rowschema
+ private ReduceSinkOperator genReducesink1(GroupByOperator mGby1,
+ Operator<? extends OperatorDesc> rs, int indexOfDist) throws CloneNotSupportedException,
+ SemanticException {
+ ReduceSinkOperator rs1 = (ReduceSinkOperator) rs.clone();
+ Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
+ ArrayList<String> outputKeyColumnNames = new ArrayList<String>();
+ ArrayList<String> outputValueColumnNames = new ArrayList<String>();
+ ArrayList<ExprNodeDesc> reduceKeys = new ArrayList<ExprNodeDesc>();
+ ArrayList<ExprNodeDesc> reduceValues = new ArrayList<ExprNodeDesc>();
+ List<String> internalNames = new ArrayList<>();
+ for (int index = 0; index < mGby1.getSchema().getSignature().size(); index++) {
+ ColumnInfo paraExprInfo = mGby1.getSchema().getSignature().get(index);
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ ExprNodeColumnDesc exprDesc = new ExprNodeColumnDesc(paraExprInfo.getType(),
+ paraExpression, paraExprInfo.getTabAlias(), paraExprInfo.getIsVirtualCol());
+ // index==0 means this is key
+ if (index == 0) {
+ reduceKeys.add(exprDesc);
+ String outputColName = SemanticAnalyzer.getColumnInternalName(index);
+ outputKeyColumnNames.add(outputColName);
+ String internalName = Utilities.ReduceField.KEY.toString() + "." + outputColName;
+ colExprMap.put(internalName, exprDesc);
+ internalNames.add(internalName);
+ } else {
+ reduceValues.add(exprDesc);
+ String outputColName = SemanticAnalyzer.getColumnInternalName(index - 1);
+ outputValueColumnNames.add(outputColName);
+ String internalName = Utilities.ReduceField.VALUE.toString() + "." + outputColName;
+ colExprMap.put(internalName, exprDesc);
+ internalNames.add(internalName);
+ }
+ }
+ List<List<Integer>> distinctColIndices = new ArrayList<>();
+ rs1.setConf(PlanUtils.getReduceSinkDesc(reduceKeys, 1, reduceValues, distinctColIndices,
+ outputKeyColumnNames, outputValueColumnNames, true, -1, 1, -1,
+ AcidUtils.Operation.NOT_ACID));
+ rs1.setColumnExprMap(colExprMap);
+
+ rs1.getSchema().getColumnNames().remove(indexOfDist + 1);
+ rs1.getSchema().getSignature().remove(indexOfDist + 1);
+ // KEY._col0:0._col0 => KEY._col0
+
+ for (int i = 0; i < rs1.getSchema().getSignature().size(); i++) {
+ rs1.getSchema().getSignature().get(i).setInternalName(internalNames.get(i));
+ rs1.getSchema().getColumnNames().set(i, internalNames.get(i));
+ }
+ return rs1;
+ }
+
+ // mGby2 ---already contains key, remove distinct and change all the others
+ private GroupByOperator genMapGroupby2(ReduceSinkOperator rs1,
+ Operator<? extends OperatorDesc> mGby) throws CloneNotSupportedException, SemanticException {
+ GroupByOperator mGby2 = (GroupByOperator) mGby.clone();
+ ArrayList<ColumnInfo> rowSchema = new ArrayList<>();
+ ArrayList<ExprNodeDesc> groupByKeys = new ArrayList<ExprNodeDesc>();
+ ArrayList<String> outputColumnNames = new ArrayList<String>();
+ Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
+
+ ColumnInfo exprInfo = rs1.getSchema().getSignature().get(0);
+ ExprNodeDesc key = new ExprNodeColumnDesc(exprInfo);
+ groupByKeys.add(key);
+ String field = SemanticAnalyzer.getColumnInternalName(0);
+ outputColumnNames.add(field);
+ ColumnInfo oColInfo = new ColumnInfo(field, exprInfo.getType(), "", false);
+ colExprMap.put(field, key);
+ rowSchema.add(oColInfo);
+
+ ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
+ for (int index = 0; index < mGby2.getConf().getAggregators().size(); index++) {
+ ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
+ if (index != indexOfDist) {
+ AggregationDesc desc = mGby2.getConf().getAggregators().get(index);
+ ColumnInfo paraExprInfo = null;
+ // for example, original it is max 0, dist 1, min 2
+ // rs1's schema is key 0, max 1, min 2
+ if (index < indexOfDist) {
+ paraExprInfo = rs1.getSchema().getSignature().get(index + 1);
+ } else {
+ paraExprInfo = rs1.getSchema().getSignature().get(index);
+ }
+
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression,
+ paraExprInfo.getTabAlias(), paraExprInfo.getIsVirtualCol()));
+
+ // for all the other aggregations, we set the mode to PARTIAL2
+ Mode amode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.PARTIAL2, false);
+ GenericUDAFEvaluator genericUDAFEvaluator = desc.getGenericUDAFEvaluator();
+ GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(genericUDAFEvaluator, amode,
+ aggParameters);
+ aggregations.add(new AggregationDesc(desc.getGenericUDAFName(),
+ udaf.genericUDAFEvaluator, udaf.convertedParameters, false, amode));
+ String f = SemanticAnalyzer.getColumnInternalName(aggregations.size());
+ outputColumnNames.add(f);
+ rowSchema.add(new ColumnInfo(f, udaf.returnType, "", false));
+ }
+ }
+ mGby2.getConf().setMode(GroupByDesc.Mode.PARTIAL2);
+ mGby2.getConf().setOutputColumnNames(outputColumnNames);
+ mGby2.getConf().getKeys().clear();
+ mGby2.getConf().getKeys().addAll(groupByKeys);
+ mGby2.getConf().getAggregators().clear();
+ mGby2.getConf().getAggregators().addAll(aggregations);
+ mGby2.getConf().setDistinct(false);
+ mGby2.setSchema(new RowSchema(rowSchema));
+ mGby2.setColumnExprMap(colExprMap);
+ return mGby2;
+ }
+
+ // mGby3 is a follow up of mGby2. Here we start to count(key).
+ private GroupByOperator genMapGroupby3(GroupByOperator mGby2,
+ Operator<? extends OperatorDesc> mGby) throws CloneNotSupportedException, SemanticException {
+ GroupByOperator mGby3 = (GroupByOperator) mGby.clone();
+ ArrayList<ColumnInfo> rowSchema = new ArrayList<>();
+ ArrayList<String> outputColumnNames = new ArrayList<String>();
+ Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
+
+ // exprInfo is the key
+ ArrayList<AggregationDesc> aggregations = new ArrayList<AggregationDesc>();
+ for (int index = 0; index <= mGby2.getConf().getAggregators().size(); index++) {
+ if (index == indexOfDist) {
+ ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
+ // add count(KEY._col0) to replace distinct
+ ColumnInfo paraExprInfo = mGby2.getSchema().getSignature().get(0);
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression,
+ paraExprInfo.getTabAlias(), paraExprInfo.getIsVirtualCol()));
+ Mode amode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.HASH, false);
+ GenericUDAFEvaluator genericUDAFEvaluator = SemanticAnalyzer.getGenericUDAFEvaluator(
+ "count", aggParameters, null, false, false);
+ assert (genericUDAFEvaluator != null);
+ GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(genericUDAFEvaluator, amode,
+ aggParameters);
+ AggregationDesc newDesc = new AggregationDesc("count", udaf.genericUDAFEvaluator,
+ udaf.convertedParameters, false, amode);
+ String f = SemanticAnalyzer.getColumnInternalName(aggregations.size());
+ aggregations.add(newDesc);
+ outputColumnNames.add(f);
+ rowSchema.add(new ColumnInfo(f, udaf.returnType, "", false));
+ }
+ if (index == mGby2.getConf().getAggregators().size()) {
+ break;
+ }
+ ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
+ AggregationDesc desc = mGby2.getConf().getAggregators().get(index);
+ ColumnInfo paraExprInfo = null;
+ // for example, original it is max 0, dist 1, min 2
+ // rs1's schema is key 0, max 1, min 2
+ paraExprInfo = mGby2.getSchema().getSignature().get(index + 1);
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression,
+ paraExprInfo.getTabAlias(), paraExprInfo.getIsVirtualCol()));
+
+ // for all the other aggregations, we set the mode to PARTIAL2
+ Mode amode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.PARTIAL2, false);
+ GenericUDAFEvaluator genericUDAFEvaluator = desc.getGenericUDAFEvaluator();
+ GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(genericUDAFEvaluator, amode,
+ aggParameters);
+ String f = SemanticAnalyzer.getColumnInternalName(aggregations.size());
+ aggregations.add(new AggregationDesc(desc.getGenericUDAFName(), udaf.genericUDAFEvaluator,
+ udaf.convertedParameters, false, amode));
+ outputColumnNames.add(f);
+ rowSchema.add(new ColumnInfo(f, udaf.returnType, "", false));
+ }
+ mGby3.getConf().setMode(GroupByDesc.Mode.PARTIAL2);
+ mGby3.getConf().setOutputColumnNames(outputColumnNames);
+ mGby3.getConf().getKeys().clear();
+ mGby3.getConf().getAggregators().clear();
+ mGby3.getConf().getAggregators().addAll(aggregations);
+ mGby3.getConf().setDistinct(false);
+ mGby3.setSchema(new RowSchema(rowSchema));
+ mGby3.setColumnExprMap(colExprMap);
+ return mGby3;
+ }
+
+ // #reducer is already 1
+ private ReduceSinkOperator genReducesink2(GroupByOperator mGby2,
+ Operator<? extends OperatorDesc> rs) throws SemanticException, CloneNotSupportedException {
+ ReduceSinkOperator rs2 = (ReduceSinkOperator) rs.clone();
+ Map<String, ExprNodeDesc> colExprMap = new HashMap<>();
+
+ ArrayList<String> outputKeyColumnNames = new ArrayList<String>();
+ ArrayList<String> outputValueColumnNames = new ArrayList<String>();
+ ArrayList<ExprNodeDesc> reduceValues = new ArrayList<ExprNodeDesc>();
+ for (int index = 0; index < mGby2.getSchema().getSignature().size(); index++) {
+ ColumnInfo paraExprInfo = mGby2.getSchema().getSignature().get(index);
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ ExprNodeColumnDesc exprDesc = new ExprNodeColumnDesc(paraExprInfo.getType(),
+ paraExpression, paraExprInfo.getTabAlias(), paraExprInfo.getIsVirtualCol());
+ reduceValues.add(exprDesc);
+ String outputColName = SemanticAnalyzer.getColumnInternalName(index);
+ outputValueColumnNames.add(outputColName);
+ String internalName = Utilities.ReduceField.VALUE.toString() + "." + outputColName;
+ colExprMap.put(internalName, exprDesc);
+ }
+ List<List<Integer>> distinctColIndices = new ArrayList<>();
+ ArrayList<ExprNodeDesc> reduceKeys = new ArrayList<>();
+ rs2.setConf(PlanUtils.getReduceSinkDesc(reduceKeys, 0, reduceValues, distinctColIndices,
+ outputKeyColumnNames, outputValueColumnNames, false, -1, 0, 1,
+ AcidUtils.Operation.NOT_ACID));
+ rs2.setColumnExprMap(colExprMap);
+ rs2.getSchema().getSignature().remove(0);
+ return rs2;
+ }
+
+ // replace the distinct with the count aggregation
+ private GroupByOperator genReduceGroupby(ReduceSinkOperator rs2,
+ Operator<? extends OperatorDesc> rGby, int indexOfDist) throws SemanticException,
+ CloneNotSupportedException {
+ GroupByOperator rGby1 = (GroupByOperator) rGby.clone();
+ ColumnInfo paraExprInfo = rs2.getSchema().getSignature().get(indexOfDist);
+ String paraExpression = paraExprInfo.getInternalName();
+ assert (paraExpression != null);
+ ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
+ aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(), paraExpression, paraExprInfo
+ .getTabAlias(), paraExprInfo.getIsVirtualCol()));
+ GenericUDAFEvaluator genericUDAFEvaluator = SemanticAnalyzer.getGenericUDAFEvaluator("count",
+ aggParameters, null, false, false);
+ assert (genericUDAFEvaluator != null);
+ Mode amode = SemanticAnalyzer.groupByDescModeToUDAFMode(GroupByDesc.Mode.MERGEPARTIAL, false);
+ GenericUDAFInfo udaf = SemanticAnalyzer.getGenericUDAFInfo(genericUDAFEvaluator, amode,
+ aggParameters);
+ AggregationDesc newDesc = new AggregationDesc("count", udaf.genericUDAFEvaluator,
+ udaf.convertedParameters, false, amode);
+ rGby1.getConf().getAggregators().set(indexOfDist, newDesc);
+ rGby1.getConf().setDistinct(false);
+ return rGby1;
+ }
+
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ GroupByOperator mGby = (GroupByOperator) stack.get(stack.size() - 3);
+ ReduceSinkOperator rs = (ReduceSinkOperator) stack.get(stack.size() - 2);
+ GroupByOperator rGby = (GroupByOperator) stack.get(stack.size() - 1);
+ if (checkCountDistinct(mGby, rs, rGby)) {
+ LOG.info("trigger count distinct rewrite");
+ try {
+ processGroupBy(mGby, rs, rGby);
+ } catch (CloneNotSupportedException e) {
+ throw new SemanticException(e.getMessage());
+ }
+ }
+ return null;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 7dace90..781e088 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -143,6 +143,10 @@ public class Optimizer {
HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_MAP_GROUPBY_SORT)) {
transformations.add(new GroupByOptimizer());
}
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVECOUNTDISTINCTOPTIMIZER)
+ && (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST) || isTezExecEngine)) {
+ transformations.add(new CountDistinctRewriteProc());
+ }
transformations.add(new ColumnPruner());
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_OPTIMIZE_SKEWJOIN_COMPILETIME)) {
if (!isTezExecEngine) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
index 38a9ef2..fe91ee7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
@@ -367,4 +367,19 @@ public class GroupByDesc extends AbstractOperatorDesc {
}
return new GroupByOperatorExplainVectorization(this, vectorDesc);
}
+
+ @Override
+ public Object clone() {
+ ArrayList<java.lang.String> outputColumnNames = new ArrayList<>();
+ outputColumnNames.addAll(this.outputColumnNames);
+ ArrayList<ExprNodeDesc> keys = new ArrayList<>();
+ keys.addAll(this.keys);
+ ArrayList<org.apache.hadoop.hive.ql.plan.AggregationDesc> aggregators = new ArrayList<>();
+ aggregators.addAll(this.aggregators);
+ List<Integer> listGroupingSets = new ArrayList<>();
+ listGroupingSets.addAll(this.listGroupingSets);
+ return new GroupByDesc(this.mode, outputColumnNames, keys, aggregators,
+ this.groupByMemoryUsage, this.memoryThreshold, listGroupingSets, this.groupingSetsPresent,
+ this.groupingSetPosition, this.isDistinct);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/queries/clientpositive/count_dist_rewrite.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/count_dist_rewrite.q b/ql/src/test/queries/clientpositive/count_dist_rewrite.q
new file mode 100644
index 0000000..0b1bc66
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/count_dist_rewrite.q
@@ -0,0 +1,65 @@
+explain select count(distinct key) from src;
+
+select count(distinct key) from src;
+
+explain select max(key), count(distinct key) B1_CNTD from src;
+
+select max(key), count(distinct key) B1_CNTD from src;
+
+explain select max(key), count(distinct key), min(key) from src;
+
+select max(key), count(distinct key), min(key) from src;
+
+explain select max(key), count(distinct key), min(key), avg(key) from src;
+
+select max(key), count(distinct key), min(key), avg(key) from src;
+
+explain select count(1), count(distinct key) from src;
+
+select count(1), count(distinct key) from src;
+
+explain select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src;
+
+select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src;
+
+explain select count(1), count(distinct key), cast(STDDEV(key) as int) from src;
+select count(1), count(distinct key), cast(STDDEV(key) as int) from src;
+select count(distinct key), count(1), cast(STDDEV(key) as int) from src;
+
+explain SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src;
+
+SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src;
+
+explain select max(key), count(distinct key), min(key), avg(key) from src group by value;
+
+select max(key), count(distinct key), min(key), avg(key) from src group by value;
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/count_dist_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/count_dist_rewrite.q.out b/ql/src/test/results/clientpositive/count_dist_rewrite.q.out
new file mode 100644
index 0000000..ceda918
--- /dev/null
+++ b/ql/src/test/results/clientpositive/count_dist_rewrite.q.out
@@ -0,0 +1,1151 @@
+PREHOOK: query: explain select count(distinct key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(distinct key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(distinct key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(distinct key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+309
+PREHOOK: query: explain select max(key), count(distinct key) B1_CNTD from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key) B1_CNTD from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key) B1_CNTD from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key) B1_CNTD from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309
+PREHOOK: query: explain select max(key), count(distinct key), min(key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(key), min(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col3 (type: string)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), min(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(_col1), count(_col0), min(_col2)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1), min(VALUE._col2)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309 0
+PREHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(key), min(key), avg(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3, _col4
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: struct<count:bigint,sum:double,input:string>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), min(VALUE._col1), avg(VALUE._col2)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(_col1), count(_col0), min(_col2), avg(_col3)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 1148 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 1148 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: struct<count:bigint,sum:double,input:string>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(VALUE._col1), min(VALUE._col2), avg(VALUE._col3)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 1148 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 1148 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+98 309 0 260.182
+PREHOOK: query: explain select count(1), count(distinct key) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(distinct key) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1)
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1), count(_col0)
+ mode: partial2
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(distinct key) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(distinct key) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 309
+PREHOOK: query: explain select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(), count(key), max(value), max(key)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint), _col4 (type: string), _col5 (type: string)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), max(VALUE._col2), max(VALUE._col3)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1), count(_col2), count(_col0), max(_col3), max(_col4)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: bigint), _col3 (type: string), _col4 (type: string)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), max(VALUE._col3), max(VALUE._col4)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 576 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select
+ count(*) as total,
+ count(key) as not_null_total,
+ count(distinct key) as unique_days,
+ max(value) as max_ss_store_sk,
+ max(key) as max_ss_promo_sk
+from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 500 309 val_98 98
+PREHOOK: query: explain select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1), stddev(_col1)
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col3 (type: struct<count:bigint,sum:double,variance:double>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), stddev(VALUE._col1)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1), count(_col0), stddev(_col2)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2 (type: struct<count:bigint,sum:double,variance:double>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), stddev(VALUE._col2)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: bigint), _col1 (type: bigint), UDFToInteger(_col2) (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 176 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(distinct key), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+500 309 142
+PREHOOK: query: select count(distinct key), count(1), cast(STDDEV(key) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(distinct key), count(1), cast(STDDEV(key) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+309 500 142
+PREHOOK: query: explain SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: substr(value, 5) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col0), avg(_col0), max(_col0), min(_col0), std(_col0), stddev_samp(_col0), variance(_col0), var_samp(_col0)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: struct<count:bigint,sum:double,input:string>), _col4 (type: string), _col5 (type: string), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>), _col9 (type: struct<count:bigint,sum:double,variance:double>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), avg(VALUE._col1), max(VALUE._col2), min(VALUE._col3), std(VALUE._col4), stddev_samp(VALUE._col5), variance(VALUE._col6), var_samp(VALUE._col7)
+ keys: KEY._col0 (type: string)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col1), avg(_col2), count(_col0), max(_col3), min(_col4), std(_col5), stddev_samp(_col6), variance(_col7), var_samp(_col8)
+ mode: partial2
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 1392 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 1392 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: double), _col1 (type: struct<count:bigint,sum:double,input:string>), _col2 (type: bigint), _col3 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: struct<count:bigint,sum:double,variance:double>), _col7 (type: struct<count:bigint,sum:double,variance:double>), _col8 (type: struct<count:bigint,sum:double,variance:double>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), avg(VALUE._col1), count(VALUE._col2), max(VALUE._col3), min(VALUE._col4), std(VALUE._col5), stddev_samp(VALUE._col6), variance(VALUE._col7), var_samp(VALUE._col8)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 1392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: double), _col2 (type: bigint), _col3 (type: string), _col4 (type: string), UDFToInteger(_col5) (type: int), UDFToInteger(_col6) (type: int), UDFToInteger(_col7) (type: int), UDFToInteger(_col8) (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 1 Data size: 1392 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 1392 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ sum(substr(src.value,5)),
+ avg(substr(src.value,5)),
+ count(DISTINCT substr(src.value,5)),
+ max(substr(src.value,5)),
+ min(substr(src.value,5)),
+ cast(std(substr(src.value,5)) as int),
+ cast(stddev_samp(substr(src.value,5)) as int),
+ cast(variance(substr(src.value,5)) as int),
+ cast(var_samp(substr(src.value,5)) as int) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+130091.0 260.182 309 98 0 142 143 20428 20469
+PREHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src group by value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select max(key), count(distinct key), min(key), avg(key) from src group by value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: max(key), count(DISTINCT key), min(key), avg(key)
+ keys: value (type: string), key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string), _col4 (type: string), _col5 (type: struct<count:bigint,sum:double,input:string>)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: max(VALUE._col0), count(DISTINCT KEY._col1:0._col0), min(VALUE._col2), avg(VALUE._col3)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type: string), _col4 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src group by value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select max(key), count(distinct key), min(key), avg(key) from src group by value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0 1 0 0.0
+10 1 10 10.0
+100 1 100 100.0
+103 1 103 103.0
+104 1 104 104.0
+105 1 105 105.0
+11 1 11 11.0
+111 1 111 111.0
+113 1 113 113.0
+114 1 114 114.0
+116 1 116 116.0
+118 1 118 118.0
+119 1 119 119.0
+12 1 12 12.0
+120 1 120 120.0
+125 1 125 125.0
+126 1 126 126.0
+128 1 128 128.0
+129 1 129 129.0
+131 1 131 131.0
+133 1 133 133.0
+134 1 134 134.0
+136 1 136 136.0
+137 1 137 137.0
+138 1 138 138.0
+143 1 143 143.0
+145 1 145 145.0
+146 1 146 146.0
+149 1 149 149.0
+15 1 15 15.0
+150 1 150 150.0
+152 1 152 152.0
+153 1 153 153.0
+155 1 155 155.0
+156 1 156 156.0
+157 1 157 157.0
+158 1 158 158.0
+160 1 160 160.0
+162 1 162 162.0
+163 1 163 163.0
+164 1 164 164.0
+165 1 165 165.0
+166 1 166 166.0
+167 1 167 167.0
+168 1 168 168.0
+169 1 169 169.0
+17 1 17 17.0
+170 1 170 170.0
+172 1 172 172.0
+174 1 174 174.0
+175 1 175 175.0
+176 1 176 176.0
+177 1 177 177.0
+178 1 178 178.0
+179 1 179 179.0
+18 1 18 18.0
+180 1 180 180.0
+181 1 181 181.0
+183 1 183 183.0
+186 1 186 186.0
+187 1 187 187.0
+189 1 189 189.0
+19 1 19 19.0
+190 1 190 190.0
+191 1 191 191.0
+192 1 192 192.0
+193 1 193 193.0
+194 1 194 194.0
+195 1 195 195.0
+196 1 196 196.0
+197 1 197 197.0
+199 1 199 199.0
+2 1 2 2.0
+20 1 20 20.0
+200 1 200 200.0
+201 1 201 201.0
+202 1 202 202.0
+203 1 203 203.0
+205 1 205 205.0
+207 1 207 207.0
+208 1 208 208.0
+209 1 209 209.0
+213 1 213 213.0
+214 1 214 214.0
+216 1 216 216.0
+217 1 217 217.0
+218 1 218 218.0
+219 1 219 219.0
+221 1 221 221.0
+222 1 222 222.0
+223 1 223 223.0
+224 1 224 224.0
+226 1 226 226.0
+228 1 228 228.0
+229 1 229 229.0
+230 1 230 230.0
+233 1 233 233.0
+235 1 235 235.0
+237 1 237 237.0
+238 1 238 238.0
+239 1 239 239.0
+24 1 24 24.0
+241 1 241 241.0
+242 1 242 242.0
+244 1 244 244.0
+247 1 247 247.0
+248 1 248 248.0
+249 1 249 249.0
+252 1 252 252.0
+255 1 255 255.0
+256 1 256 256.0
+257 1 257 257.0
+258 1 258 258.0
+26 1 26 26.0
+260 1 260 260.0
+262 1 262 262.0
+263 1 263 263.0
+265 1 265 265.0
+266 1 266 266.0
+27 1 27 27.0
+272 1 272 272.0
+273 1 273 273.0
+274 1 274 274.0
+275 1 275 275.0
+277 1 277 277.0
+278 1 278 278.0
+28 1 28 28.0
+280 1 280 280.0
+281 1 281 281.0
+282 1 282 282.0
+283 1 283 283.0
+284 1 284 284.0
+285 1 285 285.0
+286 1 286 286.0
+287 1 287 287.0
+288 1 288 288.0
+289 1 289 289.0
+291 1 291 291.0
+292 1 292 292.0
+296 1 296 296.0
+298 1 298 298.0
+30 1 30 30.0
+302 1 302 302.0
+305 1 305 305.0
+306 1 306 306.0
+307 1 307 307.0
+308 1 308 308.0
+309 1 309 309.0
+310 1 310 310.0
+311 1 311 311.0
+315 1 315 315.0
+316 1 316 316.0
+317 1 317 317.0
+318 1 318 318.0
+321 1 321 321.0
+322 1 322 322.0
+323 1 323 323.0
+325 1 325 325.0
+327 1 327 327.0
+33 1 33 33.0
+331 1 331 331.0
+332 1 332 332.0
+333 1 333 333.0
+335 1 335 335.0
+336 1 336 336.0
+338 1 338 338.0
+339 1 339 339.0
+34 1 34 34.0
+341 1 341 341.0
+342 1 342 342.0
+344 1 344 344.0
+345 1 345 345.0
+348 1 348 348.0
+35 1 35 35.0
+351 1 351 351.0
+353 1 353 353.0
+356 1 356 356.0
+360 1 360 360.0
+362 1 362 362.0
+364 1 364 364.0
+365 1 365 365.0
+366 1 366 366.0
+367 1 367 367.0
+368 1 368 368.0
+369 1 369 369.0
+37 1 37 37.0
+373 1 373 373.0
+374 1 374 374.0
+375 1 375 375.0
+377 1 377 377.0
+378 1 378 378.0
+379 1 379 379.0
+382 1 382 382.0
+384 1 384 384.0
+386 1 386 386.0
+389 1 389 389.0
+392 1 392 392.0
+393 1 393 393.0
+394 1 394 394.0
+395 1 395 395.0
+396 1 396 396.0
+397 1 397 397.0
+399 1 399 399.0
+4 1 4 4.0
+400 1 400 400.0
+401 1 401 401.0
+402 1 402 402.0
+403 1 403 403.0
+404 1 404 404.0
+406 1 406 406.0
+407 1 407 407.0
+409 1 409 409.0
+41 1 41 41.0
+411 1 411 411.0
+413 1 413 413.0
+414 1 414 414.0
+417 1 417 417.0
+418 1 418 418.0
+419 1 419 419.0
+42 1 42 42.0
+421 1 421 421.0
+424 1 424 424.0
+427 1 427 427.0
+429 1 429 429.0
+43 1 43 43.0
+430 1 430 430.0
+431 1 431 431.0
+432 1 432 432.0
+435 1 435 435.0
+436 1 436 436.0
+437 1 437 437.0
+438 1 438 438.0
+439 1 439 439.0
+44 1 44 44.0
+443 1 443 443.0
+444 1 444 444.0
+446 1 446 446.0
+448 1 448 448.0
+449 1 449 449.0
+452 1 452 452.0
+453 1 453 453.0
+454 1 454 454.0
+455 1 455 455.0
+457 1 457 457.0
+458 1 458 458.0
+459 1 459 459.0
+460 1 460 460.0
+462 1 462 462.0
+463 1 463 463.0
+466 1 466 466.0
+467 1 467 467.0
+468 1 468 468.0
+469 1 469 469.0
+47 1 47 47.0
+470 1 470 470.0
+472 1 472 472.0
+475 1 475 475.0
+477 1 477 477.0
+478 1 478 478.0
+479 1 479 479.0
+480 1 480 480.0
+481 1 481 481.0
+482 1 482 482.0
+483 1 483 483.0
+484 1 484 484.0
+485 1 485 485.0
+487 1 487 487.0
+489 1 489 489.0
+490 1 490 490.0
+491 1 491 491.0
+492 1 492 492.0
+493 1 493 493.0
+494 1 494 494.0
+495 1 495 495.0
+496 1 496 496.0
+497 1 497 497.0
+498 1 498 498.0
+5 1 5 5.0
+51 1 51 51.0
+53 1 53 53.0
+54 1 54 54.0
+57 1 57 57.0
+58 1 58 58.0
+64 1 64 64.0
+65 1 65 65.0
+66 1 66 66.0
+67 1 67 67.0
+69 1 69 69.0
+70 1 70 70.0
+72 1 72 72.0
+74 1 74 74.0
+76 1 76 76.0
+77 1 77 77.0
+78 1 78 78.0
+8 1 8 8.0
+80 1 80 80.0
+82 1 82 82.0
+83 1 83 83.0
+84 1 84 84.0
+85 1 85 85.0
+86 1 86 86.0
+87 1 87 87.0
+9 1 9 9.0
+90 1 90 90.0
+92 1 92 92.0
+95 1 95 95.0
+96 1 96 96.0
+97 1 97 97.0
+98 1 98 98.0
http://git-wip-us.apache.org/repos/asf/hive/blob/b560f492/ql/src/test/results/clientpositive/groupby_sort_11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_sort_11.q.out b/ql/src/test/results/clientpositive/groupby_sort_11.q.out
index 2b3bf4a..fe6bbb3 100644
--- a/ql/src/test/results/clientpositive/groupby_sort_11.q.out
+++ b/ql/src/test/results/clientpositive/groupby_sort_11.q.out
@@ -292,7 +292,8 @@ POSTHOOK: query: EXPLAIN select count(distinct key+key) from T1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
Stage: Stage-1
@@ -306,24 +307,50 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(DISTINCT _col0)
keys: _col0 (type: double)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
sort order: +
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Group By Operator
- aggregations: count(DISTINCT KEY._col0:0._col0)
+ keys: KEY._col0 (type: double)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col0)
+ mode: partial2
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
[33/36] hive git commit: HIVE-16571 : HiveServer2: Prefer LIFO over
round-robin for Tez session reuse (Gopal Vijayaraghavan,
reviewed by Sergey Shelukhin)
Posted by we...@apache.org.
HIVE-16571 : HiveServer2: Prefer LIFO over round-robin for Tez session reuse (Gopal Vijayaraghavan, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bdacb105
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bdacb105
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bdacb105
Branch: refs/heads/hive-14535
Commit: bdacb10523cd8b46ed11649fee62e87388c26752
Parents: 04bb1ca
Author: sergey <se...@apache.org>
Authored: Mon Jun 5 14:31:13 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Jun 5 14:31:13 2017 -0700
----------------------------------------------------------------------
.../hive/ql/exec/tez/TezSessionPoolManager.java | 9 ++--
.../hive/ql/exec/tez/TestTezSessionPool.java | 43 ++++++++++++++++----
2 files changed, 39 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bdacb105/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
index b4d8ffa..dabca3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
@@ -23,12 +23,13 @@ import com.google.common.base.Preconditions;
import java.io.IOException;
import java.net.URISyntaxException;
-
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingDeque;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
@@ -77,7 +78,7 @@ public class TezSessionPoolManager {
private static final Random rdm = new Random();
private volatile SessionState initSessionState;
- private BlockingQueue<TezSessionPoolSession> defaultQueuePool;
+ private BlockingDeque<TezSessionPoolSession> defaultQueuePool;
/** Priority queue sorted by expiration time of live sessions that could be expired. */
private PriorityBlockingQueue<TezSessionPoolSession> expirationQueue;
@@ -204,7 +205,7 @@ public class TezSessionPoolManager {
int numSessions = conf.getIntVar(ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE);
int numSessionsTotal = numSessions * (defaultQueueList.length - emptyNames);
if (numSessionsTotal > 0) {
- defaultQueuePool = new ArrayBlockingQueue<TezSessionPoolSession>(numSessionsTotal);
+ defaultQueuePool = new LinkedBlockingDeque<TezSessionPoolSession>(numSessionsTotal);
}
numConcurrentLlapQueries = conf.getIntVar(ConfVars.HIVE_SERVER2_LLAP_CONCURRENT_QUERIES);
@@ -426,7 +427,7 @@ public class TezSessionPoolManager {
TezSessionPoolSession poolSession =
(TezSessionPoolSession) tezSessionState;
if (poolSession.returnAfterUse()) {
- defaultQueuePool.put(poolSession);
+ defaultQueuePool.putFirst(poolSession);
}
}
// non default session nothing changes. The user can continue to use the existing
http://git-wip-us.apache.org/repos/asf/hive/blob/bdacb105/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
index cbd2bfe..5a3eba3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
@@ -88,24 +88,49 @@ public class TestTezSessionPool {
poolManager = new TestTezSessionPoolManager();
poolManager.setupPool(conf);
poolManager.startPool();
+ // this is now a LIFO operation
+
+ // draw 1 and replace
TezSessionState sessionState = poolManager.getSession(null, conf, true, false);
assertEquals("a", sessionState.getQueueName());
poolManager.returnSession(sessionState, false);
sessionState = poolManager.getSession(null, conf, true, false);
- assertEquals("b", sessionState.getQueueName());
+ assertEquals("a", sessionState.getQueueName());
poolManager.returnSession(sessionState, false);
- sessionState = poolManager.getSession(null, conf, true, false);
- assertEquals("c", sessionState.getQueueName());
- poolManager.returnSession(sessionState, false);
+ // [a,b,c,a,b,c]
- sessionState = poolManager.getSession(null, conf, true, false);
- if (sessionState.getQueueName().compareTo("a") != 0) {
- fail();
- }
+ // draw 2 and return in order - further run should return last returned
+ TezSessionState first = poolManager.getSession(null, conf, true, false);
+ TezSessionState second = poolManager.getSession(null, conf, true, false);
+ assertEquals("a", first.getQueueName());
+ assertEquals("b", second.getQueueName());
+ poolManager.returnSession(first, false);
+ poolManager.returnSession(second, false);
+ TezSessionState third = poolManager.getSession(null, conf, true, false);
+ assertEquals("b", third.getQueueName());
+ poolManager.returnSession(third, false);
- poolManager.returnSession(sessionState, false);
+ // [b,a,c,a,b,c]
+
+ first = poolManager.getSession(null, conf, true, false);
+ second = poolManager.getSession(null, conf, true, false);
+ third = poolManager.getSession(null, conf, true, false);
+
+ assertEquals("b", first.getQueueName());
+ assertEquals("a", second.getQueueName());
+ assertEquals("c", third.getQueueName());
+
+ poolManager.returnSession(first, false);
+ poolManager.returnSession(second, false);
+ poolManager.returnSession(third, false);
+
+ // [c,a,b,a,b,c]
+
+ first = poolManager.getSession(null, conf, true, false);
+ assertEquals("c", third.getQueueName());
+ poolManager.returnSession(first, false);
} catch (Exception e) {
e.printStackTrace();
[34/36] hive git commit: HIVE-16808 WebHCat statusdir parameter
doesn't properly handle Unicode characters when using relative path (Eugene
Koifman, reviewed by Daniel Dai, Peter Vary)
Posted by we...@apache.org.
HIVE-16808 WebHCat statusdir parameter doesn't properly handle Unicode characters when using relative path (Eugene Koifman, reviewed by Daniel Dai, Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ee3a3ed3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ee3a3ed3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ee3a3ed3
Branch: refs/heads/hive-14535
Commit: ee3a3ed35b337ff13156c6c6e27dd332cdee2009
Parents: bdacb10
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jun 6 09:20:15 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jun 6 09:20:15 2017 -0700
----------------------------------------------------------------------
.../hcatalog/templeton/tool/TempletonUtils.java | 28 +++++++++-----------
.../templeton/tool/TestTempletonUtils.java | 11 ++++----
2 files changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ee3a3ed3/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index e0ccc70..d3e801a 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -40,8 +40,6 @@ import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import javax.ws.rs.core.UriBuilder;
-
import org.apache.hadoop.hive.common.LogUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,10 +48,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.hcatalog.templeton.UgiFactory;
-import org.apache.hive.hcatalog.templeton.BadParam;
/**
* General utility methods.
@@ -319,17 +315,19 @@ public class TempletonUtils {
public static String addUserHomeDirectoryIfApplicable(String origPathStr, String user)
throws IOException, URISyntaxException {
- URI uri = new URI(origPathStr);
-
- if (uri.getPath().isEmpty()) {
- String newPath = "/user/" + user;
- uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
- } else if (!new Path(uri.getPath()).isAbsolute()) {
- String newPath = "/user/" + user + "/" + uri.getPath();
- uri = UriBuilder.fromUri(uri).replacePath(newPath).build();
- } // no work needed for absolute paths
-
- return uri.toString();
+ if(origPathStr == null || origPathStr.isEmpty()) {
+ return "/user/" + user;
+ }
+ Path p = new Path(origPathStr);
+ if(p.isAbsolute()) {
+ return origPathStr;
+ }
+ if(p.toUri().getPath().isEmpty()) {
+ //origPathStr="hdfs://host:99" for example
+ return new Path(p.toUri().getScheme(), p.toUri().getAuthority(), "/user/" + user).toString();
+ }
+ //can't have relative path if there is scheme/authority
+ return "/user/" + user + "/" + origPathStr;
}
public static Path hadoopFsPath(String fname, final Configuration conf, String user)
http://git-wip-us.apache.org/repos/asf/hive/blob/ee3a3ed3/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
index 3dcdd64..8328245 100644
--- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
+++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
@@ -26,7 +26,6 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.shims.HadoopShimsSecure;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.util.StringUtils;
import org.junit.After;
import org.junit.Assert;
@@ -271,18 +270,20 @@ public class TestTempletonUtils {
String[] sources = new String[] { "output+", "/user/hadoop/output",
"hdfs://container", "hdfs://container/", "hdfs://container/path",
"output#link", "hdfs://cointaner/output#link",
- "hdfs://container@acc/test" };
+ "hdfs://container@acc/test", "/user/webhcat/düsseldorf", "düsseldorf",
+ "䶴狝A﨩O", "hdfs://host:8080"};
String[] expectedResults = new String[] { "/user/webhcat/output+",
"/user/hadoop/output", "hdfs://container/user/webhcat",
"hdfs://container/", "hdfs://container/path",
"/user/webhcat/output#link", "hdfs://cointaner/output#link",
- "hdfs://container@acc/test" };
+ "hdfs://container@acc/test", "/user/webhcat/düsseldorf","/user/webhcat/düsseldorf",
+ "/user/webhcat/䶴狝A﨩O", "hdfs://host:8080/user/webhcat" };
for (int i = 0; i < sources.length; i++) {
String source = sources[i];
String expectedResult = expectedResults[i];
String result = TempletonUtils.addUserHomeDirectoryIfApplicable(source,
"webhcat");
- Assert.assertEquals(result, expectedResult);
+ Assert.assertEquals("i=" + i, expectedResult, result);
}
String badUri = "c:\\some\\path";
@@ -290,7 +291,7 @@ public class TestTempletonUtils {
TempletonUtils.addUserHomeDirectoryIfApplicable(badUri, "webhcat");
Assert.fail("addUserHomeDirectoryIfApplicable should fail for bad URI: "
+ badUri);
- } catch (URISyntaxException ex) {
+ } catch (IllegalArgumentException ex) {
}
}
[19/36] hive git commit: HIVE-16566: Set column stats default as true
when creating new tables/partitions (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
index 6ac97a1..14139bb 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out
@@ -314,7 +314,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -345,7 +345,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
index c1ea0ab..cbc2110 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out
@@ -310,7 +310,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -341,7 +341,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
index fc95d13..a941513 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out
@@ -300,7 +300,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -440,7 +440,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out
index f4537ff..3e74c21 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out
@@ -197,7 +197,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -281,7 +281,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
index 50a726e..a3acd5d 100644
--- a/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
+++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out
@@ -261,7 +261,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
@@ -350,7 +350,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value1,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out
index 5dd79de..8693026 100644
--- a/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out
+++ b/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out
@@ -113,7 +113,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -145,7 +145,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out b/ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out
index 34a01e2..e5c0402 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out
@@ -183,7 +183,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
@@ -214,7 +214,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out
index d4fd0bb..9895aec 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out
@@ -183,7 +183,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
@@ -214,7 +214,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out b/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out
index 88fc15b..bc14fe0 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_ppr.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
@@ -207,7 +207,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out
index 5851472..a8eafaf 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out
@@ -176,7 +176,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
@@ -207,7 +207,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"c1":"true","c2":"true","c3":"true","c4":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,c1,c2,c3,c4
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
index 418f3ec..7bb3111 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out
@@ -83,7 +83,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -169,7 +169,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -349,7 +349,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -380,7 +380,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -853,7 +853,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -939,7 +939,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1120,7 +1120,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -1151,7 +1151,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -3301,7 +3301,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -3387,7 +3387,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
index f7fca5d..bf573b7 100644
--- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
+++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out
@@ -83,7 +83,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -169,7 +169,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,cnt
@@ -368,7 +368,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -399,7 +399,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -872,7 +872,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -958,7 +958,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,cnt
@@ -1158,7 +1158,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -1189,7 +1189,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,cnt
@@ -3434,7 +3434,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
@@ -3520,7 +3520,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cnt":"true","key1":"true","key2":"true","key3":"true","key4":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,key2,key3,key4,cnt
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/input_part2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/input_part2.q.out b/ql/src/test/results/clientpositive/spark/input_part2.q.out
index 6387ed1..e1582e2 100644
--- a/ql/src/test/results/clientpositive/spark/input_part2.q.out
+++ b/ql/src/test/results/clientpositive/spark/input_part2.q.out
@@ -61,7 +61,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -101,7 +101,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -234,7 +234,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
@@ -266,7 +266,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"ds":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,hr,ds
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join17.q.out b/ql/src/test/results/clientpositive/spark/join17.q.out
index c718916..aa4736c 100644
--- a/ql/src/test/results/clientpositive/spark/join17.q.out
+++ b/ql/src/test/results/clientpositive/spark/join17.q.out
@@ -200,7 +200,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","key2":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,value1,key2,value2
@@ -231,7 +231,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key1":"true","key2":"true","value1":"true","value2":"true"}}
bucket_count -1
column.name.delimiter ,
columns key1,value1,key2,value2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join26.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join26.q.out b/ql/src/test/results/clientpositive/spark/join26.q.out
index b1de987..9f85a3e 100644
--- a/ql/src/test/results/clientpositive/spark/join26.q.out
+++ b/ql/src/test/results/clientpositive/spark/join26.q.out
@@ -225,7 +225,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -309,7 +309,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join32.q.out b/ql/src/test/results/clientpositive/spark/join32.q.out
index c93411e..2f50aef 100644
--- a/ql/src/test/results/clientpositive/spark/join32.q.out
+++ b/ql/src/test/results/clientpositive/spark/join32.q.out
@@ -229,7 +229,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -315,7 +315,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
index eec3f4a..f9e81aa 100644
--- a/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
+++ b/ql/src/test/results/clientpositive/spark/join32_lessSize.q.out
@@ -237,7 +237,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -323,7 +323,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -1226,7 +1226,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -1310,7 +1310,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join33.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join33.q.out b/ql/src/test/results/clientpositive/spark/join33.q.out
index c93411e..2f50aef 100644
--- a/ql/src/test/results/clientpositive/spark/join33.q.out
+++ b/ql/src/test/results/clientpositive/spark/join33.q.out
@@ -229,7 +229,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -315,7 +315,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join34.q.out b/ql/src/test/results/clientpositive/spark/join34.q.out
index 86ddd1d..e32f3e4 100644
--- a/ql/src/test/results/clientpositive/spark/join34.q.out
+++ b/ql/src/test/results/clientpositive/spark/join34.q.out
@@ -286,7 +286,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -317,7 +317,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join35.q.out b/ql/src/test/results/clientpositive/spark/join35.q.out
index 74b1ee4..e1a96f6 100644
--- a/ql/src/test/results/clientpositive/spark/join35.q.out
+++ b/ql/src/test/results/clientpositive/spark/join35.q.out
@@ -318,7 +318,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -367,7 +367,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join9.q.out b/ql/src/test/results/clientpositive/spark/join9.q.out
index 9b5b243..373ac8b 100644
--- a/ql/src/test/results/clientpositive/spark/join9.q.out
+++ b/ql/src/test/results/clientpositive/spark/join9.q.out
@@ -201,7 +201,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -232,7 +232,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out b/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out
index 67b2e94..86f3d9a 100644
--- a/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_map_ppr.q.out
@@ -213,7 +213,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
@@ -299,7 +299,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,val2
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/pcr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/pcr.q.out b/ql/src/test/results/clientpositive/spark/pcr.q.out
index dc8c1ab..74e1845 100644
--- a/ql/src/test/results/clientpositive/spark/pcr.q.out
+++ b/ql/src/test/results/clientpositive/spark/pcr.q.out
@@ -3538,7 +3538,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3574,7 +3574,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3657,7 +3657,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -3689,7 +3689,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
index 93b026f..804ff02 100644
--- a/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
+++ b/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out
@@ -114,7 +114,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
@@ -146,7 +146,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 2
bucket_field_name key
column.name.delimiter ,
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample1.q.out b/ql/src/test/results/clientpositive/spark/sample1.q.out
index b44a0e8..ee9eb14 100644
--- a/ql/src/test/results/clientpositive/spark/sample1.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample1.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
@@ -135,7 +135,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value,dt,hr
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample2.q.out b/ql/src/test/results/clientpositive/spark/sample2.q.out
index c109918..85266d1 100644
--- a/ql/src/test/results/clientpositive/spark/sample2.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample2.q.out
@@ -50,7 +50,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -134,7 +134,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample4.q.out b/ql/src/test/results/clientpositive/spark/sample4.q.out
index 31b9aa8..69e7ee9 100644
--- a/ql/src/test/results/clientpositive/spark/sample4.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample4.q.out
@@ -50,7 +50,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -134,7 +134,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample5.q.out b/ql/src/test/results/clientpositive/spark/sample5.q.out
index 71e0f42..558b2db 100644
--- a/ql/src/test/results/clientpositive/spark/sample5.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample5.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -135,7 +135,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample6.q.out b/ql/src/test/results/clientpositive/spark/sample6.q.out
index 37545af..4b35829 100644
--- a/ql/src/test/results/clientpositive/spark/sample6.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample6.q.out
@@ -50,7 +50,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -134,7 +134,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/sample7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/sample7.q.out b/ql/src/test/results/clientpositive/spark/sample7.q.out
index 34c293c..eae33ca 100644
--- a/ql/src/test/results/clientpositive/spark/sample7.q.out
+++ b/ql/src/test/results/clientpositive/spark/sample7.q.out
@@ -51,7 +51,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -135,7 +135,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/stats0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/stats0.q.out b/ql/src/test/results/clientpositive/spark/stats0.q.out
index 6992ffa..ece5f08 100644
--- a/ql/src/test/results/clientpositive/spark/stats0.q.out
+++ b/ql/src/test/results/clientpositive/spark/stats0.q.out
@@ -45,7 +45,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -127,7 +127,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1363,7 +1363,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1445,7 +1445,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/spark/stats3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/stats3.q.out b/ql/src/test/results/clientpositive/spark/stats3.q.out
index 10fe00f..2f76d0e 100644
--- a/ql/src/test/results/clientpositive/spark/stats3.q.out
+++ b/ql/src/test/results/clientpositive/spark/stats3.q.out
@@ -34,7 +34,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true"}}
bucket_count -1
column.name.delimiter ,
columns col1
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/stats0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/stats0.q.out b/ql/src/test/results/clientpositive/stats0.q.out
index 01e00c8..0476acb 100644
--- a/ql/src/test/results/clientpositive/stats0.q.out
+++ b/ql/src/test/results/clientpositive/stats0.q.out
@@ -42,7 +42,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -124,7 +124,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1359,7 +1359,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1450,7 +1450,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1487,7 +1487,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1517,7 +1517,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1538,7 +1538,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1574,7 +1574,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1604,7 +1604,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
@@ -1625,7 +1625,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
column.name.delimiter ,
columns key,value
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/stats3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/stats3.q.out b/ql/src/test/results/clientpositive/stats3.q.out
index 10fe00f..2f76d0e 100644
--- a/ql/src/test/results/clientpositive/stats3.q.out
+++ b/ql/src/test/results/clientpositive/stats3.q.out
@@ -34,7 +34,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true"}}
bucket_count -1
column.name.delimiter ,
columns col1
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out b/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
index 551ad5f..bd024a7 100644
--- a/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
+++ b/ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out
@@ -292,7 +292,7 @@ POSTHOOK: Input: default@empty_tab
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
a int from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\"}}
PREHOOK: query: explain
analyze table empty_tab compute statistics for columns a,b,c,d,e
PREHOOK: type: QUERY
@@ -361,7 +361,7 @@ POSTHOOK: Input: default@empty_tab
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
a int 0 0 0 0 from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\"}}
PREHOOK: query: desc formatted empty_tab b
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@empty_tab
@@ -371,7 +371,7 @@ POSTHOOK: Input: default@empty_tab
# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment
b double 0.0 0.0 0 0 from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\",\"e\":\"true\"}}
PREHOOK: query: CREATE DATABASE test
PREHOOK: type: CREATEDATABASE
PREHOOK: Output: database:test
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/unicode_notation.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unicode_notation.q.out b/ql/src/test/results/clientpositive/unicode_notation.q.out
index 506fc75..f216d72 100644
--- a/ql/src/test/results/clientpositive/unicode_notation.q.out
+++ b/ql/src/test/results/clientpositive/unicode_notation.q.out
@@ -23,7 +23,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -74,7 +74,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -125,7 +125,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}}
numFiles 0
numRows 0
rawDataSize 0
http://git-wip-us.apache.org/repos/asf/hive/blob/52a71e96/ql/src/test/results/clientpositive/unset_table_view_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unset_table_view_property.q.out b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
index d28d800..7097963 100644
--- a/ql/src/test/results/clientpositive/unset_table_view_property.q.out
+++ b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
@@ -16,7 +16,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
numFiles 0
numRows 0
rawDataSize 0
@@ -34,7 +34,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
c 3
#### A masked pattern was here ####
@@ -55,7 +55,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -74,7 +74,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
c 3
d 4
@@ -96,7 +96,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
c 3
#### A masked pattern was here ####
numFiles 0
@@ -116,7 +116,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -135,7 +135,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
b 2
c 3
@@ -158,7 +158,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
c 3
#### A masked pattern was here ####
@@ -179,7 +179,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
+COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
#### A masked pattern was here ####
numFiles 0
[24/36] hive git commit: HIVE-16653: Mergejoin should give itself a
correct tag (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Posted by we...@apache.org.
HIVE-16653: Mergejoin should give itself a correct tag (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cea9ea7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cea9ea7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cea9ea7d
Branch: refs/heads/hive-14535
Commit: cea9ea7d4fe843b1655da8f4191b6b71195db0f4
Parents: 52a71e9
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Fri Jun 2 13:30:21 2017 -0700
Committer: Pengcheng Xiong <px...@hortonworks.com>
Committed: Fri Jun 2 13:30:21 2017 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hadoop/hive/ql/optimizer/MergeJoinProc.java | 23 +-
ql/src/test/queries/clientpositive/tez-tag.q | 68 +++++
.../results/clientpositive/tez/tez-tag.q.out | 301 +++++++++++++++++++
4 files changed, 386 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cea9ea7d/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 62462bd..489f375 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -56,6 +56,7 @@ minitez.query.files=explainuser_3.q,\
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
multi_count_distinct.q,\
+ tez-tag.q,\
tez_union_with_udf.q
http://git-wip-us.apache.org/repos/asf/hive/blob/cea9ea7d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
index 5b73866..bf1d7bb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
@@ -22,9 +22,7 @@ import java.util.Stack;
import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
@@ -44,18 +42,16 @@ public class MergeJoinProc implements NodeProcessor {
throws SemanticException {
GenTezProcContext context = (GenTezProcContext) procCtx;
CommonMergeJoinOperator mergeJoinOp = (CommonMergeJoinOperator) nd;
- if (stack.size() < 2 || !(stack.get(stack.size() - 2) instanceof DummyStoreOperator)) {
+ if (stack.size() < 2) {
+ // safety check for L53 to get parentOp, although it is very unlikely that
+ // stack size is less than 2, i.e., there is only one MergeJoinOperator in the stack.
context.currentMergeJoinOperator = mergeJoinOp;
return null;
}
-
TezWork tezWork = context.currentTask.getWork();
@SuppressWarnings("unchecked")
Operator<? extends OperatorDesc> parentOp =
(Operator<? extends OperatorDesc>) ((stack.get(stack.size() - 2)));
- // Guaranteed to be just 1 because each DummyStoreOperator can be part of only one work.
- BaseWork parentWork = context.childToWorkMap.get(parentOp).get(0);
-
// we need to set the merge work that has been created as part of the dummy store walk. If a
// merge work already exists for this merge join operator, add the dummy store work to the
@@ -70,6 +66,19 @@ public class MergeJoinProc implements NodeProcessor {
context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork);
}
+ if (!(stack.get(stack.size() - 2) instanceof DummyStoreOperator)) {
+ /* this may happen in one of the following case:
+ TS[0], FIL[26], SEL[2], DUMMY_STORE[30], MERGEJOIN[29]]
+ /
+ TS[3], FIL[27], SEL[5], ---------------
+ */
+ context.currentMergeJoinOperator = mergeJoinOp;
+ mergeWork.setTag(mergeJoinOp.getTagForOperator(parentOp));
+ return null;
+ }
+
+ // Guaranteed to be just 1 because each DummyStoreOperator can be part of only one work.
+ BaseWork parentWork = context.childToWorkMap.get(parentOp).get(0);
mergeWork.addMergedWork(null, parentWork, context.leafOperatorToFollowingWork);
mergeWork.setMergeJoinOperator(mergeJoinOp);
tezWork.setVertexType(mergeWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
http://git-wip-us.apache.org/repos/asf/hive/blob/cea9ea7d/ql/src/test/queries/clientpositive/tez-tag.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez-tag.q b/ql/src/test/queries/clientpositive/tez-tag.q
new file mode 100644
index 0000000..3ab477e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez-tag.q
@@ -0,0 +1,68 @@
+set hive.strict.checks.bucketing=false;
+
+set hive.mapred.mode=nonstrict;
+set hive.join.emit.interval=2;
+
+set hive.optimize.ppd=true;
+set hive.ppd.remove.duplicatefilters=true;
+set hive.tez.dynamic.partition.pruning=true;
+set hive.tez.dynamic.semijoin.reduction=false;
+set hive.optimize.metadataonly=false;
+set hive.optimize.index.filter=true;
+set hive.stats.autogather=true;
+set hive.tez.bigtable.minsize.semijoin.reduction=1;
+set hive.tez.min.bloom.filter.entries=1;
+set hive.stats.fetch.column.stats=true;
+
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ;
+
+CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE;
+
+CREATE TABLE src2 as select * from src1;
+insert into src2 select * from src2;
+insert into src2 select * from src2;
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08');
+
+load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08');
+
+set hive.optimize.bucketingsorting=false;
+insert overwrite table tab_part partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin_part;
+
+CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE;
+insert overwrite table tab partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin;
+
+set hive.convert.join.bucket.mapjoin.tez = true;
+set hive.auto.convert.sortmerge.join = true;
+
+set hive.auto.convert.join.noconditionaltask.size=0;
+set hive.mapjoin.hybridgrace.minwbsize=125;
+set hive.mapjoin.hybridgrace.minnumpartitions=4;
+
+set hive.llap.memory.oversubscription.max.executors.per.query=3;
+
+CREATE TABLE tab2 (key int, value string, ds string);
+
+set hive.exec.dynamic.partition.mode=nonstrict
+insert into tab2select key, value, ds from tab;
+analyze table tab2 compute statistics;
+analyze table tab2 compute statistics for columns;
+
+
+explain select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value;
+
+select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value;
+
+
+explain select count(*) from (select x.key as key, min(x.value) as value from tab2 x group by x.key) a join (select x.key as key, min(x.value) as value from tab2 x group by x.key) b on a.key = b.key join src1 c on a.value = c.value where c.key < 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/cea9ea7d/ql/src/test/results/clientpositive/tez/tez-tag.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez-tag.q.out b/ql/src/test/results/clientpositive/tez/tez-tag.q.out
new file mode 100644
index 0000000..1201ee4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/tez-tag.q.out
@@ -0,0 +1,301 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin
+PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tab_part
+POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tab_part
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part
+PREHOOK: query: CREATE TABLE src2 as select * from src1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src1
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src2
+POSTHOOK: query: CREATE TABLE src2 as select * from src1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src2
+POSTHOOK: Lineage: src2.key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src2.value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert into src2 select * from src2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src2
+PREHOOK: Output: default@src2
+POSTHOOK: query: insert into src2 select * from src2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src2
+POSTHOOK: Output: default@src2
+POSTHOOK: Lineage: src2.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src2.value SIMPLE [(src2)src2.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: insert into src2 select * from src2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src2
+PREHOOK: Output: default@src2
+POSTHOOK: query: insert into src2 select * from src2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src2
+POSTHOOK: Output: default@src2
+POSTHOOK: Lineage: src2.key SIMPLE [(src2)src2.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: src2.value SIMPLE [(src2)src2.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin
+POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part
+POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08
+PREHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part
+PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08
+PREHOOK: Output: default@tab_part@ds=2008-04-08
+POSTHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part
+POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08
+POSTHOOK: Output: default@tab_part@ds=2008-04-08
+POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tab
+POSTHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tab
+PREHOOK: query: insert overwrite table tab partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin
+PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08
+PREHOOK: Output: default@tab@ds=2008-04-08
+POSTHOOK: query: insert overwrite table tab partition (ds='2008-04-08')
+select key,value from srcbucket_mapjoin
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin
+POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08
+POSTHOOK: Output: default@tab@ds=2008-04-08
+POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: CREATE TABLE tab2 (key int, value string, ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tab2
+POSTHOOK: query: CREATE TABLE tab2 (key int, value string, ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tab2
+Warning: Value had a \n character in it.
+PREHOOK: query: analyze table tab2 compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab2
+PREHOOK: Output: default@tab2
+POSTHOOK: query: analyze table tab2 compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab2
+POSTHOOK: Output: default@tab2
+PREHOOK: query: analyze table tab2 compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab2
+PREHOOK: Output: default@tab2
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table tab2 compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab2
+POSTHOOK: Output: default@tab2
+#### A masked pattern was here ####
+PREHOOK: query: explain select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 3 <- Map 2 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 4
+ File Output Operator [FS_20]
+ Group By Operator [GBY_18] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 3 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_17]
+ Group By Operator [GBY_16] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Merge Join Operator [MERGEJOIN_31] (rows=605 width=18)
+ Conds:RS_12._col1=RS_13._col0(Inner)
+ <-Map 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_29] (rows=550 width=18)
+ Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col1"]
+ <-Select Operator [SEL_2] (rows=242 width=18)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_26] (rows=242 width=18)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_0] (rows=242 width=18)
+ default@tab,a,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
+ <-Select Operator [SEL_5] (rows=500 width=18)
+ Output:["_col0"]
+ Filter Operator [FIL_27] (rows=500 width=18)
+ predicate:key is not null
+ TableScan [TS_3] (rows=500 width=18)
+ default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"]
+ <-Map 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=25 width=89)
+ Output:["_col0"]
+ Filter Operator [FIL_28] (rows=25 width=89)
+ predicate:value is not null
+ TableScan [TS_6] (rows=25 width=89)
+ default@src1,c,Tbl:COMPLETE,Col:COMPLETE,Output:["value"]
+
+PREHOOK: query: select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Input: default@tab
+PREHOOK: Input: default@tab@ds=2008-04-08
+PREHOOK: Input: default@tab_part
+PREHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from tab a join tab_part b on a.key = b.key join src1 c on a.value = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Input: default@tab
+POSTHOOK: Input: default@tab@ds=2008-04-08
+POSTHOOK: Input: default@tab_part
+POSTHOOK: Input: default@tab_part@ds=2008-04-08
+#### A masked pattern was here ####
+40
+PREHOOK: query: explain select count(*) from (select x.key as key, min(x.value) as value from tab2 x group by x.key) a join (select x.key as key, min(x.value) as value from tab2 x group by x.key) b on a.key = b.key join src1 c on a.value = c.value where c.key < 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(*) from (select x.key as key, min(x.value) as value from tab2 x group by x.key) a join (select x.key as key, min(x.value) as value from tab2 x group by x.key) b on a.key = b.key join src1 c on a.value = c.value where c.key < 0
+POSTHOOK: type: QUERY
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+Reducer 3 <- Map 7 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 4
+ File Output Operator [FS_29]
+ Group By Operator [GBY_27] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Reducer 3 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_26]
+ Group By Operator [GBY_25] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Merge Join Operator [MERGEJOIN_41] (rows=1 width=8)
+ Conds:RS_21._col1=RS_22._col1(Inner)
+ <-Map 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_22]
+ PartitionCols:_col1
+ Select Operator [SEL_17] (rows=8 width=175)
+ Output:["_col1"]
+ Filter Operator [FIL_38] (rows=8 width=175)
+ predicate:((UDFToDouble(key) < 0.0) and value is not null)
+ TableScan [TS_15] (rows=25 width=175)
+ default@src1,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_21]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_39] (rows=1 width=184)
+ Conds:FIL_35._col0=GBY_13._col0(Inner),Output:["_col1"]
+ <-Group By Operator [GBY_13] (rows=1 width=4)
+ Output:["_col0"],keys:KEY._col0
+ <-Filter Operator [FIL_35] (rows=1 width=188)
+ predicate:_col1 is not null
+ Group By Operator [GBY_5] (rows=1 width=188)
+ Output:["_col0","_col1"],aggregations:["min(VALUE._col0)"],keys:KEY._col0
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_4]
+ PartitionCols:_col0
+ Group By Operator [GBY_3] (rows=1 width=188)
+ Output:["_col0","_col1"],aggregations:["min(value)"],keys:key
+ Filter Operator [FIL_36] (rows=1 width=88)
+ predicate:key is not null
+ TableScan [TS_0] (rows=1 width=88)
+ default@tab2,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Map 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
+ PartitionCols:_col0
+ Group By Operator [GBY_11] (rows=1 width=4)
+ Output:["_col0"],keys:key
+ Filter Operator [FIL_37] (rows=1 width=4)
+ predicate:key is not null
+ TableScan [TS_8] (rows=1 width=4)
+ default@tab2,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+