You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/02/28 00:42:42 UTC
hive git commit: HIVE-16494 : udaf percentile_approx() may fail on
CBO (Miklos Gergely via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 36974dd15 -> a0b262875
HIVE-16494 : udaf percentile_approx() may fail on CBO (Miklos Gergely via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a0b26287
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a0b26287
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a0b26287
Branch: refs/heads/master
Commit: a0b262875cd22120ab04ef598cf2bf6ce6163935
Parents: 36974dd
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Mon Feb 26 06:57:00 2018 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Feb 27 16:41:56 2018 -0800
----------------------------------------------------------------------
.../hive/ql/parse/TypeCheckProcFactory.java | 89 ++++++++++++++++++--
.../clientpositive/udaf_percentile_approx_23.q | 1 -
.../clientpositive/llap/vector_windowing.q.out | 6 +-
3 files changed, 85 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a0b26287/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index 14217e3..1a7e7e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -79,9 +79,13 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
@@ -703,13 +707,31 @@ public class TypeCheckProcFactory {
static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) {
ObjectInspector inspector = colInfo.getObjectInspector();
- if (inspector instanceof ConstantObjectInspector &&
- inspector instanceof PrimitiveObjectInspector) {
- PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
- Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
- ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), poi.getPrimitiveJavaObject(constant));
- constantExpr.setFoldedFromCol(colInfo.getInternalName());
- return constantExpr;
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector);
+ }
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof ListObjectInspector) {
+ ObjectInspector listElementOI = ((ListObjectInspector)inspector).getListElementObjectInspector();
+ if (listElementOI instanceof PrimitiveObjectInspector) {
+ return toListConstDesc(colInfo, inspector, listElementOI);
+ }
+ }
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof MapObjectInspector) {
+ ObjectInspector keyOI = ((MapObjectInspector)inspector).getMapKeyObjectInspector();
+ ObjectInspector valueOI = ((MapObjectInspector)inspector).getMapValueObjectInspector();
+ if (keyOI instanceof PrimitiveObjectInspector && valueOI instanceof PrimitiveObjectInspector) {
+ return toMapConstDesc(colInfo, inspector, keyOI, valueOI);
+ }
+ }
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof StructObjectInspector) {
+ boolean allPrimitive = true;
+ List<? extends StructField> fields = ((StructObjectInspector)inspector).getAllStructFieldRefs();
+ for (StructField field : fields) {
+ allPrimitive &= field.getFieldObjectInspector() instanceof PrimitiveObjectInspector;
+ }
+ if (allPrimitive) {
+ return toStructConstDesc(colInfo, inspector, fields);
+ }
}
// non-constant or non-primitive constants
ExprNodeColumnDesc column = new ExprNodeColumnDesc(colInfo);
@@ -717,6 +739,59 @@ public class TypeCheckProcFactory {
return column;
}
+ private static ExprNodeConstantDesc toPrimitiveConstDesc(ColumnInfo colInfo, ObjectInspector inspector) {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ ExprNodeConstantDesc constantExpr =
+ new ExprNodeConstantDesc(colInfo.getType(), poi.getPrimitiveJavaObject(constant));
+ constantExpr.setFoldedFromCol(colInfo.getInternalName());
+ return constantExpr;
+ }
+
+ private static ExprNodeConstantDesc toListConstDesc(ColumnInfo colInfo, ObjectInspector inspector,
+ ObjectInspector listElementOI) {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)listElementOI;
+ List<?> values = (List<?>)((ConstantObjectInspector) inspector).getWritableConstantValue();
+ List<Object> constant = new ArrayList<Object>();
+ for (Object o : values) {
+ constant.add(poi.getPrimitiveJavaObject(o));
+ }
+
+ ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant);
+ constantExpr.setFoldedFromCol(colInfo.getInternalName());
+ return constantExpr;
+ }
+
+ private static ExprNodeConstantDesc toMapConstDesc(ColumnInfo colInfo, ObjectInspector inspector,
+ ObjectInspector keyOI, ObjectInspector valueOI) {
+ PrimitiveObjectInspector keyPoi = (PrimitiveObjectInspector)keyOI;
+ PrimitiveObjectInspector valuePoi = (PrimitiveObjectInspector)valueOI;
+ Map<?,?> values = (Map<?,?>)((ConstantObjectInspector) inspector).getWritableConstantValue();
+ Map<Object, Object> constant = new HashMap<Object, Object>();
+ for (Map.Entry<?, ?> e : values.entrySet()) {
+ constant.put(keyPoi.getPrimitiveJavaObject(e.getKey()), valuePoi.getPrimitiveJavaObject(e.getValue()));
+ }
+
+ ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant);
+ constantExpr.setFoldedFromCol(colInfo.getInternalName());
+ return constantExpr;
+ }
+
+ private static ExprNodeConstantDesc toStructConstDesc(ColumnInfo colInfo, ObjectInspector inspector,
+ List<? extends StructField> fields) {
+ List<?> values = (List<?>)((ConstantObjectInspector) inspector).getWritableConstantValue();
+ List<Object> constant = new ArrayList<Object>();
+ for (int i = 0; i < values.size(); i++) {
+ Object value = values.get(i);
+ PrimitiveObjectInspector fieldPoi = (PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector();
+ constant.add(fieldPoi.getPrimitiveJavaObject(value));
+ }
+
+ ExprNodeConstantDesc constantExpr = new ExprNodeConstantDesc(colInfo.getType(), constant);
+ constantExpr.setFoldedFromCol(colInfo.getInternalName());
+ return constantExpr;
+ }
+
/**
* Factory method to get ColumnExprProcessor.
*
http://git-wip-us.apache.org/repos/asf/hive/blob/a0b26287/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
index 38cf927..70974ba 100644
--- a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
+++ b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
@@ -99,5 +99,4 @@ select percentile_approx(key, 0.5) from bucket;
select percentile_approx(key, 0.5) between 255.0 and 257.0 from bucket;
-- test where number of elements is zero
-set hive.cbo.enable=false;
select percentile_approx(key, array(0.50, 0.70, 0.90, 0.95, 0.99)) from bucket where key > 10000;
http://git-wip-us.apache.org/repos/asf/hive/blob/a0b26287/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
index 530610d..c3f2fb0 100644
--- a/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_windowing.q.out
@@ -4958,8 +4958,8 @@ STAGE PLANS:
alias: part
Statistics: Num rows: 26 Data size: 5798 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int), array(1,2,3) (type: array<int>)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 26 Data size: 7254 Basic stats: COMPLETE Column stats: COMPLETE
Lateral View Forward
Statistics: Num rows: 26 Data size: 7254 Basic stats: COMPLETE Column stats: COMPLETE
@@ -4977,7 +4977,7 @@ STAGE PLANS:
Statistics: Num rows: 52 Data size: 10166 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Select Operator
- expressions: _col3 (type: array<int>)
+ expressions: Const array<int> [1, 2, 3] (type: array<int>)
outputColumnNames: _col0
Statistics: Num rows: 26 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
UDTF Operator