You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2014/08/20 04:34:33 UTC
svn commit: r1619024 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/orc/
java/org/apache/hadoop/hive/ql/io/sarg/
test/org/apache/hadoop/hive/ql/io/sarg/ test/queries/clientpositive/
test/results/clientpositive/
Author: navis
Date: Wed Aug 20 02:34:32 2014
New Revision: 1619024
URL: http://svn.apache.org/r1619024
Log:
HIVE-7771 : ORC PPD fails for some decimal predicates (Prasanth J, reviwed by Daniel Dai)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Wed Aug 20 02:34:32 2014
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hive.con
import java.io.EOFException;
import java.io.IOException;
+import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
@@ -2411,6 +2412,9 @@ class RecordReaderImpl implements Record
private static Object getBaseObjectForComparison(Object predObj, Object statsObj) {
if (predObj != null) {
+ if (predObj instanceof ExprNodeConstantDesc) {
+ predObj = ((ExprNodeConstantDesc) predObj).getValue();
+ }
// following are implicitly convertible
if (statsObj instanceof Long) {
if (predObj instanceof Double) {
@@ -2429,10 +2433,6 @@ class RecordReaderImpl implements Record
return Double.valueOf(predObj.toString());
}
} else if (statsObj instanceof String) {
- // Ex: where d = date '1970-02-01' will be ExprNodeConstantDesc
- if (predObj instanceof ExprNodeConstantDesc) {
- return ((ExprNodeConstantDesc) predObj).getValue().toString();
- }
return predObj.toString();
} else if (statsObj instanceof HiveDecimal) {
if (predObj instanceof Long) {
@@ -2441,6 +2441,8 @@ class RecordReaderImpl implements Record
return HiveDecimal.create(predObj.toString());
} else if (predObj instanceof String) {
return HiveDecimal.create(predObj.toString());
+ } else if (predObj instanceof BigDecimal) {
+ return HiveDecimal.create((BigDecimal)predObj);
}
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java Wed Aug 20 02:34:32 2014
@@ -18,14 +18,6 @@
package org.apache.hadoop.hive.ql.io.sarg;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
@@ -57,6 +49,15 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import java.math.BigDecimal;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
/**
* The implementation of SearchArguments.
*/
@@ -947,7 +948,8 @@ final class SearchArgumentImpl implement
literal instanceof Long ||
literal instanceof Double ||
literal instanceof DateWritable ||
- literal instanceof HiveDecimal) {
+ literal instanceof HiveDecimal ||
+ literal instanceof BigDecimal) {
return literal;
} else if (literal instanceof HiveChar ||
literal instanceof HiveVarchar) {
@@ -979,7 +981,8 @@ final class SearchArgumentImpl implement
return PredicateLeaf.Type.FLOAT;
} else if (literal instanceof DateWritable) {
return PredicateLeaf.Type.DATE;
- } else if (literal instanceof HiveDecimal) {
+ } else if (literal instanceof HiveDecimal ||
+ literal instanceof BigDecimal) {
return PredicateLeaf.Type.DECIMAL;
}
throw new IllegalArgumentException("Unknown type for literal " + literal);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java Wed Aug 20 02:34:32 2014
@@ -18,15 +18,7 @@
package org.apache.hadoop.hive.ql.io.sarg;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertTrue;
-
-import java.beans.XMLDecoder;
-import java.io.ByteArrayInputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-import java.util.Set;
-
+import com.google.common.collect.Sets;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
@@ -37,7 +29,15 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.junit.Test;
-import com.google.common.collect.Sets;
+import java.beans.XMLDecoder;
+import java.io.ByteArrayInputStream;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Set;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
/**
* These test the SARG implementation.
@@ -2847,4 +2847,36 @@ public class TestSearchArgumentImpl {
"leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
"expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
}
+
+ @Test
+ public void testBuilderComplexTypes2() throws Exception {
+ SearchArgument sarg =
+ SearchArgument.FACTORY.newBuilder()
+ .startAnd()
+ .lessThan("x", new DateWritable(10))
+ .lessThanEquals("y", new HiveChar("hi", 10))
+ .equals("z", new BigDecimal("1.0"))
+ .end()
+ .build();
+ assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" +
+ "leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
+ "leaf-2 = (EQUALS z 1.0)\n" +
+ "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
+
+ sarg = SearchArgument.FACTORY.newBuilder()
+ .startNot()
+ .startOr()
+ .isNull("x")
+ .between("y", new BigDecimal(10), 20.0)
+ .in("z", (byte)1, (short)2, (int)3)
+ .nullSafeEquals("a", new HiveVarchar("stinger", 100))
+ .end()
+ .end()
+ .build();
+ assertEquals("leaf-0 = (IS_NULL x)\n" +
+ "leaf-1 = (BETWEEN y 10 20.0)\n" +
+ "leaf-2 = (IN z 1 2 3)\n" +
+ "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
+ "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
+ }
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q Wed Aug 20 02:34:32 2014
@@ -85,6 +85,18 @@ set hive.optimize.index.filter=true;
select sum(hash(*)) from newtypesorc where d<=cast('11.22' as float);
set hive.optimize.index.filter=false;
+select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal);
+
+set hive.optimize.index.filter=true;
+select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal);
+
+set hive.optimize.index.filter=false;
+select sum(hash(*)) from newtypesorc where d<=11.22BD;
+
+set hive.optimize.index.filter=true;
+select sum(hash(*)) from newtypesorc where d<=11.22BD;
+
+set hive.optimize.index.filter=false;
select sum(hash(*)) from newtypesorc where d<=12;
set hive.optimize.index.filter=true;
Modified: hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out Wed Aug 20 02:34:32 2014
@@ -254,6 +254,42 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@newtypesorc
#### A masked pattern was here ####
81475875500
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+-252951929000
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+-252951929000
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+81475875500
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+81475875500
PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=12
PREHOOK: type: QUERY
PREHOOK: Input: default@newtypesorc