You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/09/27 07:55:31 UTC
svn commit: r1526797 - in /hive/trunk:
ql/src/test/org/apache/hadoop/hive/ql/udf/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/
serde/src/test/org/apache/hadoop/hive/serde2/lazy/
Author: thejas
Date: Fri Sep 27 05:55:30 2013
New Revision: 1526797
URL: http://svn.apache.org/r1526797
Log:
HIVE-5352: cast(1.0 as int) returns null (Gunther Hagleitner via Thejas Nair)
Added:
hive/trunk/ql/src/test/queries/clientpositive/cast_to_int.q
hive/trunk/ql/src/test/results/clientpositive/cast_to_int.q.out
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java?rev=1526797&r1=1526796&r2=1526797&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestToInteger.java Fri Sep 27 05:55:30 2013
@@ -25,6 +25,6 @@ public class TestToInteger extends TestC
Text t4 = new Text("1.1");
IntWritable i4 = ti.evaluate(t4);
- assertNull(i4);
+ assertEquals(1, i4.get());
}
}
Added: hive/trunk/ql/src/test/queries/clientpositive/cast_to_int.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/cast_to_int.q?rev=1526797&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/cast_to_int.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/cast_to_int.q Fri Sep 27 05:55:30 2013
@@ -0,0 +1,30 @@
+-- cast string floats to integer types
+select
+ cast('1' as float),
+ cast('1.4' as float),
+ cast('1.6' as float),
+ cast('1' as int),
+ cast('1.4' as int),
+ cast('1.6' as int),
+ cast('1' as tinyint),
+ cast('1.4' as tinyint),
+ cast('1.6' as tinyint),
+ cast('1' as smallint),
+ cast('1.4' as smallint),
+ cast('1.6' as smallint),
+ cast('1' as bigint),
+ cast('1.4' as bigint),
+ cast('1.6' as bigint),
+ cast (cast('1' as float) as int),
+ cast(cast ('1.4' as float) as int),
+ cast(cast ('1.6' as float) as int),
+ cast('+1e5' as int),
+ cast('2147483647' as int),
+ cast('-2147483648' as int),
+ cast('32767' as smallint),
+ cast('-32768' as smallint),
+ cast('-128' as tinyint),
+ cast('127' as tinyint),
+ cast('1.0a' as int),
+ cast('-1.-1' as int)
+from src limit 1;
Added: hive/trunk/ql/src/test/results/clientpositive/cast_to_int.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/cast_to_int.q.out?rev=1526797&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/cast_to_int.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/cast_to_int.q.out Fri Sep 27 05:55:30 2013
@@ -0,0 +1,67 @@
+PREHOOK: query: -- cast string floats to integer types
+select
+ cast('1' as float),
+ cast('1.4' as float),
+ cast('1.6' as float),
+ cast('1' as int),
+ cast('1.4' as int),
+ cast('1.6' as int),
+ cast('1' as tinyint),
+ cast('1.4' as tinyint),
+ cast('1.6' as tinyint),
+ cast('1' as smallint),
+ cast('1.4' as smallint),
+ cast('1.6' as smallint),
+ cast('1' as bigint),
+ cast('1.4' as bigint),
+ cast('1.6' as bigint),
+ cast (cast('1' as float) as int),
+ cast(cast ('1.4' as float) as int),
+ cast(cast ('1.6' as float) as int),
+ cast('+1e5' as int),
+ cast('2147483647' as int),
+ cast('-2147483648' as int),
+ cast('32767' as smallint),
+ cast('-32768' as smallint),
+ cast('-128' as tinyint),
+ cast('127' as tinyint),
+ cast('1.0a' as int),
+ cast('-1.-1' as int)
+from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- cast string floats to integer types
+select
+ cast('1' as float),
+ cast('1.4' as float),
+ cast('1.6' as float),
+ cast('1' as int),
+ cast('1.4' as int),
+ cast('1.6' as int),
+ cast('1' as tinyint),
+ cast('1.4' as tinyint),
+ cast('1.6' as tinyint),
+ cast('1' as smallint),
+ cast('1.4' as smallint),
+ cast('1.6' as smallint),
+ cast('1' as bigint),
+ cast('1.4' as bigint),
+ cast('1.6' as bigint),
+ cast (cast('1' as float) as int),
+ cast(cast ('1.4' as float) as int),
+ cast(cast ('1.6' as float) as int),
+ cast('+1e5' as int),
+ cast('2147483647' as int),
+ cast('-2147483648' as int),
+ cast('32767' as smallint),
+ cast('-32768' as smallint),
+ cast('-128' as tinyint),
+ cast('127' as tinyint),
+ cast('1.0a' as int),
+ cast('-1.-1' as int)
+from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1.0 1.4 1.6 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 NULL 2147483647 -2147483648 32767 -32768 -128 127 NULL NULL
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=1526797&r1=1526796&r2=1526797&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Fri Sep 27 05:55:30 2013
@@ -136,11 +136,18 @@ public class LazyInteger extends
*/
private static int parse(byte[] bytes, int start, int length, int offset,
int radix, boolean negative) {
+ byte separator = '.';
int max = Integer.MIN_VALUE / radix;
int result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1) {
+ if (bytes[offset-1] == separator) {
+ // We allow decimals and will return a truncated integer in that case.
+ // Therefore we won't throw an exception here (checking the fractional
+ // part happens below.)
+ break;
+ }
throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
length));
}
@@ -155,6 +162,18 @@ public class LazyInteger extends
}
result = next;
}
+
+ // This is the case when we've encountered a decimal separator. The fractional
+ // part will not change the number, but we will verify that the fractional part
+ // is well formed.
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ }
+
if (!negative) {
result = -result;
if (result < 0) {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=1526797&r1=1526796&r2=1526797&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Fri Sep 27 05:55:30 2013
@@ -138,11 +138,18 @@ public class LazyLong extends
*/
private static long parse(byte[] bytes, int start, int length, int offset,
int radix, boolean negative) {
+ byte separator = '.';
long max = Long.MIN_VALUE / radix;
long result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1 || max > result) {
+ if (bytes[offset-1] == separator) {
+ // We allow decimals and will return a truncated integer in that case.
+ // Therefore we won't throw an exception here (checking the fractional
+ // part happens below.)
+ break;
+ }
throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
length));
}
@@ -153,6 +160,18 @@ public class LazyLong extends
}
result = next;
}
+
+ // This is the case when we've encountered a decimal separator. The fractional
+ // part will not change the number, but we will verify that the fractional part
+ // is well formed.
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ }
+
if (!negative) {
result = -result;
if (result < 0) {
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java?rev=1526797&r1=1526796&r2=1526797&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java Fri Sep 27 05:55:30 2013
@@ -62,12 +62,12 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL\t");
t.append(new byte[]{(byte)Integer.parseInt("10111111", 2)}, 0, 1);
- StringBuffer sb = new StringBuffer("123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\tNULL\t");
+ StringBuffer sb = new StringBuffer("123\t456\t789\t1000\t5.3\thive and hadoop\t1\tNULL\t");
String s = sb.append(new String(Base64.encodeBase64(new byte[]{(byte)Integer.parseInt("10111111", 2)}))).toString();
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, null, new BytesWritable(new byte[]{(byte)Integer.parseInt("10111111", 2)})};
+ new Text("hive and hadoop"), new IntWritable(1), null, new BytesWritable(new byte[]{(byte)Integer.parseInt("10111111", 2)})};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
@@ -128,11 +128,11 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
- String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t";
+ String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta\tb\t";
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, new Text("a\tb\t")};
+ new Text("hive and hadoop"), new IntWritable(1), new Text("a\tb\t")};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
@@ -156,11 +156,11 @@ public class TestLazySimpleSerDe extends
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
- String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta";
+ String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta";
Object[] expectedFieldsData = {new ByteWritable((byte) 123),
new ShortWritable((short) 456), new IntWritable(789),
new LongWritable(1000), new DoubleWritable(5.3),
- new Text("hive and hadoop"), null, new Text("a")};
+ new Text("hive and hadoop"), new IntWritable(1), new Text("a")};
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);