You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/10/05 14:48:28 UTC
[1/2] hive git commit: HIVE-11835: Type decimal(1, 1) reads 0.0, 0.00,
etc from text file as NULL (Reviewed by Szehon)
Repository: hive
Updated Branches:
refs/heads/branch-1 93b981f5d -> 7c0bf304b
HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/01b55f4a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/01b55f4a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/01b55f4a
Branch: refs/heads/branch-1
Commit: 01b55f4aad72e3efb3f3165ab41deeeb81908453
Parents: 93b981f
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Mon Oct 5 05:42:20 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Oct 5 05:48:10 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/common/type/HiveDecimal.java | 10 +-
.../hive/common/type/TestHiveDecimal.java | 12 +--
.../test/resources/testconfiguration.properties | 1 +
.../test/queries/clientpositive/decimal_1_1.q | 9 ++
.../results/clientpositive/decimal_1_1.q.out | 104 +++++++++++++++++++
.../clientpositive/spark/decimal_1_1.q.out | 104 +++++++++++++++++++
6 files changed, 233 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
index 5d2b4d2..da88b7a 100644
--- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
+++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -272,11 +272,19 @@ public class HiveDecimal implements Comparable<HiveDecimal> {
return bd;
}
- public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
+ private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) {
if (bd == null) {
return null;
}
+ /**
+ * Specially handling the case that bd=0, and we are converting it to a type where precision=scale,
+ * such as decimal(1, 1).
+ */
+ if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) {
+ return bd.setScale(maxScale);
+ }
+
bd = trim(bd);
if (bd.scale() > maxScale) {
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
index ba5ef71..f68842c 100644
--- a/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
+++ b/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
@@ -42,15 +42,15 @@ public class TestHiveDecimal {
Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE);
decStr = "57847525803324040144343378.09799306448796128931113691624";
- BigDecimal bd = new BigDecimal(decStr);
- BigDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
+ HiveDecimal bd = HiveDecimal.create(decStr);
+ HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5);
Assert.assertNull(bd1);
bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5);
Assert.assertEquals("57847525803324040144343378.09799", bd1.toString());
bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20);
Assert.assertNull(bd1);
- dec = HiveDecimal.create(bd, false);
+ dec = HiveDecimal.create(new BigDecimal(decStr), false);
Assert.assertNull(dec);
dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624");
@@ -65,10 +65,10 @@ public class TestHiveDecimal {
// Rounding numbers that increase int digits
Assert.assertEquals("10",
- HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString());
- Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0));
+ HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString());
+ Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0));
Assert.assertEquals("9",
- HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString());
+ HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString());
}
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 0699441..faa9913 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -653,6 +653,7 @@ spark.query.files=add_part_multiple.q, \
custom_input_output_format.q, \
date_join1.q, \
date_udf.q, \
+ decimal_1_1.q, \
decimal_join.q, \
disable_merge_for_bucketing.q, \
dynamic_rdd_cache.q, \
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/queries/clientpositive/decimal_1_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/decimal_1_1.q b/ql/src/test/queries/clientpositive/decimal_1_1.q
new file mode 100644
index 0000000..83ce521
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_1_1.q
@@ -0,0 +1,9 @@
+drop table if exists decimal_1_1;
+
+create table decimal_1_1 (d decimal(1,1));
+load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1;
+select * from decimal_1_1;
+
+select d from decimal_1_1 order by d desc;
+
+drop table decimal_1_1;
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/decimal_1_1.q.out b/ql/src/test/results/clientpositive/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1
http://git-wip-us.apache.org/repos/asf/hive/blob/01b55f4a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
new file mode 100644
index 0000000..b2704c6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/spark/decimal_1_1.q.out
@@ -0,0 +1,104 @@
+PREHOOK: query: drop table if exists decimal_1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1_1 (d decimal(1,1))
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: create table decimal_1_1 (d decimal(1,1))
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: load data local inpath '../../data/files/decimal_1_1.txt' into table decimal_1_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@decimal_1_1
+PREHOOK: query: select * from decimal_1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from decimal_1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0
+0
+0
+0.1
+0.2
+0.9
+0.9
+NULL
+0.3
+NULL
+NULL
+0
+0
+NULL
+0
+0
+0
+0
+-0.1
+-0.2
+-0.9
+-0.9
+NULL
+-0.3
+NULL
+NULL
+0
+0
+NULL
+0
+PREHOOK: query: select d from decimal_1_1 order by d desc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select d from decimal_1_1 order by d desc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@decimal_1_1
+#### A masked pattern was here ####
+0.9
+0.9
+0.3
+0.2
+0.1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+-0.1
+-0.2
+-0.3
+-0.9
+-0.9
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+PREHOOK: query: drop table decimal_1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@decimal_1_1
+PREHOOK: Output: default@decimal_1_1
+POSTHOOK: query: drop table decimal_1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@decimal_1_1
+POSTHOOK: Output: default@decimal_1_1
[2/2] hive git commit: HIVE-11835: Type decimal(1, 1) reads 0.0, 0.00,
etc from text file as NULL (Reviewed by Szehon) -- Add the forgotten
new data file
Posted by xu...@apache.org.
HIVE-11835: Type decimal(1,1) reads 0.0, 0.00, etc from text file as NULL (Reviewed by Szehon)
-- Add the forgotten new data file
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c0bf304
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c0bf304
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c0bf304
Branch: refs/heads/branch-1
Commit: 7c0bf304bb61e32fb5b6bbfd3f4ed12e2f9a25b3
Parents: 01b55f4
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Mon Oct 5 05:46:20 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Mon Oct 5 05:48:11 2015 -0700
----------------------------------------------------------------------
data/files/decimal_1_1.txt | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7c0bf304/data/files/decimal_1_1.txt
----------------------------------------------------------------------
diff --git a/data/files/decimal_1_1.txt b/data/files/decimal_1_1.txt
new file mode 100644
index 0000000..ec16804
--- /dev/null
+++ b/data/files/decimal_1_1.txt
@@ -0,0 +1,30 @@
+0.0
+0.0000
+.0
+0.1
+0.15
+0.9
+0.94
+0.99
+0.345
+1.0
+1
+0
+00
+22
+1E-9
+-0.0
+-0.0000
+-.0
+-0.1
+-0.15
+-0.9
+-0.94
+-0.99
+-0.345
+-1.0
+-1
+-0
+-00
+-22
+-1E-9