You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/10/23 19:53:01 UTC
hive git commit: HIVE-12084 : Hive queries with ORDER BY and large
LIMIT fails with OutOfMemoryError Java heap space (Hari Subramaniyan,
reviewed by Laljo John Pullokkaran)
Repository: hive
Updated Branches:
refs/heads/master 87989da55 -> d6f9a8e9c
HIVE-12084 : Hive queries with ORDER BY and large LIMIT fails with OutOfMemoryError Java heap space (Hari Subramaniyan, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d6f9a8e9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d6f9a8e9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d6f9a8e9
Branch: refs/heads/master
Commit: d6f9a8e9cbd7dddb6211fc46dcd722f9ad56557f
Parents: 87989da
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Fri Oct 23 10:52:46 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Fri Oct 23 10:52:46 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/TopNHash.java | 2 +-
ql/src/test/queries/clientpositive/topn.q | 13 ++++++
ql/src/test/results/clientpositive/topn.q.out | 42 ++++++++++++++++++++
3 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d6f9a8e9/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
index 484006a..8859add 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TopNHash.java
@@ -104,7 +104,7 @@ public class TopNHash {
}
// limit * 64 : compensation of arrays for key/value/hashcodes
- this.threshold = (long) (memUsage * Runtime.getRuntime().maxMemory()) - topN * 64;
+ this.threshold = (long) (memUsage * Runtime.getRuntime().freeMemory()) - topN * 64L;
if (threshold < 0) {
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d6f9a8e9/ql/src/test/queries/clientpositive/topn.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/topn.q b/ql/src/test/queries/clientpositive/topn.q
new file mode 100644
index 0000000..4821b5a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/topn.q
@@ -0,0 +1,13 @@
+CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp` int , `salary` int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile;
+set hive.limit.pushdown.memory.usage=0.9999999;
+
+select * from sample_07 order by salary LIMIT 999999999;
+
+SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code;
+
+select sum(total_emp) over(partition by salary+salary order by code) from sample_07 limit 9999999;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/d6f9a8e9/ql/src/test/results/clientpositive/topn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/topn.q.out b/ql/src/test/results/clientpositive/topn.q.out
new file mode 100644
index 0000000..4846d8f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/topn.q.out
@@ -0,0 +1,42 @@
+PREHOOK: query: CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp` int , `salary` int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sample_07
+POSTHOOK: query: CREATE TABLE `sample_07` ( `code` string , `description` string , `total_emp` int , `salary` int ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sample_07
+PREHOOK: query: select * from sample_07 order by salary LIMIT 999999999
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: select * from sample_07 order by salary LIMIT 999999999
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+PREHOOK: query: SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM (
+SELECT *, rank() over(PARTITION BY code ORDER BY salary DESC) as rank
+FROM sample_07
+) ranked_claim
+WHERE ranked_claim.rank < 2
+ORDER BY code
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+PREHOOK: query: select sum(total_emp) over(partition by salary+salary order by code) from sample_07 limit 9999999
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sample_07
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(total_emp) over(partition by salary+salary order by code) from sample_07 limit 9999999
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sample_07
+#### A masked pattern was here ####