You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2020/04/29 09:15:11 UTC
[hive] branch master updated: HIVE-23265: Duplicate rowsets are
returned with Limit and Offset set (Attila Magyar reviewed by Laszlo Bodor
and Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5d478a8 HIVE-23265: Duplicate rowsets are returned with Limit and Offset set (Attila Magyar reviewed by Laszlo Bodor and Peter Vary)
5d478a8 is described below
commit 5d478a88fcb118cc33498f6709cda36264622c89
Author: Attila Magyar <am...@cloudera.com>
AuthorDate: Wed Apr 29 11:14:01 2020 +0200
HIVE-23265: Duplicate rowsets are returned with Limit and Offset set (Attila Magyar reviewed by Laszlo Bodor and Peter Vary)
---
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/exec/vector/VectorLimitOperator.java | 8 ++--
.../clientpositive/vector_offset_limit_reduce.q | 16 +++++++
.../llap/vector_offset_limit_reduce.q.out | 51 ++++++++++++++++++++++
4 files changed, 73 insertions(+), 3 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index c55f8db..48ecc35 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -926,6 +926,7 @@ minillaplocal.query.files=\
vector_null_map.q,\
vector_number_compare_projection.q,\
vector_orc_merge_incompat_schema.q,\
+ vector_offset_limit_reduce.q,\
vector_orc_nested_column_pruning.q,\
vector_orc_null_check.q,\
vector_order_null.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java
index 79b073f..528b8f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorLimitOperator.java
@@ -79,18 +79,20 @@ public class VectorLimitOperator extends LimitOperator implements VectorizationO
}
//skip skipSize rows of batch
batch.size = Math.min(batch.size, offset + limit - currCount);
+ int newBatchSize = batch.size - skipSize;
if (batch.selectedInUse == false) {
batch.selectedInUse = true;
- for (int i = 0; i < batch.size - skipSize; i++) {
+ for (int i = 0; i < newBatchSize; i++) {
batch.selected[i] = skipSize + i;
}
} else {
- for (int i = 0; i < batch.size - skipSize; i++) {
+ for (int i = 0; i < newBatchSize; i++) {
batch.selected[i] = batch.selected[skipSize + i];
}
}
- vectorForward(batch);
currCount += batch.size;
+ batch.size = newBatchSize;
+ vectorForward(batch);
}
}
diff --git a/ql/src/test/queries/clientpositive/vector_offset_limit_reduce.q b/ql/src/test/queries/clientpositive/vector_offset_limit_reduce.q
new file mode 100644
index 0000000..78cf5db
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_offset_limit_reduce.q
@@ -0,0 +1,16 @@
+set hive.vectorized.execution.reduce.enabled=true;
+
+CREATE EXTERNAL TABLE orderdatatest_ext (col1 int, col2 int);
+INSERT INTO orderdatatest_ext VALUES
+ (1,1),
+ (1,2),
+ (1,3),
+ (1,4),
+ (1,5),
+ (1,6),
+ (1,7),
+ (1,8),
+ (1,9),
+ (1,10);
+
+select * from orderdatatest_ext order by col1 limit 5,100
diff --git a/ql/src/test/results/clientpositive/llap/vector_offset_limit_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_offset_limit_reduce.q.out
new file mode 100644
index 0000000..ed793f0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_offset_limit_reduce.q.out
@@ -0,0 +1,51 @@
+PREHOOK: query: CREATE EXTERNAL TABLE orderdatatest_ext (col1 int, col2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orderdatatest_ext
+POSTHOOK: query: CREATE EXTERNAL TABLE orderdatatest_ext (col1 int, col2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orderdatatest_ext
+PREHOOK: query: INSERT INTO orderdatatest_ext VALUES
+ (1,1),
+ (1,2),
+ (1,3),
+ (1,4),
+ (1,5),
+ (1,6),
+ (1,7),
+ (1,8),
+ (1,9),
+ (1,10)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@orderdatatest_ext
+POSTHOOK: query: INSERT INTO orderdatatest_ext VALUES
+ (1,1),
+ (1,2),
+ (1,3),
+ (1,4),
+ (1,5),
+ (1,6),
+ (1,7),
+ (1,8),
+ (1,9),
+ (1,10)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@orderdatatest_ext
+POSTHOOK: Lineage: orderdatatest_ext.col1 SCRIPT []
+POSTHOOK: Lineage: orderdatatest_ext.col2 SCRIPT []
+PREHOOK: query: select * from orderdatatest_ext order by col1 limit 5,100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orderdatatest_ext
+#### A masked pattern was here ####
+POSTHOOK: query: select * from orderdatatest_ext order by col1 limit 5,100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orderdatatest_ext
+#### A masked pattern was here ####
+1 6
+1 7
+1 8
+1 9
+1 10