You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2009/08/05 07:59:40 UTC
svn commit: r801068 - in /hadoop/hive/trunk: ./
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/
Author: namit
Date: Wed Aug 5 05:59:39 2009
New Revision: 801068
URL: http://svn.apache.org/viewvc?rev=801068&view=rev
Log:
HIVE-719. Lazy Array/Map index out of bound bug.
(Zheng Shao via namit)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_lazyserde.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Aug 5 05:59:39 2009
@@ -455,6 +455,9 @@
HIVE-710. Fix serialization bug in BinarySortableSerDe for negative double
and float numbers (Zheng Shao via pchakka)
+ HIVE-719. Lazy Array/Map index out of bound bug.
+ (Zheng Shao via namit)
+
Release 0.3.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_lazyserde.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_lazyserde.q?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_lazyserde.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input_lazyserde.q Wed Aug 5 05:59:39 2009
@@ -16,3 +16,15 @@
SELECT dest1.* FROM dest1 DISTRIBUTE BY 1;
SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1 DISTRIBUTE BY 1;
+
+DROP TABLE dest1;
+
+CREATE TABLE dest1(a array<int>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\';
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint FROM src_thrift DISTRIBUTE BY 1;
+SELECT * from dest1;
+DROP TABLE dest1;
+
+CREATE TABLE dest1(a map<string,string>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\';
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1;
+SELECT * from dest1;
+DROP TABLE dest1;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out Wed Aug 5 05:59:39 2009
@@ -78,7 +78,7 @@
Output: default/dest1
query: SELECT dest1.* FROM dest1 DISTRIBUTE BY 1
Input: default/dest1
-Output: file:/data/users/zshao/tools/670-trunk-apache-hive/.ptest_2/build/ql/tmp/1198263956/10000
+Output: file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/604433188/10000
[0,0,0] ["0","0","0"] {"key_0":"value_0"} 1712634731 record_0
[1,2,3] ["10","100","1000"] {"key_1":"value_1"} 465985200 record_1
[2,4,6] ["20","200","2000"] {"key_2":"value_2"} -751827638 record_2
@@ -92,7 +92,7 @@
null null null 0 NULL
query: SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1 DISTRIBUTE BY 1
Input: default/dest1
-Output: file:/data/users/zshao/tools/670-trunk-apache-hive/.ptest_2/build/ql/tmp/1682820914/10000
+Output: file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/848270212/10000
0 0 NULL 1712634731 record_0
1 10 NULL 465985200 record_1
2 20 NULL -751827638 record_2
@@ -104,3 +104,42 @@
8 80 NULL 1638581578 record_8
9 90 NULL 336964413 record_9
NULL NULL NULL 0 NULL
+query: DROP TABLE dest1
+query: CREATE TABLE dest1(a array<int>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint FROM src_thrift DISTRIBUTE BY 1
+Input: default/src_thrift
+Output: default/dest1
+query: SELECT * from dest1
+Input: default/dest1
+Output: file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1928951519/10000
+[0,0,0]
+[1,2,3]
+[2,4,6]
+[3,6,9]
+[4,8,12]
+[5,10,15]
+[6,12,18]
+[7,14,21]
+[8,16,24]
+[9,18,27]
+null
+query: DROP TABLE dest1
+query: CREATE TABLE dest1(a map<string,string>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1
+Input: default/src_thrift
+Output: default/dest1
+query: SELECT * from dest1
+Input: default/dest1
+Output: file:/data/users/zshao/tools/deploy-trunk-apache-hive/build/ql/tmp/1672243759/10000
+{"key_0":"value_0"}
+{"key_1":"value_1"}
+{"key_2":"value_2"}
+{"key_3":"value_3"}
+{"key_4":"value_4"}
+{"key_5":"value_5"}
+{"key_6":"value_6"}
+{"key_7":"value_7"}
+{"key_8":"value_8"}
+{"key_9":"value_9"}
+null
+query: DROP TABLE dest1
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java Wed Aug 5 05:59:39 2009
@@ -135,20 +135,21 @@
startPosition[arrayLength] = elementByteBegin;
arrayLength++;
elementByteBegin = elementByteEnd + 1;
- }
- if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
- // ignore the char after escape_char
- elementByteEnd += 2;
- } else {
elementByteEnd ++;
- }
-
+ } else {
+ if (isEscaped && bytes[elementByteEnd] == escapeChar
+ && elementByteEnd+1 < arrayByteEnd) {
+ // ignore the char after escape_char
+ elementByteEnd += 2;
+ } else {
+ elementByteEnd ++;
+ }
+ }
}
// Store arrayByteEnd+1 in startPosition[arrayLength]
// so that we can use the same formula to compute the length of
// each element in the array: startPosition[i+1] - startPosition[i] - 1
- startPosition[arrayLength] = elementByteEnd;
+ startPosition[arrayLength] = arrayByteEnd + 1;
if (arrayLength > 0) {
Arrays.fill(elementInited, 0, arrayLength, false);
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Wed Aug 5 05:59:39 2009
@@ -163,18 +163,20 @@
keyValueSeparatorPosition = -1;
mapSize++;
elementByteBegin = elementByteEnd + 1;
- }
- // Is this the first keyValueSeparator in this entry?
- if (keyValueSeparatorPosition == -1 && elementByteEnd != arrayByteEnd
- && bytes[elementByteEnd] == keyValueSeparator) {
- keyValueSeparatorPosition = elementByteEnd;
- }
- if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
- // ignore the char after escape_char
- elementByteEnd += 2;
- } else {
elementByteEnd ++;
+ } else {
+ // Is this the first keyValueSeparator in this entry?
+ if (keyValueSeparatorPosition == -1
+ && bytes[elementByteEnd] == keyValueSeparator) {
+ keyValueSeparatorPosition = elementByteEnd;
+ }
+ if (isEscaped && bytes[elementByteEnd] == escapeChar
+ && elementByteEnd+1 < arrayByteEnd) {
+ // ignore the char after escape_char
+ elementByteEnd += 2;
+ } else {
+ elementByteEnd ++;
+ }
}
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java?rev=801068&r1=801067&r2=801068&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java Wed Aug 5 05:59:39 2009
@@ -131,13 +131,15 @@
break;
}
fieldByteBegin = fieldByteEnd + 1;
- }
- if (isEscaped && bytes[fieldByteEnd] == escapeChar
- && fieldByteEnd+1 < structByteEnd) {
- // ignore the char after escape_char
- fieldByteEnd += 2;
+ fieldByteEnd ++;
} else {
- fieldByteEnd++;
+ if (isEscaped && bytes[fieldByteEnd] == escapeChar
+ && fieldByteEnd+1 < structByteEnd) {
+ // ignore the char after escape_char
+ fieldByteEnd += 2;
+ } else {
+ fieldByteEnd++;
+ }
}
}