You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2014/09/19 00:56:13 UTC
svn commit: r1626101 - in /hive/trunk: data/files/
itests/hive-unit/src/test/java/org/apache/hive/jdbc/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/
serde/src/java/org/apache...
Author: jdere
Date: Thu Sep 18 22:56:12 2014
New Revision: 1626101
URL: http://svn.apache.org/r1626101
Log:
HIVE-8047: Lazy char/varchar are not using escape char defined in serde params (Jason Dere, reviewed by Thejas Nair)
Added:
hive/trunk/data/files/data_with_escape.txt
hive/trunk/ql/src/test/queries/clientpositive/escape3.q
hive/trunk/ql/src/test/results/clientpositive/escape3.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java
Added: hive/trunk/data/files/data_with_escape.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/data_with_escape.txt?rev=1626101&view=auto
==============================================================================
--- hive/trunk/data/files/data_with_escape.txt (added)
+++ hive/trunk/data/files/data_with_escape.txt Thu Sep 18 22:56:12 2014
@@ -0,0 +1,5 @@
+re\|ading|V\|A|100
+writ\|ing|MD|200
+w\|aiting|\|NC|300
+seein\|g|TN\||400
+runn\|ing|WV|500
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Thu Sep 18 22:56:12 2014
@@ -735,7 +735,7 @@ public class TestJdbcDriver2 {
assertTrue(res.next());
// skip the last (partitioning) column since it is always non-null
for (int i = 1; i < meta.getColumnCount(); i++) {
- assertNull(res.getObject(i));
+ assertNull("Column " + i + " should be null", res.getObject(i));
}
// getXXX returns 0 for numeric types, false for boolean and null for other
assertEquals(0, res.getInt(1));
Added: hive/trunk/ql/src/test/queries/clientpositive/escape3.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/escape3.q?rev=1626101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/escape3.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/escape3.q Thu Sep 18 22:56:12 2014
@@ -0,0 +1,48 @@
+-- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1;
+
+select * from escape3_1;
+
+-- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2;
+
+select * from escape3_2;
+
+-- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3;
+
+select * from escape3_3;
+
+DROP TABLE escape3_1;
+DROP TABLE escape3_2;
+DROP TABLE escape3_3;
Added: hive/trunk/ql/src/test/results/clientpositive/escape3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/escape3.q.out?rev=1626101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/escape3.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/escape3.q.out Thu Sep 18 22:56:12 2014
@@ -0,0 +1,165 @@
+PREHOOK: query: -- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: -- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: select * from escape3_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_1
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: -- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: -- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: select * from escape3_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_2
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: -- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: -- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_3
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_3
+PREHOOK: query: select * from escape3_3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_3
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: DROP TABLE escape3_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_1
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: DROP TABLE escape3_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_1
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: DROP TABLE escape3_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_2
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: DROP TABLE escape3_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_2
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: DROP TABLE escape3_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_3
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: DROP TABLE escape3_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_3
+POSTHOOK: Output: default@escape3_3
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java Thu Sep 18 22:56:12 2014
@@ -55,14 +55,24 @@ public class LazyHiveChar extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ } else {
+ String byteData = null;
+ try {
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ }
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java Thu Sep 18 22:56:12 2014
@@ -55,14 +55,24 @@ public class LazyHiveVarchar extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ } else {
+ try {
+ String byteData = null;
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ }
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Thu Sep 18 22:56:12 2014
@@ -40,40 +40,7 @@ public class LazyString extends LazyPrim
if (oi.isEscaped()) {
byte escapeChar = oi.getEscapeChar();
byte[] inputBytes = bytes.getData();
-
- // First calculate the length of the output string
- int outputLength = 0;
- for (int i = 0; i < length; i++) {
- if (inputBytes[start + i] != escapeChar) {
- outputLength++;
- } else {
- outputLength++;
- i++;
- }
- }
-
- // Copy the data over, so that the internal state of Text will be set to
- // the required outputLength.
- data.set(bytes.getData(), start, outputLength);
-
- // We need to copy the data byte by byte only in case the
- // "outputLength < length" (which means there is at least one escaped
- // byte.
- if (outputLength < length) {
- int k = 0;
- byte[] outputBytes = data.getBytes();
- for (int i = 0; i < length; i++) {
- byte b = inputBytes[start + i];
- if (b != escapeChar || i == length - 1) {
- outputBytes[k++] = b;
- } else {
- // get the next byte
- i++;
- outputBytes[k++] = inputBytes[start + i];
- }
- }
- assert (k == outputLength);
- }
+ LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data);
} else {
// if the data is not escaped, simply copy the data.
data.set(bytes.getData(), start, length);
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Thu Sep 18 22:56:12 2014
@@ -412,6 +412,44 @@ public final class LazyUtils {
}
}
+ public static void copyAndEscapeStringDataToText(byte[] inputBytes, int start, int length,
+ byte escapeChar, Text data) {
+
+ // First calculate the length of the output string
+ int outputLength = 0;
+ for (int i = 0; i < length; i++) {
+ if (inputBytes[start + i] != escapeChar) {
+ outputLength++;
+ } else {
+ outputLength++;
+ i++;
+ }
+ }
+
+ // Copy the data over, so that the internal state of Text will be set to
+ // the required outputLength.
+ data.set(inputBytes, start, outputLength);
+
+ // We need to copy the data byte by byte only in case the
+ // "outputLength < length" (which means there is at least one escaped
+ // byte.
+ if (outputLength < length) {
+ int k = 0;
+ byte[] outputBytes = data.getBytes();
+ for (int i = 0; i < length; i++) {
+ byte b = inputBytes[start + i];
+ if (b != escapeChar || i == length - 1) {
+ outputBytes[k++] = b;
+ } else {
+ // get the next byte
+ i++;
+ outputBytes[k++] = inputBytes[start + i];
+ }
+ }
+ assert (k == outputLength);
+ }
+ }
+
private LazyUtils() {
// prevent instantiation
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -29,12 +29,21 @@ public class LazyHiveCharObjectInspector
extends AbstractPrimitiveLazyObjectInspector<HiveCharWritable>
implements HiveCharObjectInspector {
+ private boolean escaped;
+ private byte escapeChar;
+
// no-arg ctor required for Kyro
public LazyHiveCharObjectInspector() {
}
public LazyHiveCharObjectInspector(CharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveCharObjectInspector(CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
}
@Override
@@ -63,6 +72,14 @@ public class LazyHiveCharObjectInspector
return ret;
}
+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
@Override
public String toString() {
return getTypeName();
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -29,12 +29,21 @@ public class LazyHiveVarcharObjectInspec
extends AbstractPrimitiveLazyObjectInspector<HiveVarcharWritable>
implements HiveVarcharObjectInspector {
+ private boolean escaped;
+ private byte escapeChar;
+
// no-arg ctor required for Kyro
public LazyHiveVarcharObjectInspector() {
}
public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
}
@Override
@@ -63,6 +72,14 @@ public class LazyHiveVarcharObjectInspec
return ret;
}
+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
@Override
public String toString() {
return getTypeName();
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Thu Sep 18 22:56:12 2014
@@ -78,8 +78,10 @@ public final class LazyPrimitiveObjectIn
// prevent instantiation
}
- private static HashMap<ArrayList<Object>, LazyStringObjectInspector> cachedLazyStringObjectInspector =
- new HashMap<ArrayList<Object>, LazyStringObjectInspector>();
+ // Lazy object inspectors for string/char/varchar will all be cached in the same map.
+ // Map key will be list of [typeInfo, isEscaped, escapeChar]
+ private static HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector> cachedLazyStringTypeOIs =
+ new HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector>();
private static Map<PrimitiveTypeInfo, AbstractPrimitiveLazyObjectInspector<?>>
cachedPrimitiveLazyObjectInspectors =
@@ -121,6 +123,10 @@ public final class LazyPrimitiveObjectIn
switch(primitiveCategory) {
case STRING:
return getLazyStringObjectInspector(escaped, escapeChar);
+ case CHAR:
+ return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar);
+ case VARCHAR:
+ return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar);
case BOOLEAN:
return getLazyBooleanObjectInspector(extBoolean);
default:
@@ -157,13 +163,44 @@ public final class LazyPrimitiveObjectIn
public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) {
ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(TypeInfoFactory.stringTypeInfo);
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazyStringObjectInspector result = cachedLazyStringObjectInspector
+ LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs
.get(signature);
if (result == null) {
result = new LazyStringObjectInspector(escaped, escapeChar);
- cachedLazyStringObjectInspector.put(signature, result);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveCharObjectInspector getLazyHiveCharObjectInspector(
+ CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveCharObjectInspector result = (LazyHiveCharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveCharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveVarcharObjectInspector getLazyHiveVarcharObjectInspector(
+ VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveVarcharObjectInspector result = (LazyHiveVarcharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveVarcharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
}
return result;
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -91,6 +91,9 @@ public class WritableHiveCharObjectInspe
@Override
public Object set(Object o, HiveChar value) {
+ if (value == null) {
+ return null;
+ }
HiveCharWritable writable = (HiveCharWritable) o;
writable.set(value, getMaxLength());
return o;
@@ -98,6 +101,9 @@ public class WritableHiveCharObjectInspe
@Override
public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
HiveCharWritable writable = (HiveCharWritable) o;
writable.set(value, getMaxLength());
return o;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -96,6 +96,9 @@ implements SettableHiveVarcharObjectInsp
@Override
public Object set(Object o, HiveVarchar value) {
+ if (value == null) {
+ return null;
+ }
HiveVarcharWritable writable = (HiveVarcharWritable)o;
writable.set(value, getMaxLength());
return o;
@@ -103,6 +106,9 @@ implements SettableHiveVarcharObjectInsp
@Override
public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
HiveVarcharWritable writable = (HiveVarcharWritable)o;
writable.set(value, getMaxLength());
return o;