You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/03/04 11:59:45 UTC
spark git commit: [SPARK-12925] Improve HiveInspectors.unwrap for StringObjectInspector.…
Repository: spark
Updated Branches:
refs/heads/master c04dc27ce -> 204b02b56
[SPARK-12925] Improve HiveInspectors.unwrap for StringObjectInspector.…
Earlier fix did not copy the bytes and it is possible for higher level to reuse Text object. This was causing issues. Proposed fix now copies the bytes from Text. This still avoids the expensive encoding/decoding
Author: Rajesh Balamohan <rb...@apache.org>
Closes #11477 from rajeshbalamohan/SPARK-12925.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/204b02b5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/204b02b5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/204b02b5
Branch: refs/heads/master
Commit: 204b02b56afe358b7f2d403fb6e2b9e8a7122798
Parents: c04dc27
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Mar 4 10:59:40 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Mar 4 10:59:40 2016 +0000
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/hive/HiveInspectors.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/204b02b5/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 3e91569..589862c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -320,9 +320,10 @@ private[hive] trait HiveInspectors {
case hvoi: HiveCharObjectInspector =>
UTF8String.fromString(hvoi.getPrimitiveJavaObject(data).getValue)
case x: StringObjectInspector if x.preferWritable() =>
- // Text is in UTF-8 already. No need to convert again via fromString
+ // Text is in UTF-8 already. No need to convert again via fromString. Copy bytes
val wObj = x.getPrimitiveWritableObject(data)
- UTF8String.fromBytes(wObj.getBytes, 0, wObj.getLength)
+ val result = wObj.copyBytes()
+ UTF8String.fromBytes(result, 0, result.length)
case x: StringObjectInspector =>
UTF8String.fromString(x.getPrimitiveJavaObject(data))
case x: IntObjectInspector if x.preferWritable() => x.get(data)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org