You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/11/18 01:35:51 UTC
spark git commit: [SPARK-4448] [SQL] unwrap for the
ConstantObjectInspector
Repository: spark
Updated Branches:
refs/heads/master 42389b178 -> ef7c464ef
[SPARK-4448] [SQL] unwrap for the ConstantObjectInspector
Author: Cheng Hao <ha...@intel.com>
Closes #3308 from chenghao-intel/unwrap_constant_oi and squashes the following commits:
156b500 [Cheng Hao] rebase the master
c5b20ab [Cheng Hao] unwrap for the ConstantObjectInspector
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ef7c464e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ef7c464e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ef7c464e
Branch: refs/heads/master
Commit: ef7c464effa1510b24bd8e665e4df6c4839b0c87
Parents: 42389b1
Author: Cheng Hao <ha...@intel.com>
Authored: Mon Nov 17 16:35:49 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Nov 17 16:35:49 2014 -0800
----------------------------------------------------------------------
.../apache/spark/sql/hive/HiveInspectors.scala | 36 +++++++++++++++++---
1 file changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ef7c464e/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 7e76aff..ada980a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -86,13 +86,41 @@ private[hive] trait HiveInspectors {
* @param data the data in Hive type
* @param oi the ObjectInspector associated with the Hive Type
* @return convert the data into catalyst type
+ * TODO return the function of (data => Any) instead for performance consideration
*/
def unwrap(data: Any, oi: ObjectInspector): Any = oi match {
case _ if data == null => null
- case hvoi: HiveVarcharObjectInspector =>
- if (data == null) null else hvoi.getPrimitiveJavaObject(data).getValue
- case hdoi: HiveDecimalObjectInspector =>
- if (data == null) null else HiveShim.toCatalystDecimal(hdoi, data)
+ case poi: VoidObjectInspector => null
+ case poi: WritableConstantHiveVarcharObjectInspector =>
+ poi.getWritableConstantValue.getHiveVarchar.getValue
+ case poi: WritableConstantHiveDecimalObjectInspector =>
+ HiveShim.toCatalystDecimal(
+ PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector,
+ poi.getWritableConstantValue.getHiveDecimal)
+ case poi: WritableConstantTimestampObjectInspector =>
+ poi.getWritableConstantValue.getTimestamp.clone()
+ case poi: WritableConstantIntObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantDoubleObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantBooleanObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantLongObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantFloatObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantShortObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantByteObjectInspector =>
+ poi.getWritableConstantValue.get()
+ case poi: WritableConstantBinaryObjectInspector =>
+ val writable = poi.getWritableConstantValue
+ val temp = new Array[Byte](writable.getLength)
+ System.arraycopy(writable.getBytes, 0, temp, 0, temp.length)
+ temp
+ case poi: WritableConstantDateObjectInspector => poi.getWritableConstantValue.get()
+ case hvoi: HiveVarcharObjectInspector => hvoi.getPrimitiveJavaObject(data).getValue
+ case hdoi: HiveDecimalObjectInspector => HiveShim.toCatalystDecimal(hdoi, data)
// org.apache.hadoop.hive.serde2.io.TimestampWritable.set will reset current time object
// if next timestamp is null, so Timestamp object is cloned
case ti: TimestampObjectInspector => ti.getPrimitiveJavaObject(data).clone()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org