You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/06/20 08:39:10 UTC
git commit: [SQL] Improve Speed of InsertIntoHiveTable
Repository: spark
Updated Branches:
refs/heads/master 278ec8a20 -> d3b7671c1
[SQL] Improve Speed of InsertIntoHiveTable
Author: Michael Armbrust <mi...@databricks.com>
Closes #1130 from marmbrus/noFunctional and squashes the following commits:
ccdb68c [Michael Armbrust] Remove functional programming and Array allocations from fast path in InsertIntoHiveTable.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3b7671c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3b7671c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3b7671c
Branch: refs/heads/master
Commit: d3b7671c1f9c1eca956fda15fa7573649fd284b3
Parents: 278ec8a
Author: Michael Armbrust <mi...@databricks.com>
Authored: Thu Jun 19 23:39:03 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Thu Jun 19 23:39:03 2014 -0700
----------------------------------------------------------------------
.../spark/sql/hive/execution/hiveOperators.scala | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d3b7671c/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala
index 240aa0d..b195793 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala
@@ -371,12 +371,18 @@ case class InsertIntoHiveTable(
ObjectInspectorCopyOption.JAVA)
.asInstanceOf[StructObjectInspector]
+
+ val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray
+ val outputData = new Array[Any](fieldOIs.length)
iter.map { row =>
- // Casts Strings to HiveVarchars when necessary.
- val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector)
- val mappedRow = row.zip(fieldOIs).map(wrap)
+ var i = 0
+ while (i < row.length) {
+ // Casts Strings to HiveVarchars when necessary.
+ outputData(i) = wrap(row(i), fieldOIs(i))
+ i += 1
+ }
- serializer.serialize(mappedRow.toArray, standardOI)
+ serializer.serialize(outputData, standardOI)
}
}