You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2017/05/19 12:17:17 UTC
spark git commit: [SPARK-20773][SQL] ParquetWriteSupport.writeFields
is quadratic in number of fields
Repository: spark
Updated Branches:
refs/heads/master ce8edb8bf -> 3f2cd51ee
[SPARK-20773][SQL] ParquetWriteSupport.writeFields is quadratic in number of fields
Fix quadratic List indexing in ParquetWriteSupport.
I noticed this function while profiling some code with today. It showed up as a significant factor in a table with twenty columns; with hundreds of columns, it could dominate any other function call.
## What changes were proposed in this pull request?
The writeFields method iterates from 0 until number of fields, indexing into rootFieldWriters for each element. rootFieldWriters is a List, so indexing is a linear operation. The complexity of the writeFields method is thus quadratic in the number of fields.
Solution: explicitly convert rootFieldWriters to Array (implicitly converted to WrappedArray) for constant-time indexing.
## How was this patch tested?
This is a one-line change for performance reasons.
Author: tpoterba <tp...@broadinstitute.org>
Author: Tim Poterba <tp...@gmail.com>
Closes #18005 from tpoterba/tpoterba-patch-1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3f2cd51e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3f2cd51e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3f2cd51e
Branch: refs/heads/master
Commit: 3f2cd51ee06f2c6d735754e5440bc4b74f8dcbc8
Parents: ce8edb8
Author: tpoterba <tp...@broadinstitute.org>
Authored: Fri May 19 14:17:12 2017 +0200
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Fri May 19 14:17:12 2017 +0200
----------------------------------------------------------------------
.../execution/datasources/parquet/ParquetWriteSupport.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3f2cd51e/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index 38b0e33..63a8666 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -58,7 +58,7 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
private var schema: StructType = _
// `ValueWriter`s for all fields of the schema
- private var rootFieldWriters: Seq[ValueWriter] = _
+ private var rootFieldWriters: Array[ValueWriter] = _
// The Parquet `RecordConsumer` to which all `InternalRow`s are written
private var recordConsumer: RecordConsumer = _
@@ -90,7 +90,7 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
}
- this.rootFieldWriters = schema.map(_.dataType).map(makeWriter)
+ this.rootFieldWriters = schema.map(_.dataType).map(makeWriter).toArray[ValueWriter]
val messageType = new ParquetSchemaConverter(configuration).convert(schema)
val metadata = Map(ParquetReadSupport.SPARK_METADATA_KEY -> schemaString).asJava
@@ -116,7 +116,7 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
}
private def writeFields(
- row: InternalRow, schema: StructType, fieldWriters: Seq[ValueWriter]): Unit = {
+ row: InternalRow, schema: StructType, fieldWriters: Array[ValueWriter]): Unit = {
var i = 0
while (i < row.numFields) {
if (!row.isNullAt(i)) {
@@ -192,7 +192,7 @@ private[parquet] class ParquetWriteSupport extends WriteSupport[InternalRow] wit
makeDecimalWriter(precision, scale)
case t: StructType =>
- val fieldWriters = t.map(_.dataType).map(makeWriter)
+ val fieldWriters = t.map(_.dataType).map(makeWriter).toArray[ValueWriter]
(row: SpecializedGetters, ordinal: Int) =>
consumeGroup {
writeFields(row.getStruct(ordinal, t.length), t, fieldWriters)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org