You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/08/04 00:08:54 UTC
spark git commit: [SPARK-16814][SQL] Fix deprecated parquet
constructor usage
Repository: spark
Updated Branches:
refs/heads/master 4775eb414 -> c5eb1df72
[SPARK-16814][SQL] Fix deprecated parquet constructor usage
## What changes were proposed in this pull request?
Replace deprecated ParquetWriter with the new builders
## How was this patch tested?
Existing tests
Author: Holden Karau <ho...@us.ibm.com>
Closes #14419 from holdenk/SPARK-16814-fix-deprecated-parquet-constructor-usage.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c5eb1df7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c5eb1df7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c5eb1df7
Branch: refs/heads/master
Commit: c5eb1df72fea2ecc71369f13416a8aee040b55d2
Parents: 4775eb4
Author: Holden Karau <ho...@us.ibm.com>
Authored: Wed Aug 3 17:08:51 2016 -0700
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Aug 3 17:08:51 2016 -0700
----------------------------------------------------------------------
.../parquet/ParquetAvroCompatibilitySuite.scala | 5 +++--
.../parquet/ParquetCompatibilityTest.scala | 14 ++++++++++++--
.../datasources/parquet/ParquetIOSuite.scala | 16 ++++++++++++++--
3 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
index 6509e04..1b99fbe 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
@@ -27,6 +27,7 @@ import org.apache.avro.Schema
import org.apache.avro.generic.IndexedRecord
import org.apache.hadoop.fs.Path
import org.apache.parquet.avro.AvroParquetWriter
+import org.apache.parquet.hadoop.ParquetWriter
import org.apache.spark.sql.Row
import org.apache.spark.sql.execution.datasources.parquet.test.avro._
@@ -35,14 +36,14 @@ import org.apache.spark.sql.test.SharedSQLContext
class ParquetAvroCompatibilitySuite extends ParquetCompatibilityTest with SharedSQLContext {
private def withWriter[T <: IndexedRecord]
(path: String, schema: Schema)
- (f: AvroParquetWriter[T] => Unit): Unit = {
+ (f: ParquetWriter[T] => Unit): Unit = {
logInfo(
s"""Writing Avro records with the following Avro schema into Parquet file:
|
|${schema.toString(true)}
""".stripMargin)
- val writer = new AvroParquetWriter[T](new Path(path), schema)
+ val writer = AvroParquetWriter.builder[T](new Path(path)).withSchema(schema).build()
try f(writer) finally writer.close()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
index 57cd70e..a43a856 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
@@ -119,8 +119,18 @@ private[sql] object ParquetCompatibilityTest {
metadata: Map[String, String],
recordWriters: (RecordConsumer => Unit)*): Unit = {
val messageType = MessageTypeParser.parseMessageType(schema)
- val writeSupport = new DirectWriteSupport(messageType, metadata)
- val parquetWriter = new ParquetWriter[RecordConsumer => Unit](new Path(path), writeSupport)
+ val testWriteSupport = new DirectWriteSupport(messageType, metadata)
+ /**
+ * Provide a builder for constructing a parquet writer - after PARQUET-248 directly constructing
+ * the writer is deprecated and should be done through a builder. The default builders include
+ * Avro - but for raw Parquet writing we must create our own builder.
+ */
+ class ParquetWriterBuilder() extends
+ ParquetWriter.Builder[RecordConsumer => Unit, ParquetWriterBuilder](new Path(path)) {
+ override def getWriteSupport(conf: Configuration) = testWriteSupport
+ override def self() = this
+ }
+ val parquetWriter = new ParquetWriterBuilder().build()
try recordWriters.foreach(parquetWriter.write) finally parquetWriter.close()
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c5eb1df7/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index fc9ce6b..0f74094 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -325,8 +325,20 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
|}
""".stripMargin)
- val writeSupport = new TestGroupWriteSupport(schema)
- val writer = new ParquetWriter[Group](path, writeSupport)
+ val testWriteSupport = new TestGroupWriteSupport(schema)
+ /**
+ * Provide a builder for constructing a parquet writer - after PARQUET-248 directly
+ * constructing the writer is deprecated and should be done through a builder. The default
+ * builders include Avro - but for raw Parquet writing we must create our own builder.
+ */
+ class ParquetWriterBuilder() extends
+ ParquetWriter.Builder[Group, ParquetWriterBuilder](path) {
+ override def getWriteSupport(conf: Configuration) = testWriteSupport
+
+ override def self() = this
+ }
+
+ val writer = new ParquetWriterBuilder().build()
(0 until 10).foreach { i =>
val record = new SimpleGroup(schema)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org