You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2016/01/21 01:10:49 UTC
spark git commit: [SPARK-12921] Use SparkHadoopUtil reflection in
SpecificParquetRecordReaderBase
Repository: spark
Updated Branches:
refs/heads/branch-1.6 962e618ec -> 40fa21856
[SPARK-12921] Use SparkHadoopUtil reflection in SpecificParquetRecordReaderBase
It looks like there's one place left in the codebase, SpecificParquetRecordReaderBase, where we didn't use SparkHadoopUtil's reflective accesses of TaskAttemptContext methods, which could create problems when using a single Spark artifact with both Hadoop 1.x and 2.x.
Author: Josh Rosen <jo...@databricks.com>
Closes #10843 from JoshRosen/SPARK-12921.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/40fa2185
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/40fa2185
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/40fa2185
Branch: refs/heads/branch-1.6
Commit: 40fa21856aded0e8b0852cdc2d8f8bc577891908
Parents: 962e618
Author: Josh Rosen <jo...@databricks.com>
Authored: Wed Jan 20 16:10:28 2016 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Wed Jan 20 16:10:28 2016 -0800
----------------------------------------------------------------------
.../datasources/parquet/SpecificParquetRecordReaderBase.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/40fa2185/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
index 2ed30c1..bf9799b 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java
@@ -57,6 +57,8 @@ import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.ConfigurationUtil;
import org.apache.parquet.schema.MessageType;
+import org.apache.spark.deploy.SparkHadoopUtil;
+
/**
* Base class for custom RecordReaaders for Parquet that directly materialize to `T`.
* This class handles computing row groups, filtering on them, setting up the column readers,
@@ -81,7 +83,8 @@ public abstract class SpecificParquetRecordReaderBase<T> extends RecordReader<Vo
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException {
- Configuration configuration = taskAttemptContext.getConfiguration();
+ Configuration configuration =
+ SparkHadoopUtil.get().getConfigurationFromJobContext(taskAttemptContext);
ParquetInputSplit split = (ParquetInputSplit)inputSplit;
this.file = split.getPath();
long[] rowGroupOffsets = split.getRowGroupOffsets();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org