You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/02/12 19:35:00 UTC
spark git commit: [SPARK-23388][SQL] Support for Parquet Binary
DecimalType in VectorizedColumnReader
Repository: spark
Updated Branches:
refs/heads/master 4a4dd4f36 -> 5bb11411a
[SPARK-23388][SQL] Support for Parquet Binary DecimalType in VectorizedColumnReader
## What changes were proposed in this pull request?
Re-add support for parquet binary DecimalType in VectorizedColumnReader
## How was this patch tested?
Existing test suite
Author: James Thompson <ja...@users.noreply.github.com>
Closes #20580 from jamesthomp/jt/add-back-binary-decimal.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5bb11411
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5bb11411
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5bb11411
Branch: refs/heads/master
Commit: 5bb11411aec18b8d623e54caba5397d7cb8e89f0
Parents: 4a4dd4f
Author: James Thompson <ja...@users.noreply.github.com>
Authored: Mon Feb 12 11:34:56 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Feb 12 11:34:56 2018 -0800
----------------------------------------------------------------------
.../sql/execution/datasources/parquet/VectorizedColumnReader.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5bb11411/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index c120863..47dd625 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -444,7 +444,8 @@ public class VectorizedColumnReader {
// This is where we implement support for the valid type conversions.
// TODO: implement remaining type conversions
VectorizedValuesReader data = (VectorizedValuesReader) dataColumn;
- if (column.dataType() == DataTypes.StringType || column.dataType() == DataTypes.BinaryType) {
+ if (column.dataType() == DataTypes.StringType || column.dataType() == DataTypes.BinaryType
+ || DecimalType.isByteArrayDecimalType(column.dataType())) {
defColumn.readBinarys(num, column, rowId, maxDefLevel, data);
} else if (column.dataType() == DataTypes.TimestampType) {
if (!shouldConvertTimestamps()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org