You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/06/24 04:53:47 UTC
spark git commit: [SPARK-20555][SQL] Fix mapping of Oracle DECIMAL
types to Spark types in read path
Repository: spark
Updated Branches:
refs/heads/master 7525ce98b -> b837bf9ae
[SPARK-20555][SQL] Fix mapping of Oracle DECIMAL types to Spark types in read path
## What changes were proposed in this pull request?
This PR is to revert some code changes in the read path of https://github.com/apache/spark/pull/14377. The original fix is https://github.com/apache/spark/pull/17830
When merging this PR, please give the credit to gaborfeher
## How was this patch tested?
Added a test case to OracleIntegrationSuite.scala
Author: Gabor Feher <ga...@lynxanalytics.com>
Author: gatorsmile <ga...@gmail.com>
Closes #18408 from gatorsmile/OracleType.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b837bf9a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b837bf9a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b837bf9a
Branch: refs/heads/master
Commit: b837bf9ae97cf7ee7558c10a5a34636e69367a05
Parents: 7525ce9
Author: Gabor Feher <ga...@lynxanalytics.com>
Authored: Fri Jun 23 21:53:38 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Fri Jun 23 21:53:38 2017 -0700
----------------------------------------------------------------------
.../spark/sql/jdbc/OracleIntegrationSuite.scala | 65 ++++++++++++++------
.../apache/spark/sql/jdbc/OracleDialect.scala | 4 --
2 files changed, 45 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b837bf9a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
----------------------------------------------------------------------
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index f7b1ec3..b2f0969 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.jdbc
import java.sql.{Connection, Date, Timestamp}
import java.util.Properties
+import java.math.BigDecimal
import org.apache.spark.sql.Row
import org.apache.spark.sql.test.SharedSQLContext
@@ -93,8 +94,31 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
|USING org.apache.spark.sql.jdbc
|OPTIONS (url '$jdbcUrl', dbTable 'datetime1', oracle.jdbc.mapDateToTimestamp 'false')
""".stripMargin.replaceAll("\n", " "))
+
+
+ conn.prepareStatement("CREATE TABLE numerics (b DECIMAL(1), f DECIMAL(3, 2), i DECIMAL(10))").executeUpdate();
+ conn.prepareStatement(
+ "INSERT INTO numerics VALUES (4, 1.23, 9999999999)").executeUpdate();
+ conn.commit();
}
+
+ test("SPARK-16625 : Importing Oracle numeric types") {
+ val df = sqlContext.read.jdbc(jdbcUrl, "numerics", new Properties);
+ val rows = df.collect()
+ assert(rows.size == 1)
+ val row = rows(0)
+ // The main point of the below assertions is not to make sure that these Oracle types are
+ // mapped to decimal types, but to make sure that the returned values are correct.
+ // A value > 1 from DECIMAL(1) is correct:
+ assert(row.getDecimal(0).compareTo(BigDecimal.valueOf(4)) == 0)
+ // A value with fractions from DECIMAL(3, 2) is correct:
+ assert(row.getDecimal(1).compareTo(BigDecimal.valueOf(1.23)) == 0)
+ // A value > Int.MaxValue from DECIMAL(10) is correct:
+ assert(row.getDecimal(2).compareTo(BigDecimal.valueOf(9999999999l)) == 0)
+ }
+
+
test("SPARK-12941: String datatypes to be mapped to Varchar in Oracle") {
// create a sample dataframe with string type
val df1 = sparkContext.parallelize(Seq(("foo"))).toDF("x")
@@ -154,27 +178,28 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
val dfRead = spark.read.jdbc(jdbcUrl, tableName, props)
val rows = dfRead.collect()
// verify the data type is inserted
- val types = rows(0).toSeq.map(x => x.getClass.toString)
- assert(types(0).equals("class java.lang.Boolean"))
- assert(types(1).equals("class java.lang.Integer"))
- assert(types(2).equals("class java.lang.Long"))
- assert(types(3).equals("class java.lang.Float"))
- assert(types(4).equals("class java.lang.Float"))
- assert(types(5).equals("class java.lang.Integer"))
- assert(types(6).equals("class java.lang.Integer"))
- assert(types(7).equals("class java.lang.String"))
- assert(types(8).equals("class [B"))
- assert(types(9).equals("class java.sql.Date"))
- assert(types(10).equals("class java.sql.Timestamp"))
+ val types = dfRead.schema.map(field => field.dataType)
+ assert(types(0).equals(DecimalType(1, 0)))
+ assert(types(1).equals(DecimalType(10, 0)))
+ assert(types(2).equals(DecimalType(19, 0)))
+ assert(types(3).equals(DecimalType(19, 4)))
+ assert(types(4).equals(DecimalType(19, 4)))
+ assert(types(5).equals(DecimalType(3, 0)))
+ assert(types(6).equals(DecimalType(5, 0)))
+ assert(types(7).equals(StringType))
+ assert(types(8).equals(BinaryType))
+ assert(types(9).equals(DateType))
+ assert(types(10).equals(TimestampType))
+
// verify the value is the inserted correct or not
val values = rows(0)
- assert(values.getBoolean(0).equals(booleanVal))
- assert(values.getInt(1).equals(integerVal))
- assert(values.getLong(2).equals(longVal))
- assert(values.getFloat(3).equals(floatVal))
- assert(values.getFloat(4).equals(doubleVal.toFloat))
- assert(values.getInt(5).equals(byteVal.toInt))
- assert(values.getInt(6).equals(shortVal.toInt))
+ assert(values.getDecimal(0).compareTo(BigDecimal.valueOf(1)) == 0)
+ assert(values.getDecimal(1).compareTo(BigDecimal.valueOf(integerVal)) == 0)
+ assert(values.getDecimal(2).compareTo(BigDecimal.valueOf(longVal)) == 0)
+ assert(values.getDecimal(3).compareTo(BigDecimal.valueOf(floatVal)) == 0)
+ assert(values.getDecimal(4).compareTo(BigDecimal.valueOf(doubleVal)) == 0)
+ assert(values.getDecimal(5).compareTo(BigDecimal.valueOf(byteVal)) == 0)
+ assert(values.getDecimal(6).compareTo(BigDecimal.valueOf(shortVal)) == 0)
assert(values.getString(7).equals(stringVal))
assert(values.getAs[Array[Byte]](8).mkString.equals("678"))
assert(values.getDate(9).equals(dateVal))
@@ -183,7 +208,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
test("SPARK-19318: connection property keys should be case-sensitive") {
def checkRow(row: Row): Unit = {
- assert(row.getInt(0) == 1)
+ assert(row.getDecimal(0).equals(BigDecimal.valueOf(1)))
assert(row.getDate(1).equals(Date.valueOf("1991-11-09")))
assert(row.getTimestamp(2).equals(Timestamp.valueOf("1996-01-01 01:23:45")))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/b837bf9a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
index f541996..20e634c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
@@ -43,10 +43,6 @@ private case object OracleDialect extends JdbcDialect {
// Not sure if there is a more robust way to identify the field as a float (or other
// numeric types that do not specify a scale.
case _ if scale == -127L => Option(DecimalType(DecimalType.MAX_PRECISION, 10))
- case 1 => Option(BooleanType)
- case 3 | 5 | 10 => Option(IntegerType)
- case 19 if scale == 0L => Option(LongType)
- case 19 if scale == 4L => Option(FloatType)
case _ => None
}
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org