You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/08/26 20:29:26 UTC
spark git commit: [SPARK-17246][SQL] Add BigDecimal literal
Repository: spark
Updated Branches:
refs/heads/master 8e5475be3 -> a11d10f18
[SPARK-17246][SQL] Add BigDecimal literal
## What changes were proposed in this pull request?
This PR adds parser support for `BigDecimal` literals. If you append the suffix `BD` to a valid number then this will be interpreted as a `BigDecimal`, for example `12.0E10BD` will interpreted into a BigDecimal with scale -9 and precision 3. This is useful in situations where you need exact values.
## How was this patch tested?
Added tests to `ExpressionParserSuite`, `ExpressionSQLBuilderSuite` and `SQLQueryTestSuite`.
Author: Herman van Hovell <hv...@databricks.com>
Closes #14819 from hvanhovell/SPARK-17246.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a11d10f1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a11d10f1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a11d10f1
Branch: refs/heads/master
Commit: a11d10f1826b578ff721c4738224eef2b3c3b9f3
Parents: 8e5475b
Author: Herman van Hovell <hv...@databricks.com>
Authored: Fri Aug 26 13:29:22 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Aug 26 13:29:22 2016 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 6 +++++
.../sql/catalyst/expressions/literals.scala | 2 +-
.../spark/sql/catalyst/parser/AstBuilder.scala | 16 ++++++++++++-
.../catalyst/parser/ExpressionParserSuite.scala | 7 ++++++
.../resources/sql-tests/inputs/literals.sql | 6 +++++
.../sql-tests/results/literals.sql.out | 24 +++++++++++++++++++-
.../catalyst/ExpressionSQLBuilderSuite.scala | 1 +
7 files changed, 59 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index cab7c3f..a8af840 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -633,6 +633,7 @@ number
| MINUS? SMALLINT_LITERAL #smallIntLiteral
| MINUS? TINYINT_LITERAL #tinyIntLiteral
| MINUS? DOUBLE_LITERAL #doubleLiteral
+ | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
;
nonReserved
@@ -928,6 +929,11 @@ DOUBLE_LITERAL
(INTEGER_VALUE | DECIMAL_VALUE | SCIENTIFIC_DECIMAL_VALUE) 'D'
;
+BIGDECIMAL_LITERAL
+ :
+ (INTEGER_VALUE | DECIMAL_VALUE | SCIENTIFIC_DECIMAL_VALUE) 'BD'
+ ;
+
IDENTIFIER
: (LETTER | DIGIT | '_')+
;
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 730a7f6..41e3952 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -266,7 +266,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression with
case Double.NegativeInfinity => s"CAST('-Infinity' AS ${DoubleType.sql})"
case _ => v + "D"
}
- case (v: Decimal, t: DecimalType) => s"CAST($v AS ${t.sql})"
+ case (v: Decimal, t: DecimalType) => v + "BD"
case (v: Int, DateType) => s"DATE '${DateTimeUtils.toJavaDate(v)}'"
case (v: Long, TimestampType) => s"TIMESTAMP('${DateTimeUtils.toJavaTimestamp(v)}')"
case _ => value.toString
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 8b98efc..893db93 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -26,7 +26,8 @@ import org.antlr.v4.runtime.{ParserRuleContext, Token}
import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
@@ -1324,6 +1325,19 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
}
/**
+ * Create a BigDecimal Literal expression.
+ */
+ override def visitBigDecimalLiteral(ctx: BigDecimalLiteralContext): Literal = {
+ val raw = ctx.getText.substring(0, ctx.getText.length - 2)
+ try {
+ Literal(BigDecimal(raw).underlying())
+ } catch {
+ case e: AnalysisException =>
+ throw new ParseException(e.message, ctx)
+ }
+ }
+
+ /**
* Create a String literal expression.
*/
override def visitStringLiteral(ctx: StringLiteralContext): Literal = withOrigin(ctx) {
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 401d9cd..dbc5db3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -392,6 +392,13 @@ class ExpressionParserSuite extends PlanTest {
intercept("1.8E308D", s"does not fit in range")
// TODO we need to figure out if we should throw an exception here!
assertEqual("1E309", Literal(Double.PositiveInfinity))
+
+ // BigDecimal Literal
+ assertEqual("90912830918230182310293801923652346786BD",
+ Literal(BigDecimal("90912830918230182310293801923652346786").underlying()))
+ assertEqual("123.0E-28BD", Literal(BigDecimal("123.0E-28").underlying()))
+ assertEqual("123.08BD", Literal(BigDecimal("123.08").underlying()))
+ intercept("1.20E-38BD", "DecimalType can only support precision up to 38")
}
test("strings") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/core/src/test/resources/sql-tests/inputs/literals.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
index 62f0d3d..a532a59 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/literals.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
@@ -90,3 +90,9 @@ select interval 10 nanoseconds;
-- unsupported data type
select GEO '(10,-6)';
+
+-- big decimal parsing
+select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD;
+
+-- out of range big decimal
+select 1.20E-38BD;
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/core/src/test/resources/sql-tests/results/literals.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
index 67e6d78..85629f7 100644
--- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 38
+-- Number of queries: 40
-- !query 0
@@ -354,3 +354,25 @@ Literals of type 'GEO' are currently not supported.(line 1, pos 7)
== SQL ==
select GEO '(10,-6)'
-------^^^
+
+
+-- !query 38
+select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
+-- !query 38 schema
+struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
+-- !query 38 output
+90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
+
+
+-- !query 39
+select 1.20E-38BD
+-- !query 39 schema
+struct<>
+-- !query 39 output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+DecimalType can only support precision up to 38(line 1, pos 7)
+
+== SQL ==
+select 1.20E-38BD
+-------^^^
http://git-wip-us.apache.org/repos/asf/spark/blob/a11d10f1/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala
index 86724cb..43a218b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala
@@ -39,6 +39,7 @@ class ExpressionSQLBuilderSuite extends SQLBuilderTest {
checkSQL(Literal(Double.PositiveInfinity), "CAST('Infinity' AS DOUBLE)")
checkSQL(Literal(Double.NegativeInfinity), "CAST('-Infinity' AS DOUBLE)")
checkSQL(Literal(Double.NaN), "CAST('NaN' AS DOUBLE)")
+ checkSQL(Literal(BigDecimal("10.0000000").underlying), "10.0000000BD")
checkSQL(
Literal(Timestamp.valueOf("2016-01-01 00:00:00")), "TIMESTAMP('2016-01-01 00:00:00.0')")
// TODO tests for decimals
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org