You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Raza Jafri (Jira)" <ji...@apache.org> on 2021/11/24 04:30:00 UTC
[jira] [Updated] (SPARK-37451) Performance improvement regressed String to Decimal cast
[ https://issues.apache.org/jira/browse/SPARK-37451?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Raza Jafri updated SPARK-37451:
-------------------------------
Description:
A performance improvement to how Spark casts Strings to Decimal in this [PR title|https://issues.apache.org/jira/browse/SPARK-32706], has introduced a regression
{noformat}
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+--------------------+
| a|
+--------------------+
|7.836725755512218...|
+--------------------+
scala> spark.version
res2: String = 3.0.1
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+----+
| a|
+----+
|null|
+----+
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
data: Seq[org.apache.spark.sql.Row] = List([7.836725755512218E38])
schema: org.apache.spark.sql.types.StructType = StructType(StructField(a,StringType,false))
df: org.apache.spark.sql.DataFrame = [a: string]
scala> spark.version
res1: String = 3.1.1
{noformat}
was:
A performance improvement to how Spark casts Strings to Decimal in this [PR title|https://issues.apache.org/jira/browse/SPARK-32706], has introduced a regression
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+--------------------+
| a|
+--------------------+
|7.836725755512218...|
+--------------------+
scala> spark.version
res2: String = 3.0.1
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+----+
| a|
+----+
|null|
+----+
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
data: Seq[org.apache.spark.sql.Row] = List([7.836725755512218E38])
schema: org.apache.spark.sql.types.StructType = StructType(StructField(a,StringType,false))
df: org.apache.spark.sql.DataFrame = [a: string]
scala> spark.version
res1: String = 3.1.1
The problem is that
> Performance improvement regressed String to Decimal cast
> --------------------------------------------------------
>
> Key: SPARK-37451
> URL: https://issues.apache.org/jira/browse/SPARK-37451
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
> Affects Versions: 3.1.0, 3.1.1, 3.1.2, 3.2.0, 3.1.3, 3.2.1
> Reporter: Raza Jafri
> Priority: Major
>
> A performance improvement to how Spark casts Strings to Decimal in this [PR title|https://issues.apache.org/jira/browse/SPARK-32706], has introduced a regression
> {noformat}
> scala> :paste
> // Entering paste mode (ctrl-D to finish)
> import org.apache.spark.sql.types._
> import org.apache.spark.sql.Row
> spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
> spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
> spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
> val data = Seq(Row("7.836725755512218E38"))
> val schema=StructType(Array(StructField("a", StringType, false)))
> val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
> df.select(col("a").cast(DecimalType(37,-17))).show
> // Exiting paste mode, now interpreting.
> +--------------------+
> | a|
> +--------------------+
> |7.836725755512218...|
> +--------------------+
> scala> spark.version
> res2: String = 3.0.1
> scala> :paste
> // Entering paste mode (ctrl-D to finish)
> import org.apache.spark.sql.types._
> import org.apache.spark.sql.Row
> spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
> spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
> spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
> val data = Seq(Row("7.836725755512218E38"))
> val schema=StructType(Array(StructField("a", StringType, false)))
> val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
> df.select(col("a").cast(DecimalType(37,-17))).show
> // Exiting paste mode, now interpreting.
> +----+
> | a|
> +----+
> |null|
> +----+
> import org.apache.spark.sql.types._
> import org.apache.spark.sql.Row
> data: Seq[org.apache.spark.sql.Row] = List([7.836725755512218E38])
> schema: org.apache.spark.sql.types.StructType = StructType(StructField(a,StringType,false))
> df: org.apache.spark.sql.DataFrame = [a: string]
> scala> spark.version
> res1: String = 3.1.1
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org