You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/12/07 21:42:38 UTC
[spark] branch branch-3.1 updated: [SPARK-33480][SQL][FOLLOWUP] do
not expose user data in error message
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 742d9fa [SPARK-33480][SQL][FOLLOWUP] do not expose user data in error message
742d9fa is described below
commit 742d9fa581ef612445730fe42d3887723e561cb1
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Mon Dec 7 13:35:37 2020 -0800
[SPARK-33480][SQL][FOLLOWUP] do not expose user data in error message
### What changes were proposed in this pull request?
This is a followup of https://github.com/apache/spark/pull/30412. This PR updates the error message of char/varchar table insertion length check, to not expose user data.
### Why are the changes needed?
This is risky to expose user data in the error message, especially the string data, as it may contain sensitive data.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
updated tests
Closes #30653 from cloud-fan/minor2.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit c0874ba9f13b9802eef4418490020692e37652ba)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/sql/catalyst/util/CharVarcharUtils.scala | 6 ++---
.../apache/spark/sql/CharVarcharTestSuite.scala | 26 +++++++++++-----------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
index b551d96..e42e384 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
@@ -183,9 +183,9 @@ object CharVarcharUtils extends Logging {
private def raiseError(expr: Expression, typeName: String, length: Int): Expression = {
val errorMsg = Concat(Seq(
- Literal("input string '"),
- expr,
- Literal(s"' exceeds $typeName type length limitation: $length")))
+ Literal("input string of length "),
+ Cast(Length(expr), StringType),
+ Literal(s" exceeds $typeName type length limitation: $length")))
Cast(RaiseError(errorMsg), StringType)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
index fcd334b..b0f1198 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -190,7 +190,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(null))
val e = intercept[SparkException](sql("INSERT INTO t VALUES ('123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -203,7 +203,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(1, null))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (1, '123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
}
@@ -215,7 +215,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Row(null)))
val e = intercept[SparkException](sql("INSERT INTO t SELECT struct('123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -226,7 +226,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(null)))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (array('a', '123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -235,7 +235,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
sql(s"CREATE TABLE t(c MAP<$typeName(5), STRING>) USING $format")
val e = intercept[SparkException](sql("INSERT INTO t VALUES (map('123456', 'a'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -246,7 +246,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Map("a" -> null)))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (map('a', '123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -255,10 +255,10 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
sql(s"CREATE TABLE t(c MAP<$typeName(5), $typeName(5)>) USING $format")
val e1 = intercept[SparkException](sql("INSERT INTO t VALUES (map('123456', 'a'))"))
assert(e1.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (map('a', '123456'))"))
assert(e2.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -269,7 +269,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Row(Seq(null))))
val e = intercept[SparkException](sql("INSERT INTO t SELECT struct(array('123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -280,7 +280,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(Row(null))))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (array(struct('123456')))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -291,7 +291,7 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(Seq(null))))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (array(array('123456')))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation: 5"))
}
}
@@ -313,10 +313,10 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
checkAnswer(spark.table("t"), Row("1234 ", "1234"))
val e1 = intercept[SparkException](sql("INSERT INTO t VALUES (123456, 1)"))
assert(e1.getCause.getMessage.contains(
- "input string '123456' exceeds char type length limitation: 5"))
+ "input string of length 6 exceeds char type length limitation: 5"))
val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (1, 123456)"))
assert(e2.getCause.getMessage.contains(
- "input string '123456' exceeds varchar type length limitation: 5"))
+ "input string of length 6 exceeds varchar type length limitation: 5"))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org