You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/05/31 02:00:56 UTC
[spark] branch master updated: [SPARK-39335][SQL] DescribeTableCommand should redact properties
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8bbbdb5a466 [SPARK-39335][SQL] DescribeTableCommand should redact properties
8bbbdb5a466 is described below
commit 8bbbdb5a4667d90b27b202870cd73c9a19b74781
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Tue May 31 10:00:26 2022 +0800
[SPARK-39335][SQL] DescribeTableCommand should redact properties
### What changes were proposed in this pull request?
In current code we only redact storage's properties, for normal properties we should redact too.
### Why are the changes needed?
Redact properties
### Does this PR introduce _any_ user-facing change?
After this pr, when describe table, table properties will be redacted.
### How was this patch tested?
Added UT
Closes #36722 from AngersZhuuuu/SPARK-39335.
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/catalog/interface.scala | 11 ++++----
.../test/resources/sql-tests/inputs/describe.sql | 4 +--
.../resources/sql-tests/results/describe.sql.out | 32 +++++++++++-----------
3 files changed, 24 insertions(+), 23 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 2af8fd8456c..d6f8352d4ec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -119,7 +119,8 @@ case class CatalogTablePartition(
map.put("Partition Values", s"[$specString]")
map ++= storage.toLinkedHashMap
if (parameters.nonEmpty) {
- map.put("Partition Parameters", s"{${parameters.map(p => p._1 + "=" + p._2).mkString(", ")}}")
+ map.put("Partition Parameters", s"{" +
+ s"${SQLConf.get.redactOptions(parameters).map(p => p._1 + "=" + p._2).mkString(", ")}}")
}
map.put("Created Time", new Date(createTime).toString)
val lastAccess = {
@@ -383,10 +384,10 @@ case class CatalogTable(
def toLinkedHashMap: mutable.LinkedHashMap[String, String] = {
val map = new mutable.LinkedHashMap[String, String]()
- val tableProperties = properties
- .filterKeys(!_.startsWith(VIEW_PREFIX))
- .toSeq.sortBy(_._1)
- .map(p => p._1 + "=" + p._2)
+ val tableProperties =
+ SQLConf.get.redactOptions(properties.filterKeys(!_.startsWith(VIEW_PREFIX)).toMap)
+ .toSeq.sortBy(_._1)
+ .map(p => p._1 + "=" + p._2)
val partitionColumns = partitionColumnNames.map(quoteIdentifier).mkString("[", ", ", "]")
val lastAccess = {
if (lastAccessTime <= 0) "UNKNOWN" else new Date(lastAccessTime).toString
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
index deff5bb7ca6..493ea68baed 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
@@ -1,8 +1,8 @@
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
- OPTIONS (a '1', b '2')
+ OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
- TBLPROPERTIES (t 'test');
+ TBLPROPERTIES (t 'test', password 'password');
CREATE TEMPORARY VIEW temp_v AS SELECT * FROM t;
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index 1219e472556..7c0cc29a814 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -4,10 +4,10 @@
-- !query
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
- OPTIONS (a '1', b '2')
+ OPTIONS (a '1', b '2', password 'password')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
- TBLPROPERTIES (t 'test')
+ TBLPROPERTIES (t 'test', password 'password')
-- !query schema
struct<>
-- !query output
@@ -130,9 +130,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
-Table Properties [e=3, t=test]
+Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog
@@ -162,9 +162,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
-Table Properties [e=3, t=test]
+Table Properties [e=3, password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog
@@ -202,9 +202,9 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
-Table Properties [t=test]
+Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog
@@ -241,9 +241,9 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
-Table Properties [t=test]
+Table Properties [password=*********(redacted), t=test]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Partition Provider Catalog
@@ -281,7 +281,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]
@@ -290,7 +290,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
-- !query
@@ -312,7 +312,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]
@@ -321,7 +321,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
-- !query
@@ -343,7 +343,7 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]/{warehouse_dir}/t/c=Us/d=1
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
Created Time [not included in comparison]
Last Access [not included in comparison]
@@ -352,7 +352,7 @@ Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]/{warehouse_dir}/t
-Storage Properties [a=1, b=2]
+Storage Properties [a=1, b=2, password=*********(redacted)]
-- !query
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org