You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/09/11 00:07:24 UTC
[spark] branch master updated: [SPARK-45109][SQL][CONNECT] Fix aes_decrypt and ln functions in Connect
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5e97c79ad82 [SPARK-45109][SQL][CONNECT] Fix aes_decrypt and ln functions in Connect
5e97c79ad82 is described below
commit 5e97c79ad82914fc77601ad33cc304c61de93e87
Author: Peter Toth <pe...@gmail.com>
AuthorDate: Mon Sep 11 08:06:56 2023 +0800
[SPARK-45109][SQL][CONNECT] Fix aes_decrypt and ln functions in Connect
### What changes were proposed in this pull request?
Fix `aes_descrypt` and `ln` implementations in Spark Connect. The previous `aes_descrypt` reference to `aes_encrypt` is clearly a bug. The `ln` reference to `log` is more like a cosmetic issue, but because `ln` and `log` function implementations are different in Spark SQL we should use the same implementation in Spark Connect too.
### Why are the changes needed?
Bugfix.
### Does this PR introduce _any_ user-facing change?
No, these Spark Connect functions haven't been released.
### How was this patch tested?
Esiting UTs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #42863 from peter-toth/SPARK-45109-fix-eas_decrypt-and-ln.
Authored-by: Peter Toth <pe...@gmail.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../src/main/scala/org/apache/spark/sql/functions.scala | 10 +++++-----
.../explain-results/function_aes_decrypt.explain | 2 +-
.../function_aes_decrypt_with_mode.explain | 2 +-
.../function_aes_decrypt_with_mode_padding.explain | 2 +-
.../function_aes_decrypt_with_mode_padding_aad.explain | 2 +-
.../query-tests/explain-results/function_ln.explain | 2 +-
.../query-tests/queries/function_aes_decrypt.json | 2 +-
.../query-tests/queries/function_aes_decrypt.proto.bin | Bin 187 -> 187 bytes
.../queries/function_aes_decrypt_with_mode.json | 2 +-
.../queries/function_aes_decrypt_with_mode.proto.bin | Bin 194 -> 194 bytes
.../queries/function_aes_decrypt_with_mode_padding.json | 2 +-
.../function_aes_decrypt_with_mode_padding.proto.bin | Bin 201 -> 201 bytes
.../function_aes_decrypt_with_mode_padding_aad.json | 2 +-
...function_aes_decrypt_with_mode_padding_aad.proto.bin | Bin 208 -> 208 bytes
.../test/resources/query-tests/queries/function_ln.json | 2 +-
.../resources/query-tests/queries/function_ln.proto.bin | Bin 172 -> 171 bytes
16 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
index bf536c349cb..ab71cb80186 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2624,7 +2624,7 @@ object functions {
* @group math_funcs
* @since 3.5.0
*/
- def ln(e: Column): Column = log(e)
+ def ln(e: Column): Column = Column.fn("ln", e)
/**
* Computes the natural logarithm of the given value.
@@ -3503,7 +3503,7 @@ object functions {
mode: Column,
padding: Column,
aad: Column): Column =
- Column.fn("aes_encrypt", input, key, mode, padding, aad)
+ Column.fn("aes_decrypt", input, key, mode, padding, aad)
/**
* Returns a decrypted value of `input`.
@@ -3515,7 +3515,7 @@ object functions {
* @since 3.5.0
*/
def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column =
- Column.fn("aes_encrypt", input, key, mode, padding)
+ Column.fn("aes_decrypt", input, key, mode, padding)
/**
* Returns a decrypted value of `input`.
@@ -3527,7 +3527,7 @@ object functions {
* @since 3.5.0
*/
def aes_decrypt(input: Column, key: Column, mode: Column): Column =
- Column.fn("aes_encrypt", input, key, mode)
+ Column.fn("aes_decrypt", input, key, mode)
/**
* Returns a decrypted value of `input`.
@@ -3539,7 +3539,7 @@ object functions {
* @since 3.5.0
*/
def aes_decrypt(input: Column, key: Column): Column =
- Column.fn("aes_encrypt", input, key)
+ Column.fn("aes_decrypt", input, key)
/**
* This is a special version of `aes_decrypt` that performs the same operation, but returns a
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt.explain
index 44084a8e60f..31e03b79eb9 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt.explain
@@ -1,2 +1,2 @@
-Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesEncrypt, cast(g#0 as binary), cast(g#0 as binary), GCM, DEFAULT, cast( as binary), cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, BinaryType, true, true, true) AS aes_encrypt(g, g, GCM, DEFAULT, , )#0]
+Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesDecrypt, cast(g#0 as binary), cast(g#0 as binary), GCM, DEFAULT, cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, true, true, true) AS aes_decrypt(g, g, GCM, DEFAULT, )#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode.explain
index 29ccf0c1c83..fc572e8fe7c 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode.explain
@@ -1,2 +1,2 @@
-Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesEncrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, DEFAULT, cast( as binary), cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, BinaryType, true, true, true) AS aes_encrypt(g, g, g, DEFAULT, , )#0]
+Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesDecrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, DEFAULT, cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, true, true, true) AS aes_decrypt(g, g, g, DEFAULT, )#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding.explain
index 5591363426a..c6c693013dd 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding.explain
@@ -1,2 +1,2 @@
-Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesEncrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, g#0, cast( as binary), cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, BinaryType, true, true, true) AS aes_encrypt(g, g, g, g, , )#0]
+Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesDecrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, g#0, cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, true, true, true) AS aes_decrypt(g, g, g, g, )#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding_aad.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding_aad.explain
index 0e8d4df71b3..97bb528b84b 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding_aad.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_aes_decrypt_with_mode_padding_aad.explain
@@ -1,2 +1,2 @@
-Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesEncrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, g#0, cast(g#0 as binary), cast( as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, BinaryType, true, true, true) AS aes_encrypt(g, g, g, g, g, )#0]
+Project [staticinvoke(class org.apache.spark.sql.catalyst.expressions.ExpressionImplUtils, BinaryType, aesDecrypt, cast(g#0 as binary), cast(g#0 as binary), g#0, g#0, cast(g#0 as binary), BinaryType, BinaryType, StringType, StringType, BinaryType, true, true, true) AS aes_decrypt(g, g, g, g, g)#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ln.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ln.explain
index d3c3743b1ef..66b782ac817 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ln.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ln.explain
@@ -1,2 +1,2 @@
-Project [LOG(E(), b#0) AS LOG(E(), b)#0]
+Project [ln(b#0) AS ln(b)#0]
+- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json
index 06469d48405..4204a44b44c 100644
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json
@@ -13,7 +13,7 @@
},
"expressions": [{
"unresolvedFunction": {
- "functionName": "aes_encrypt",
+ "functionName": "aes_decrypt",
"arguments": [{
"unresolvedAttribute": {
"unparsedIdentifier": "g"
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin
index c7a70b51707..f635e1fc689 100644
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json
index 7eb9b4ed8b4..9c630e12534 100644
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json
@@ -13,7 +13,7 @@
},
"expressions": [{
"unresolvedFunction": {
- "functionName": "aes_encrypt",
+ "functionName": "aes_decrypt",
"arguments": [{
"unresolvedAttribute": {
"unparsedIdentifier": "g"
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin
index ecd81ae44fc..41d024cdb7e 100644
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json
index 59a6a5e35fd..8f5be474ab4 100644
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json
@@ -13,7 +13,7 @@
},
"expressions": [{
"unresolvedFunction": {
- "functionName": "aes_encrypt",
+ "functionName": "aes_decrypt",
"arguments": [{
"unresolvedAttribute": {
"unparsedIdentifier": "g"
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin
index 9de01ddc5ea..cd6764581f2 100644
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json
index a87ec1b7f4d..9381042b718 100644
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json
@@ -13,7 +13,7 @@
},
"expressions": [{
"unresolvedFunction": {
- "functionName": "aes_encrypt",
+ "functionName": "aes_decrypt",
"arguments": [{
"unresolvedAttribute": {
"unparsedIdentifier": "g"
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin
index 13da507fe6f..ca789f04ce1 100644
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ln.json b/connector/connect/common/src/test/resources/query-tests/queries/function_ln.json
index 1b2d0ed0b14..ababbc52d08 100644
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_ln.json
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_ln.json
@@ -13,7 +13,7 @@
},
"expressions": [{
"unresolvedFunction": {
- "functionName": "log",
+ "functionName": "ln",
"arguments": [{
"unresolvedAttribute": {
"unparsedIdentifier": "b"
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin
index 548fb480dd2..ecb87a1fc41 100644
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin differ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org