You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/08/12 00:29:49 UTC
(spark) branch branch-4.x updated: [SPARK-58717][PYTHON] Add Python tests for xxh3_64 and xxh3_128 hash functions
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 42318d582807 [SPARK-58717][PYTHON] Add Python tests for xxh3_64 and xxh3_128 hash functions
42318d582807 is described below
commit 42318d58280748c3551140a6c81ced1fc5c48809
Author: Pratham Manja <pr...@gmail.com>
AuthorDate: Wed Aug 12 08:28:10 2026 +0800
[SPARK-58717][PYTHON] Add Python tests for xxh3_64 and xxh3_128 hash functions
### What changes were proposed in this pull request?
Addition of python test cases for xxh3_64 and xxh3_128 hash functions, Initial support for these functions were added as part of https://issues.apache.org/jira/browse/SPARK-45900
### Why are the changes needed?
for test coverage.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit Tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #57935 from pratham76/tests_for_xxh3_64_xxh3_128.
Authored-by: Pratham Manja <pr...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 812ba8c26853dabcd4819c261281a8d698dad904)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
python/pyspark/sql/tests/test_functions.py | 48 ++++++++++++++++++++++++++++++
1 file changed, 48 insertions(+)
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index 61b4c4d258a0..44c1047cbe3c 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -4236,6 +4236,54 @@ class FunctionsTestsMixin:
self.assertEqual(result[0][1], ["Alice", "Carol"]) # Eng
self.assertEqual(result[1][1], ["Frank", "Dave"]) # Sales
+ def test_xxh3_64(self):
+ """Test xxh3_64 hash function"""
+ # Test with string input
+ df = self.spark.createDataFrame([("Spark",), ("",), (None,)], ["data"])
+ result = df.select(F.xxh3_64("data")).collect()
+
+ # Verify against known values from Scala tests
+ self.assertEqual(result[0][0], 80997306238743657) # "Spark"
+ self.assertEqual(result[1][0], 0x2D06800538D394C2) # empty string
+ self.assertIsNone(result[2][0]) # null
+
+ # Test with binary input
+ df_binary = self.spark.createDataFrame([(bytearray([1, 2, 3, 4, 5, 6]),)], ["data"])
+ result_binary = df_binary.select(F.xxh3_64("data")).collect()
+ # Value from DataFrameFunctionsSuite.scala
+ self.assertEqual(result_binary[0][0], -4044731995552965649)
+
+ def test_xxh3_128(self):
+ """Test xxh3_128 hash function"""
+ # Test with string input
+ df = self.spark.createDataFrame([("Spark",), ("",), (None,)], ["data"])
+ result = df.select(F.xxh3_128("data")).collect()
+
+ # Verify against known values from Scala tests
+ self.assertEqual(result[0][0], "7d57dd84c60c86ca1f4e82ab91a12b5e") # "Spark"
+ self.assertEqual(result[1][0], "99aa06d3014798d86001c324468d497f") # empty string
+ self.assertIsNone(result[2][0]) # null
+
+ # Test with binary input
+ df_binary = self.spark.createDataFrame([(bytearray([1, 2, 3, 4, 5, 6]),)], ["data"])
+ result_binary = df_binary.select(F.xxh3_128("data")).collect()
+ # Value from DataFrameFunctionsSuite.scala
+ self.assertEqual(result_binary[0][0], "866737830f560dbf3e1f439d2d785f44")
+
+ def test_xxh3_with_cast(self):
+ """Test xxh3 functions with explicit cast to binary"""
+ df = self.spark.createDataFrame([("ABC",)], ["a"])
+
+ # Test xxh3_64 with cast
+ result_64 = df.select(F.xxh3_64(F.col("a").cast("binary"))).collect()
+ # Value from DataFrameFunctionsSuite.scala
+ self.assertEqual(result_64[0][0], 2615927343983396622)
+
+ # Test xxh3_128 with cast
+ result_128 = df.select(F.xxh3_128(F.col("a").cast("binary"))).collect()
+ # Value from DataFrameFunctionsSuite.scala
+ self.assertEqual(result_128[0][0], "9e947f00ecd6acb2244da40f405c870e")
+
class FunctionsTests(FunctionsTestsMixin, ReusedSQLTestCase):
pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org