You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2025/04/07 16:54:40 UTC
(datafu) branch main updated: Add dedupRandomN and dedupByAllExcept to df_utils.py
This is an automated email from the ASF dual-hosted git repository.
eyal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafu.git
The following commit(s) were added to refs/heads/main by this push:
new a099c71 Add dedupRandomN and dedupByAllExcept to df_utils.py
a099c71 is described below
commit a099c71cc98f3f7c7c0c34aaa968b71c34da1dd1
Author: Alon Hartanu <ah...@paypal.com>
AuthorDate: Mon Feb 24 19:47:17 2025 +0200
Add dedupRandomN and dedupByAllExcept to df_utils.py
---
.../src/main/resources/pyspark_utils/df_utils.py | 34 ++++++++++++++++++++++
.../src/main/scala/datafu/spark/SparkDFUtils.scala | 4 +++
.../test/resources/python_tests/df_utils_tests.py | 6 ++++
.../scala/datafu/spark/TestScalaPythonBridge.scala | 7 +++++
4 files changed, 51 insertions(+)
diff --git a/datafu-spark/src/main/resources/pyspark_utils/df_utils.py b/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
index 8cd1e3e..cbb28b6 100644
--- a/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
+++ b/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
@@ -17,6 +17,7 @@
import pyspark
from pyspark.sql import DataFrame
+import pyspark.sql.functions
from pyspark_utils.bridge_utils import _getjvm_class
@@ -144,6 +145,37 @@ def join_with_range_and_dedup(df_single, col_single, df_range, col_range_start,
jdf = _get_utils(df_single).joinWithRangeAndDedup(df_single._jdf, col_single, df_range._jdf, col_range_start, col_range_end, decrease_factor, dedup_small_range)
return DataFrame(jdf, df_single.sql_ctx)
+
+def dedup_random_n(df, group_col, max_size):
+ """
+ Method for reducing rows when there is one column whose value is not important, but you don't want to lose any
+ actual data from the other rows. For example if a server creates events with an autogenerated event id, and sometimes
+ events are duplicated. You don't want double rows just for the event ids, but if any of the other fields are distinct
+ you want to keep the rows (with their event ids)
+ :param df: DataFrame to dedup
+ :param group_col: column to group by the records
+ :param max_size: The maximal number of rows per group
+ :return: DataFrame representing the data after the representation
+ """
+ jdf = _get_utils(df).dedupRandomN(df._jdf, group_col._jc, max_size)
+ return DataFrame(jdf, df.sql_ctx)
+
+
+def dedup_by_all_except(df, ignored_column):
+ """
+ Method for reducing rows when there is one column whose value is not important, but you don't want to lose any
+ actual data from the other rows. For example if a server creates events with an autogenerated event id, and sometimes
+ events are duplicated. You don't want double rows just for the event ids, but if any of the other fields are distinct
+ you want to keep the rows (with their event ids)
+ :param df: DataFrame
+ :param ignored_column: The one column whose value you need only one of
+ :param agg_function: Default is max
+ :return:
+ """
+ jdf = _get_utils(df).dedupByAllExcept(df._jdf, ignored_column)
+ return DataFrame(jdf, df.sql_ctx)
+
+
def explode_array(df, array_col, alias):
"""
Given an array column that you need to explode into different columns, use this method.
@@ -184,4 +216,6 @@ def activate():
pyspark.sql.DataFrame.join_with_range = join_with_range
pyspark.sql.DataFrame.join_with_range_and_dedup = join_with_range_and_dedup
pyspark.sql.DataFrame.explode_array = explode_array
+ pyspark.sql.DataFrame.dedup_random_n = dedup_random_n
+ pyspark.sql.DataFrame.dedup_by_all_except = dedup_by_all_except
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
index 4d571c7..5cbb21c 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
@@ -143,6 +143,10 @@ class SparkDFUtilsBridge {
SparkDFUtils.dedupRandomN(df, groupCol, maxSize)
}
+ def dedupByAllExcept(df: DataFrame, ignoredColumn: String): DataFrame = {
+ SparkDFUtils.dedupByAllExcept(df, ignoredColumn, org.apache.spark.sql.functions.max)
+ }
+
private def convertJavaListToSeq[T](list: JavaList[T]): Seq[T] = {
scala.collection.JavaConverters
.asScalaIteratorConverter(list.iterator())
diff --git a/datafu-spark/src/test/resources/python_tests/df_utils_tests.py b/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
index eec47ec..1d020d1 100644
--- a/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
+++ b/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
@@ -95,3 +95,9 @@ dfArray = sqlContext.createDataFrame([
func_explodeArray_res = df_utils.explode_array(df=dfArray, array_col=dfArray.sentence_arr, alias="token")
func_explodeArray_res.registerTempTable("explodeArray")
+
+func_dedupRandomN_res = df_utils.dedup_random_n(df_people, df_people.id, 1)
+func_dedupRandomN_res.registerTempTable("dedupRandomN")
+
+func_dedupByAllExcept_res = df_utils.dedup_by_all_except(df_people, "age")
+func_dedupByAllExcept_res.registerTempTable("dedupByAllExcept")
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
index b529145..a0765f5 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
@@ -120,6 +120,13 @@ class TestScalaPythonBridge extends FunSuite {
"[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
assertTable("explodeArray",
"[0.0,WrappedArray(Hi, I heard, about, Spark),Hi,I heard,about,Spark]")
+ assertTable("explodeArray",
+ "[0.0,WrappedArray(Hi, I heard, about, Spark),Hi,I heard,about,Spark]")
+ assertTable("dedupRandomN",
+ "[a,[a,Alice,34]], [b,[b,Bob,36]], [c,[c,David,29]]")
+ assertTable("dedupByAllExcept",
+ "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], [c,Fanny,36], [c,Zoey,36]"
+ )
}
}