You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2025/04/07 16:54:40 UTC

(datafu) branch main updated: Add dedupRandomN and dedupByAllExcept to df_utils.py

This is an automated email from the ASF dual-hosted git repository.

eyal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafu.git


The following commit(s) were added to refs/heads/main by this push:
     new a099c71  Add dedupRandomN and dedupByAllExcept to df_utils.py
a099c71 is described below

commit a099c71cc98f3f7c7c0c34aaa968b71c34da1dd1
Author: Alon Hartanu <ah...@paypal.com>
AuthorDate: Mon Feb 24 19:47:17 2025 +0200

    Add dedupRandomN and dedupByAllExcept to df_utils.py
---
 .../src/main/resources/pyspark_utils/df_utils.py   | 34 ++++++++++++++++++++++
 .../src/main/scala/datafu/spark/SparkDFUtils.scala |  4 +++
 .../test/resources/python_tests/df_utils_tests.py  |  6 ++++
 .../scala/datafu/spark/TestScalaPythonBridge.scala |  7 +++++
 4 files changed, 51 insertions(+)

diff --git a/datafu-spark/src/main/resources/pyspark_utils/df_utils.py b/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
index 8cd1e3e..cbb28b6 100644
--- a/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
+++ b/datafu-spark/src/main/resources/pyspark_utils/df_utils.py
@@ -17,6 +17,7 @@
  
 import pyspark
 from pyspark.sql import DataFrame
+import pyspark.sql.functions
 from pyspark_utils.bridge_utils import _getjvm_class
 
 
@@ -144,6 +145,37 @@ def join_with_range_and_dedup(df_single, col_single, df_range, col_range_start,
     jdf = _get_utils(df_single).joinWithRangeAndDedup(df_single._jdf, col_single, df_range._jdf, col_range_start, col_range_end, decrease_factor, dedup_small_range)
     return DataFrame(jdf, df_single.sql_ctx)
 
+
+def dedup_random_n(df, group_col, max_size):
+    """
+    Method for reducing rows when there is one column whose value is not important, but you don't want to lose any
+    actual data from the other rows. For example if a server creates events with an autogenerated event id, and sometimes
+    events are duplicated. You don't want double rows just for the event ids, but if any of the other fields are distinct
+    you want to keep the rows (with their event ids)
+    :param df:  DataFrame to dedup
+    :param group_col: column to group by the records
+    :param max_size: The maximal number of rows per group
+    :return: DataFrame representing the data after the representation
+    """
+    jdf = _get_utils(df).dedupRandomN(df._jdf, group_col._jc, max_size)
+    return DataFrame(jdf, df.sql_ctx)
+
+
+def dedup_by_all_except(df, ignored_column):
+    """
+    Method for reducing rows when there is one column whose value is not important, but you don't want to lose any
+    actual data from the other rows. For example if a server creates events with an autogenerated event id, and sometimes
+    events are duplicated. You don't want double rows just for the event ids, but if any of the other fields are distinct
+    you want to keep the rows (with their event ids)
+    :param df: DataFrame
+    :param ignored_column: The one column whose value you need only one of
+    :param agg_function: Default is max
+    :return:
+    """
+    jdf = _get_utils(df).dedupByAllExcept(df._jdf, ignored_column)
+    return DataFrame(jdf, df.sql_ctx)
+
+
 def explode_array(df, array_col, alias):
     """
     Given an array column that you need to explode into different columns, use this method.
@@ -184,4 +216,6 @@ def activate():
     pyspark.sql.DataFrame.join_with_range = join_with_range
     pyspark.sql.DataFrame.join_with_range_and_dedup = join_with_range_and_dedup
     pyspark.sql.DataFrame.explode_array = explode_array
+    pyspark.sql.DataFrame.dedup_random_n = dedup_random_n
+    pyspark.sql.DataFrame.dedup_by_all_except = dedup_by_all_except
 
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
index 4d571c7..5cbb21c 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
@@ -143,6 +143,10 @@ class SparkDFUtilsBridge {
     SparkDFUtils.dedupRandomN(df, groupCol, maxSize)
   }
 
+  def dedupByAllExcept(df: DataFrame, ignoredColumn: String): DataFrame = {
+    SparkDFUtils.dedupByAllExcept(df, ignoredColumn, org.apache.spark.sql.functions.max)
+  }
+
   private def convertJavaListToSeq[T](list: JavaList[T]): Seq[T] = {
     scala.collection.JavaConverters
       .asScalaIteratorConverter(list.iterator())
diff --git a/datafu-spark/src/test/resources/python_tests/df_utils_tests.py b/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
index eec47ec..1d020d1 100644
--- a/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
+++ b/datafu-spark/src/test/resources/python_tests/df_utils_tests.py
@@ -95,3 +95,9 @@ dfArray = sqlContext.createDataFrame([
 
 func_explodeArray_res = df_utils.explode_array(df=dfArray, array_col=dfArray.sentence_arr, alias="token")
 func_explodeArray_res.registerTempTable("explodeArray")
+
+func_dedupRandomN_res = df_utils.dedup_random_n(df_people, df_people.id, 1)
+func_dedupRandomN_res.registerTempTable("dedupRandomN")
+
+func_dedupByAllExcept_res = df_utils.dedup_by_all_except(df_people, "age")
+func_dedupByAllExcept_res.registerTempTable("dedupByAllExcept")
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
index b529145..a0765f5 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
@@ -120,6 +120,13 @@ class TestScalaPythonBridge extends FunSuite {
                 "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
     assertTable("explodeArray",
       "[0.0,WrappedArray(Hi, I heard, about, Spark),Hi,I heard,about,Spark]")
+    assertTable("explodeArray",
+      "[0.0,WrappedArray(Hi, I heard, about, Spark),Hi,I heard,about,Spark]")
+    assertTable("dedupRandomN",
+      "[a,[a,Alice,34]], [b,[b,Bob,36]], [c,[c,David,29]]")
+    assertTable("dedupByAllExcept",
+      "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], [c,Fanny,36], [c,Zoey,36]"
+    )
   }
 
 }