You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2023/10/01 07:54:43 UTC

[datafu] branch main updated: DATAFU-168 fix collectLimitedList

This is an automated email from the ASF dual-hosted git repository.

eyal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafu.git


The following commit(s) were added to refs/heads/main by this push:
     new 41f1a91  DATAFU-168 fix collectLimitedList
41f1a91 is described below

commit 41f1a91b0febd163e134948c5cb5f290a3d7198e
Author: Yanir Damti <yd...@paypal.com>
AuthorDate: Wed Sep 27 16:45:53 2023 +0300

    DATAFU-168 fix collectLimitedList
    
    Signed-off-by: Eyal Allweil <ey...@apache.org>
---
 datafu-spark/build.gradle                                         | 6 ++++--
 .../main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala   | 8 +++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/datafu-spark/build.gradle b/datafu-spark/build.gradle
index 878fc10..3493e33 100644
--- a/datafu-spark/build.gradle
+++ b/datafu-spark/build.gradle
@@ -67,9 +67,11 @@ dependencies {
 
 // there isn't a spark-test-base version for 2.3.4, but 2.3.3 will work instead
     if (sparkVersion == "2.3.4") {
-    	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.3.3_" + sparkTestingBaseVersion
+        testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.3.3_" + sparkTestingBaseVersion
+    } else if (sparkVersion > "2.4.5" && sparkVersion < "3" ) {
+        testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.4.5_" + sparkTestingBaseVersion
     } else {
-	testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
+	    testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
     }
 }
 
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
index 2dcb5be..7c3ffbb 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
@@ -23,11 +23,12 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Collect, DeclarativeAggregate, ImperativeAggregate}
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryComparison, ExpectsInputTypes, Expression, GreaterThan, If, IsNull, LessThan, Literal}
-import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.catalyst.util.{GenericArrayData, TypeUtils}
 import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
 
 import scala.collection.generic.Growable
 import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
 
 object SparkOverwriteUDAFs {
   def minValueByKey(key: Column, value: Column): Column =
@@ -108,6 +109,10 @@ case class CollectLimitedList(child: Expression,
 
   def this(child: Expression) = this(child, 0, 0)
 
+  override lazy val bufferElementType = child.dataType
+
+  override protected def convertToBufferElement(value: Any): Any = InternalRow.copyValue(value)
+
   override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
     copy(mutableAggBufferOffset = newMutableAggBufferOffset)
 
@@ -118,6 +123,7 @@ case class CollectLimitedList(child: Expression,
 
   override def prettyName: String = "collect_limited_list"
 
+  override def eval(buffer: ArrayBuffer[Any]): Any = new GenericArrayData(buffer.toArray)
 }
 
 /** *