You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2023/10/01 07:54:43 UTC
[datafu] branch main updated: DATAFU-168 fix collectLimitedList
This is an automated email from the ASF dual-hosted git repository.
eyal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafu.git
The following commit(s) were added to refs/heads/main by this push:
new 41f1a91 DATAFU-168 fix collectLimitedList
41f1a91 is described below
commit 41f1a91b0febd163e134948c5cb5f290a3d7198e
Author: Yanir Damti <yd...@paypal.com>
AuthorDate: Wed Sep 27 16:45:53 2023 +0300
DATAFU-168 fix collectLimitedList
Signed-off-by: Eyal Allweil <ey...@apache.org>
---
datafu-spark/build.gradle | 6 ++++--
.../main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala | 8 +++++++-
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/datafu-spark/build.gradle b/datafu-spark/build.gradle
index 878fc10..3493e33 100644
--- a/datafu-spark/build.gradle
+++ b/datafu-spark/build.gradle
@@ -67,9 +67,11 @@ dependencies {
// there isn't a spark-test-base version for 2.3.4, but 2.3.3 will work instead
if (sparkVersion == "2.3.4") {
- testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.3.3_" + sparkTestingBaseVersion
+ testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.3.3_" + sparkTestingBaseVersion
+ } else if (sparkVersion > "2.4.5" && sparkVersion < "3" ) {
+ testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":2.4.5_" + sparkTestingBaseVersion
} else {
- testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
+ testCompile "com.holdenkarau:spark-testing-base_" + scalaVersion + ":" + sparkVersion + "_" + sparkTestingBaseVersion
}
}
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
index 2dcb5be..7c3ffbb 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
@@ -23,11 +23,12 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.aggregate.{Collect, DeclarativeAggregate, ImperativeAggregate}
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryComparison, ExpectsInputTypes, Expression, GreaterThan, If, IsNull, LessThan, Literal}
-import org.apache.spark.sql.catalyst.util.TypeUtils
+import org.apache.spark.sql.catalyst.util.{GenericArrayData, TypeUtils}
import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
import scala.collection.generic.Growable
import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
object SparkOverwriteUDAFs {
def minValueByKey(key: Column, value: Column): Column =
@@ -108,6 +109,10 @@ case class CollectLimitedList(child: Expression,
def this(child: Expression) = this(child, 0, 0)
+ override lazy val bufferElementType = child.dataType
+
+ override protected def convertToBufferElement(value: Any): Any = InternalRow.copyValue(value)
+
override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
@@ -118,6 +123,7 @@ case class CollectLimitedList(child: Expression,
override def prettyName: String = "collect_limited_list"
+ override def eval(buffer: ArrayBuffer[Any]): Any = new GenericArrayData(buffer.toArray)
}
/** *