You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2019/05/15 14:13:36 UTC
[datafu] 01/02: code reformatting according to Spark's scalastyle
This is an automated email from the ASF dual-hosted git repository.
eyal pushed a commit to branch spark-tmp
in repository https://gitbox.apache.org/repos/asf/datafu.git
commit 8a5ccc89a1b38df4abd581d2a58c8f572d105df9
Author: oraviv <or...@paypal.com>
AuthorDate: Wed May 15 13:40:28 2019 +0300
code reformatting according to Spark's scalastyle
---
.../META-INF/services/datafu.spark.PythonResource | 2 +-
.../src/main/scala/datafu/spark/DataFrameOps.scala | 65 ++-
.../scala/datafu/spark/PythonPathsManager.scala | 141 +++---
.../scala/datafu/spark/ScalaPythonBridge.scala | 58 ++-
.../src/main/scala/datafu/spark/SparkDFUtils.scala | 408 +++++++++++-----
.../src/main/scala/datafu/spark/SparkUDAFs.scala | 168 ++++---
.../utils/overwrites/SparkOverwriteUDAFs.scala | 38 +-
.../spark/utils/overwrites/SparkPythonRunner.scala | 69 ++-
.../META-INF/services/datafu.spark.PythonResource | 2 +-
.../test/resources/example_tests/pyfromscala.py | 4 +-
.../datafu/spark/PySparkLibTestResources.scala | 8 +-
.../scala/datafu/spark/TestScalaPythonBridge.scala | 66 ++-
.../test/scala/datafu/spark/TestSparkDFUtils.scala | 528 ++++++++++++---------
.../test/scala/datafu/spark/TestSparkUDAFs.scala | 230 +++++----
14 files changed, 1134 insertions(+), 653 deletions(-)
diff --git a/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource b/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
index d43f740..ee69a0d 100644
--- a/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
+++ b/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-datafu.spark.CoreBridgeDirectory
\ No newline at end of file
+datafu.spark.CoreBridgeDirectory
diff --git a/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala b/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
index f1ac607..110f13f 100644
--- a/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
@@ -30,26 +30,63 @@ object DataFrameOps {
def dedupTopN(n: Int, groupCol: Column, orderCols: Column*): DataFrame =
SparkDFUtils.dedupTopN(df, n, groupCol, orderCols: _*)
- def dedup2(groupCol: Column, orderByCol: Column, desc: Boolean = true, moreAggFunctions: Seq[Column] = Nil, columnsFilter: Seq[String] = Nil, columnsFilterKeep: Boolean = true): DataFrame =
- SparkDFUtils.dedup2(df, groupCol, orderByCol, desc, moreAggFunctions, columnsFilter, columnsFilterKeep)
+ def dedup2(groupCol: Column,
+ orderByCol: Column,
+ desc: Boolean = true,
+ moreAggFunctions: Seq[Column] = Nil,
+ columnsFilter: Seq[String] = Nil,
+ columnsFilterKeep: Boolean = true): DataFrame =
+ SparkDFUtils.dedup2(df,
+ groupCol,
+ orderByCol,
+ desc,
+ moreAggFunctions,
+ columnsFilter,
+ columnsFilterKeep)
def flatten(colName: String): DataFrame = SparkDFUtils.flatten(df, colName)
- def changeSchema(newScheme: String*): DataFrame = SparkDFUtils.changeSchema(df, newScheme: _*)
+ def changeSchema(newScheme: String*): DataFrame =
+ SparkDFUtils.changeSchema(df, newScheme: _*)
- def joinWithRange(colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long = 2^8) =
- SparkDFUtils.joinWithRange(df, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
+ def joinWithRange(colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long = 2 ^ 8): DataFrame =
+ SparkDFUtils.joinWithRange(df,
+ colSingle,
+ dfRange,
+ colRangeStart,
+ colRangeEnd,
+ DECREASE_FACTOR)
- def joinWithRangeAndDedup(colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long = 2^8, dedupSmallRange: Boolean = true): DataFrame = {
- SparkDFUtils.joinWithRangeAndDedup(df, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR, dedupSmallRange)
- }
+ def joinWithRangeAndDedup(colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long = 2 ^ 8,
+ dedupSmallRange: Boolean = true): DataFrame =
+ SparkDFUtils.joinWithRangeAndDedup(df,
+ colSingle,
+ dfRange,
+ colRangeStart,
+ colRangeEnd,
+ DECREASE_FACTOR,
+ dedupSmallRange)
- def broadcastJoinSkewed(skewed: DataFrame, joinCol: String, numberCustsToBroadcast: Int): DataFrame = {
- SparkDFUtils.broadcastJoinSkewed(df, skewed, joinCol, numberCustsToBroadcast)
- }
+ def broadcastJoinSkewed(skewed: DataFrame,
+ joinCol: String,
+ numberCustsToBroadcast: Int): DataFrame =
+ SparkDFUtils.broadcastJoinSkewed(df,
+ skewed,
+ joinCol,
+ numberCustsToBroadcast)
- def joinSkewed(notSkewed: DataFrame, joinExprs: Column, numShards: Int = 1000, joinType: String = "inner") : DataFrame = {
+ def joinSkewed(notSkewed: DataFrame,
+ joinExprs: Column,
+ numShards: Int = 1000,
+ joinType: String = "inner"): DataFrame =
SparkDFUtils.joinSkewed(df, notSkewed, joinExprs, numShards, joinType)
- }
}
-}
\ No newline at end of file
+}
diff --git a/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala b/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
index 9c16898..26377b8 100644
--- a/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
@@ -21,105 +21,131 @@ package datafu.spark
import java.io.{File, IOException}
import java.net.JarURLConnection
import java.nio.file.Paths
+import java.util
import java.util.{MissingResourceException, ServiceLoader}
+import scala.collection.JavaConverters._
+
import org.apache.log4j.Logger
-import scala.collection.JavaConverters._
/**
- * Represents a resource that needs to be added to PYTHONPATH used by ScalaPythonBridge.
- *
- * To ensure your python resources (modules, files, etc.) are properly added to the bridge, do the following:
- * 1) Put all the resource under some root directory with a unique name x, and make sure path/to/x
- * is visible to the class loader (usually just use src/main/resources/x).
- * 2) Extend this class like this:
- * class MyResource extends PythonResource("x")
- * This assumes x is under src/main/resources/x
- * 3) (since we use ServiceLoader) Add a file to your jar/project:
- * META-INF/services/spark.utils.PythonResource
- * with a single line containing the full name (including package) of MyResource.
- *
- * This process involves scanning the entire jar and copying files from the jar to some temporary location,
- * so if your jar is really big consider putting the resources in a smaller jar.
- *
- * @param resourcePath Path to the resource, will be loaded via getClass.getClassLoader.getResource()
- * @param isAbsolutePath Set to true if the resource is in some absolute path rather than in jar (try to avoid that).
- */
-abstract class PythonResource(val resourcePath: String, val isAbsolutePath: Boolean = false)
+ * Represents a resource that needs to be added to PYTHONPATH used by ScalaPythonBridge.
+ *
+ * To ensure your python resources (modules, files, etc.) are properly added to the bridge,
+ * do the following:
+ * 1) Put all the resource under some root directory with a unique name x, and make sure path/to/x
+ * is visible to the class loader (usually just use src/main/resources/x).
+ * 2) Extend this class like this:
+ * class MyResource extends PythonResource("x")
+ * This assumes x is under src/main/resources/x
+ * 3) (since we use ServiceLoader) Add a file to your jar/project:
+ * META-INF/services/spark.utils.PythonResource
+ * with a single line containing the full name (including package) of MyResource.
+ *
+ * This process involves scanning the entire jar and copying files from the jar to some temporary
+ * location, so if your jar is really big consider putting the resources in a smaller jar.
+ *
+ * @param resourcePath Path to the resource, will be loaded via
+ * getClass.getClassLoader.getResource()
+ * @param isAbsolutePath Set to true if the resource is in some absolute path rather than in jar
+ * (try to avoid that).
+ */
+abstract class PythonResource(val resourcePath: String,
+ val isAbsolutePath: Boolean = false)
/**
- * There are two phases of resolving python files path:
- *
- * 1) When launching spark:
- * the files need to be added to spark.executorEnv.PYTHONPATH
- *
- * 2) When executing python file via bridge:
- * the files need to be added to the process PYTHONPATH. This is different than the previous phase because
- * this python process is spawned by datafu-spark, not by spark, and always on the driver.
- */
+ * There are two phases of resolving python files path:
+ *
+ * 1) When launching spark:
+ * the files need to be added to spark.executorEnv.PYTHONPATH
+ *
+ * 2) When executing python file via bridge:
+ * the files need to be added to the process PYTHONPATH.
+ * This is different than the previous phase because
+ * this python process is spawned by datafu-spark, not by spark, and always on the driver.
+ */
object PythonPathsManager {
- case class ResolvedResource(resource: PythonResource, resolvedLocation: String)
+ case class ResolvedResource(resource: PythonResource,
+ resolvedLocation: String)
private val logger: Logger = Logger.getLogger(getClass)
val resources: Seq[ResolvedResource] =
- ServiceLoader.load(classOf[PythonResource])
+ ServiceLoader
+ .load(classOf[PythonResource])
.asScala
.map(p => ResolvedResource(p, resolveDependencyLocation(p)))
.toSeq
logResolved
- def getAbsolutePaths() = resources.map(_.resolvedLocation).distinct
- def getAbsolutePathsForJava() = resources.map(_.resolvedLocation).distinct.asJava
+ def getAbsolutePaths(): Seq[String] = resources.map(_.resolvedLocation).distinct
+ def getAbsolutePathsForJava(): util.List[String] =
+ resources.map(_.resolvedLocation).distinct.asJava
def getPYTHONPATH(): String =
resources
.map(_.resolvedLocation)
.map(p => new File(p))
- .map(_.getName) //get just the name of the file
+ .map(_.getName) // get just the name of the file
.mkString(":")
private def resolveDependencyLocation(resource: PythonResource): String =
if (resource.isAbsolutePath) {
- if (!new File(resource.resourcePath).exists())
- throw new IOException("Could not find resource in absolute path: " + resource.resourcePath)
- else {
+ if (!new File(resource.resourcePath).exists()) {
+ throw new IOException(
+ "Could not find resource in absolute path: " + resource.resourcePath)
+ } else {
logger.info("Using file absolute path: " + resource.resourcePath)
resource.resourcePath
}
} else {
Option(getClass.getClassLoader.getResource(resource.resourcePath)) match {
case None =>
- logger.error("Didn't find resource in classpath! resource path: " + resource.resourcePath)
- throw new MissingResourceException("Didn't find resource in classpath!", resource.getClass.getName, resource.resourcePath)
+ logger.error(
+ "Didn't find resource in classpath! resource path: " + resource.resourcePath)
+ throw new MissingResourceException(
+ "Didn't find resource in classpath!",
+ resource.getClass.getName,
+ resource.resourcePath)
case Some(p) =>
p.toURI.getScheme match {
case "jar" =>
- //if dependency is inside jar file, use jar file path:
- val jarPath = new File(p.openConnection().asInstanceOf[JarURLConnection].getJarFileURL.toURI).getPath
- logger.info(s"Dependency ${resource.resourcePath} found inside jar: " + jarPath)
+ // if dependency is inside jar file, use jar file path:
+ val jarPath = new File(
+ p.openConnection()
+ .asInstanceOf[JarURLConnection]
+ .getJarFileURL
+ .toURI).getPath
+ logger.info(
+ s"Dependency ${resource.resourcePath} found inside jar: " + jarPath)
jarPath
case "file" =>
val file = new File(p.getFile)
if (!file.exists()) {
logger.warn("Dependency not found, skipping: " + file.getPath)
null
- }
- else {
- if (file.isDirectory) {
+ } else {
+ if (file.isDirectory) {
val t_path =
- if (System.getProperty("os.name").toLowerCase().contains("win")&& p.getPath().startsWith("/")) {
+ if (System
+ .getProperty("os.name")
+ .toLowerCase()
+ .contains("win") && p.getPath().startsWith("/")) {
val path = p.getPath.substring(1)
- logger.warn(s"Fixing path for windows operating system! converted ${p.getPath} to $path")
+ logger.warn(
+ s"Fixing path for windows operating system! " +
+ s"converted ${p.getPath} to $path")
path
- }
- else
+ } else {
p.getPath
+ }
val path = Paths.get(t_path)
- logger.info(s"Dependency found as directory: ${t_path}\n\tusing parent path: ${path.getParent}")
+ logger.info(
+ s"Dependency found as directory: ${t_path}\n\tusing " +
+ s"parent path: ${path.getParent}")
path.getParent.toString
} else {
logger.info("Dependency found as a file: " + p.getPath)
@@ -131,15 +157,16 @@ object PythonPathsManager {
}
private def logResolved = {
- logger.info(
- s"Discovered ${resources.size} python paths:\n" +
- resources
- .map(p => s"className: ${p.resource.getClass.getName}\n\tresource: ${p.resource.resourcePath}\n\tlocation: ${p.resolvedLocation}")
- .mkString("\n")) + "\n\n"
+ logger.info(s"Discovered ${resources.size} python paths:\n" +
+ resources
+ .map(p =>
+ s"className: ${p.resource.getClass.getName}\n\tresource: " +
+ s"${p.resource.resourcePath}\n\tlocation: ${p.resolvedLocation}")
+ .mkString("\n")) + "\n\n"
}
}
/**
- * Contains all python files needed by the bridge itself
- */
+ * Contains all python files needed by the bridge itself
+ */
class CoreBridgeDirectory extends PythonResource("pyspark_utils")
diff --git a/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala b/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
index 8727f82..4e0fc77 100644
--- a/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
@@ -23,22 +23,27 @@ import java.net.URL
import java.nio.file.Files
import java.util.UUID
+import org.slf4j.LoggerFactory
+
import org.apache.spark.SparkConf
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.deploy.SparkPythonRunner
import org.apache.spark.sql.SparkSession
-import org.slf4j.LoggerFactory
-
case class ScalaPythonBridgeRunner(extraPath: String = "") {
val logger = LoggerFactory.getLogger(this.getClass)
- //for the bridge we take the full resolved location, since this runs on the driver where the files are local:
+ // for the bridge we take the full resolved location,
+ // since this runs on the driver where the files are local:
logger.info("constructing PYTHONPATH")
-
+
// we include multiple options for py4j because on any given cluster only one should be found
val pythonPath = (PythonPathsManager.getAbsolutePaths() ++
- Array("pyspark.zip", "py4j-0.10.4-src.zip", "py4j-0.10.6-src.zip", "py4j-0.10.7-src.zip", "py4j-0.10.8.1-src.zip") ++
+ Array("pyspark.zip",
+ "py4j-0.10.4-src.zip",
+ "py4j-0.10.6-src.zip",
+ "py4j-0.10.7-src.zip",
+ "py4j-0.10.8.1-src.zip") ++
Option(extraPath).getOrElse("").split(",")).distinct
logger.info("Bridge PYTHONPATH: " + pythonPath.mkString(":"))
@@ -47,28 +52,29 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
def runPythonFile(filename: String): String = {
val pyScript = resolveRunnableScript(filename)
- // this.getClass.getClassLoader.asInstanceOf[URLClassLoader].getURLs.foreach(s => logger.info("class: " + s))
logger.info(s"Running python file $pyScript")
runner.runPyFile(pyScript)
}
def runPythonString(str: String): String = {
val tmpFile = writeToTempFile(str, "pyspark-tmp-file-", ".py")
- logger.info("Running tmp PySpark file: " + tmpFile.getAbsolutePath + " with content:\n" + str)
+ logger.info(
+ "Running tmp PySpark file: " + tmpFile.getAbsolutePath + " with content:\n" + str)
runner.runPyFile(tmpFile.getAbsolutePath)
}
private def resolveRunnableScript(path: String): String = {
logger.info("Resolving python script location for: " + path)
- val res: String = Option(this.getClass.getClassLoader.getResource(path)) match {
+ val res
+ : String = Option(this.getClass.getClassLoader.getResource(path)) match {
case None =>
logger.info("Didn't find script via classLoader, using as is: " + path)
path
case Some(resource) =>
resource.toURI.getScheme match {
case "jar" =>
- //if inside jar, extract it and return cloned file:
+ // if inside jar, extract it and return cloned file:
logger.info("Script found inside jar, extracting...")
val outputFile = ResourceCloning.cloneResource(resource, path)
logger.info("Extracted file path: " + outputFile.getPath)
@@ -81,7 +87,9 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
res
}
- private def writeToTempFile(contents: String, prefix: String, suffix: String): File = {
+ private def writeToTempFile(contents: String,
+ prefix: String,
+ suffix: String): File = {
val tempFi = File.createTempFile(prefix, suffix)
tempFi.deleteOnExit()
val bw = new BufferedWriter(new FileWriter(tempFi))
@@ -93,12 +101,13 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
}
/**
- * Do not instantiate this class! Use the companion object instead.
- * This class should only be used by python
- */
-object ScalaPythonBridge { //need empty ctor for py4j gateway
+ * Do not instantiate this class! Use the companion object instead.
+ * This class should only be used by python
+ */
+object ScalaPythonBridge { // need empty ctor for py4j gateway
- /** members used to allow python script share context with main Scala program calling it.
+ /**
+ * members used to allow python script share context with main Scala program calling it.
* Python script calls :
* sc, sqlContext, spark = utils.get_contexts()
* our Python util function get_contexts
@@ -106,20 +115,22 @@ object ScalaPythonBridge { //need empty ctor for py4j gateway
*/
// Called by python util get_contexts()
def pyGetSparkSession(): SparkSession = SparkSession.builder().getOrCreate()
- def pyGetJSparkContext(sparkSession: SparkSession): JavaSparkContext = new JavaSparkContext(sparkSession.sparkContext)
+ def pyGetJSparkContext(sparkSession: SparkSession): JavaSparkContext =
+ new JavaSparkContext(sparkSession.sparkContext)
def pyGetSparkConf(jsc: JavaSparkContext): SparkConf = jsc.getConf
}
/**
- * Utility for extracting resource from a jar and copy it to a temporary location
- */
+ * Utility for extracting resource from a jar and copy it to a temporary location
+ */
object ResourceCloning {
private val logger = LoggerFactory.getLogger(this.getClass)
val uuid = UUID.randomUUID().toString.substring(6)
- val outputTempDir = new File(System.getProperty("java.io.tmpdir"), s"risk_tmp/$uuid/cloned_resources/")
+ val outputTempDir = new File(System.getProperty("java.io.tmpdir"),
+ s"risk_tmp/$uuid/cloned_resources/")
forceMkdirs(outputTempDir)
def cloneResource(resource: URL, outputFileName: String): File = {
@@ -129,7 +140,8 @@ object ResourceCloning {
outputTmpFile
} else {
logger.info("cloning resource: " + resource)
- if (!outputTmpFile.exists()) { //it is possible that the file was already extracted in the session
+ if (!outputTmpFile.exists()) {
+ // it is possible that the file was already extracted in the session
forceMkdirs(outputTmpFile.getParentFile)
val inputStream = resource.openStream()
streamToFile(outputTmpFile, inputStream)
@@ -139,14 +151,14 @@ object ResourceCloning {
}
private def forceMkdirs(dir: File) =
- if (!dir.exists() && !dir.mkdirs())
+ if (!dir.exists() && !dir.mkdirs()) {
throw new IOException("Failed to create " + dir.getPath)
+ }
private def streamToFile(outputFile: File, inputStream: InputStream) = {
try {
Files.copy(inputStream, outputFile.toPath)
- }
- finally {
+ } finally {
inputStream.close()
assert(outputFile.exists())
}
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
index 61386bd..2b48c52 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
@@ -20,30 +20,51 @@ package datafu.spark
import java.util.{List => JavaList}
+import org.apache.spark.sql.{Column, DataFrame}
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{LongType, SparkOverwriteUDAFs, StructType}
-import org.apache.spark.sql.{Column, DataFrame}
import org.apache.spark.storage.StorageLevel
/**
- * class definition so we could expose this functionality in PySpark
- */
+ * class definition so we could expose this functionality in PySpark
+ */
class SparkDFUtilsBridge {
- def dedup(df: DataFrame, groupCol: Column, orderCols: JavaList[Column]): DataFrame = {
+ def dedup(df: DataFrame,
+ groupCol: Column,
+ orderCols: JavaList[Column]): DataFrame = {
val converted = convertJavaListToSeq(orderCols)
SparkDFUtils.dedup(df = df, groupCol = groupCol, orderCols = converted: _*)
}
- def dedupTopN(df: DataFrame, n: Int, groupCol: Column, orderCols: JavaList[Column]): DataFrame = {
+ def dedupTopN(df: DataFrame,
+ n: Int,
+ groupCol: Column,
+ orderCols: JavaList[Column]): DataFrame = {
val converted = convertJavaListToSeq(orderCols)
- SparkDFUtils.dedupTopN(df = df, n = n, groupCol = groupCol, orderCols = converted: _*)
+ SparkDFUtils.dedupTopN(df = df,
+ n = n,
+ groupCol = groupCol,
+ orderCols = converted: _*)
}
- def dedup2(df: DataFrame, groupCol: Column, orderByCol: Column, desc: Boolean, columnsFilter: JavaList[String], columnsFilterKeep: Boolean): DataFrame = {
+ def dedup2(df: DataFrame,
+ groupCol: Column,
+ orderByCol: Column,
+ desc: Boolean,
+ columnsFilter: JavaList[String],
+ columnsFilterKeep: Boolean): DataFrame = {
val columnsFilter_converted = convertJavaListToSeq(columnsFilter)
- SparkDFUtils.dedup2(df = df, groupCol = groupCol, orderByCol = orderByCol, desc = desc, moreAggFunctions = Nil, columnsFilter = columnsFilter_converted, columnsFilterKeep = columnsFilterKeep)
+ SparkDFUtils.dedup2(
+ df = df,
+ groupCol = groupCol,
+ orderByCol = orderByCol,
+ desc = desc,
+ moreAggFunctions = Nil,
+ columnsFilter = columnsFilter_converted,
+ columnsFilterKeep = columnsFilterKeep
+ )
}
def changeSchema(df: DataFrame, newScheme: JavaList[String]): DataFrame = {
@@ -51,31 +72,73 @@ class SparkDFUtilsBridge {
SparkDFUtils.changeSchema(df = df, newScheme = newScheme_converted: _*)
}
- def joinSkewed(dfLeft: DataFrame, dfRight: DataFrame, joinExprs: Column, numShards: Int, joinType: String): DataFrame = {
- SparkDFUtils.joinSkewed(dfLeft = dfLeft, dfRight = dfRight, joinExprs = joinExprs, numShards = numShards, joinType = joinType)
+ def joinSkewed(dfLeft: DataFrame,
+ dfRight: DataFrame,
+ joinExprs: Column,
+ numShards: Int,
+ joinType: String): DataFrame = {
+ SparkDFUtils.joinSkewed(dfLeft = dfLeft,
+ dfRight = dfRight,
+ joinExprs = joinExprs,
+ numShards = numShards,
+ joinType = joinType)
}
- def broadcastJoinSkewed(notSkewed: DataFrame, skewed: DataFrame, joinCol: String, numRowsToBroadcast: Int): DataFrame = {
- SparkDFUtils.broadcastJoinSkewed(notSkewed = notSkewed, skewed = skewed, joinCol = joinCol, numRowsToBroadcast = numRowsToBroadcast)
+ def broadcastJoinSkewed(notSkewed: DataFrame,
+ skewed: DataFrame,
+ joinCol: String,
+ numRowsToBroadcast: Int): DataFrame = {
+ SparkDFUtils.broadcastJoinSkewed(notSkewed = notSkewed,
+ skewed = skewed,
+ joinCol = joinCol,
+ numRowsToBroadcast = numRowsToBroadcast)
}
- def joinWithRange(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
- SparkDFUtils.joinWithRange(dfSingle = dfSingle, colSingle = colSingle, dfRange = dfRange, colRangeStart = colRangeStart, colRangeEnd = colRangeEnd, DECREASE_FACTOR = DECREASE_FACTOR)
+ def joinWithRange(dfSingle: DataFrame,
+ colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long): DataFrame = {
+ SparkDFUtils.joinWithRange(dfSingle = dfSingle,
+ colSingle = colSingle,
+ dfRange = dfRange,
+ colRangeStart = colRangeStart,
+ colRangeEnd = colRangeEnd,
+ DECREASE_FACTOR = DECREASE_FACTOR)
}
- def joinWithRangeAndDedup(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long, dedupSmallRange: Boolean): DataFrame = {
- SparkDFUtils.joinWithRangeAndDedup(dfSingle = dfSingle, colSingle = colSingle, dfRange = dfRange, colRangeStart = colRangeStart, colRangeEnd = colRangeEnd, DECREASE_FACTOR = DECREASE_FACTOR, dedupSmallRange = dedupSmallRange)
+ def joinWithRangeAndDedup(dfSingle: DataFrame,
+ colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long,
+ dedupSmallRange: Boolean): DataFrame = {
+ SparkDFUtils.joinWithRangeAndDedup(
+ dfSingle = dfSingle,
+ colSingle = colSingle,
+ dfRange = dfRange,
+ colRangeStart = colRangeStart,
+ colRangeEnd = colRangeEnd,
+ DECREASE_FACTOR = DECREASE_FACTOR,
+ dedupSmallRange = dedupSmallRange
+ )
}
private def convertJavaListToSeq[T](list: JavaList[T]): Seq[T] = {
- scala.collection.JavaConverters.asScalaIteratorConverter(list.iterator()).asScala.toList
+ scala.collection.JavaConverters
+ .asScalaIteratorConverter(list.iterator())
+ .asScala
+ .toList
}
}
object SparkDFUtils {
/**
- * Used to get the 'latest' record (after ordering according to the provided order columns) in each group.
+ * Used to get the 'latest' record (after ordering according to the provided order columns)
+ * in each group.
* Different from {@link org.apache.spark.sql.Dataset#dropDuplicates} because order matters.
*
* @param df DataFrame to operate on
@@ -89,7 +152,8 @@ object SparkDFUtils {
}
/**
- * Used get the top N records (after ordering according to the provided order columns) in each group.
+ * Used get the top N records (after ordering according to the provided order columns)
+ * in each group.
*
* @param df DataFrame to operate on
* @param n number of records to return from each group
@@ -97,14 +161,19 @@ object SparkDFUtils {
* @param orderCols columns to order the records according to
* @return DataFrame representing the data after the operation
*/
- def dedupTopN(df: DataFrame, n: Int, groupCol: Column, orderCols: Column*): DataFrame = {
+ def dedupTopN(df: DataFrame,
+ n: Int,
+ groupCol: Column,
+ orderCols: Column*): DataFrame = {
val w = Window.partitionBy(groupCol).orderBy(orderCols: _*)
df.withColumn("rn", row_number.over(w)).where(col("rn") <= n).drop("rn")
}
/**
- * Used to get the 'latest' record (after ordering according to the provided order columns) in each group.
- * the same functionality as {@link #dedup} but implemented using UDAF to utilize map side aggregation.
+ * Used to get the 'latest' record (after ordering according to the provided order columns)
+ * in each group.
+ * the same functionality as {@link #dedup} but implemented using UDAF to utilize
+ * map side aggregation.
*
* @param df DataFrame to operate on
* @param groupCol column to group by the records
@@ -112,72 +181,99 @@ object SparkDFUtils {
* @param desc have the order as desc
* @param moreAggFunctions more aggregate functions
* @param columnsFilter columns to filter
- * @param columnsFilterKeep indicates whether we should filter the selected columns 'out' or alternatively have only
- * those columns in the result
+ * @param columnsFilterKeep indicates whether we should filter the selected columns 'out'
+ * or alternatively have only those columns in the result
* @return DataFrame representing the data after the operation
*/
- def dedup2(df: DataFrame, groupCol: Column, orderByCol: Column, desc: Boolean = true, moreAggFunctions: Seq[Column] = Nil, columnsFilter: Seq[String] = Nil, columnsFilterKeep: Boolean = true): DataFrame = {
- val newDF = if (columnsFilter == Nil)
- df.withColumn("sort_by_column", orderByCol)
- else {
- if(columnsFilterKeep)
- df.withColumn("sort_by_column", orderByCol).select("sort_by_column", columnsFilter: _*)
- else
- df.select(df.columns.filter(colName => !columnsFilter.contains(colName)).map(colName => new Column(colName)):_*).withColumn("sort_by_column", orderByCol)
- }
-
- val aggFunc = if (desc) SparkOverwriteUDAFs.maxValueByKey(_:Column, _:Column)
- else SparkOverwriteUDAFs.minValueByKey(_:Column, _:Column)
+ def dedup2(df: DataFrame,
+ groupCol: Column,
+ orderByCol: Column,
+ desc: Boolean = true,
+ moreAggFunctions: Seq[Column] = Nil,
+ columnsFilter: Seq[String] = Nil,
+ columnsFilterKeep: Boolean = true): DataFrame = {
+ val newDF =
+ if (columnsFilter == Nil) {
+ df.withColumn("sort_by_column", orderByCol)
+ } else {
+ if (columnsFilterKeep) {
+ df.withColumn("sort_by_column", orderByCol)
+ .select("sort_by_column", columnsFilter: _*)
+ } else {
+ df.select(
+ df.columns
+ .filter(colName => !columnsFilter.contains(colName))
+ .map(colName => new Column(colName)): _*)
+ .withColumn("sort_by_column", orderByCol)
+ }
+ }
+
+ val aggFunc =
+ if (desc) SparkOverwriteUDAFs.maxValueByKey(_: Column, _: Column)
+ else SparkOverwriteUDAFs.minValueByKey(_: Column, _: Column)
val df2 = newDF
.groupBy(groupCol.as("group_by_col"))
- .agg(aggFunc(expr("sort_by_column"), expr("struct(sort_by_column, *)")).as("h1"),
- struct(lit(1).as("lit_placeholder_col") +: moreAggFunctions: _*).as("h2"))
- .selectExpr("h2.*","h1.*")
+ .agg(aggFunc(expr("sort_by_column"), expr("struct(sort_by_column, *)"))
+ .as("h1"),
+ struct(lit(1).as("lit_placeholder_col") +: moreAggFunctions: _*)
+ .as("h2"))
+ .selectExpr("h2.*", "h1.*")
.drop("lit_placeholder_col")
.drop("sort_by_column")
df2
}
- /**
- * Returns a DataFrame with the given column (should be a StructType) replaced by its inner fields.
- * This method only flattens a single level of nesting.
- *
- * +-------+----------+----------+----------+
- * |id |s.sub_col1|s.sub_col2|s.sub_col3|
- * +-------+----------+----------+----------+
- * |123 |1 |2 |3 |
- * +-------+----------+----------+----------+
- *
- * +-------+----------+----------+----------+
- * |id |sub_col1 |sub_col2 |sub_col3 |
- * +-------+----------+----------+----------+
- * |123 |1 |2 |3 |
- * +-------+----------+----------+----------+
- *
- * @param df DataFrame to operate on
- * @param colName column name for a column of type StructType
- * @return DataFrame representing the data after the operation
- */
+ /**
+ * Returns a DataFrame with the given column (should be a StructType)
+ * replaced by its inner fields.
+ * This method only flattens a single level of nesting.
+ *
+ * +-------+----------+----------+----------+
+ * |id |s.sub_col1|s.sub_col2|s.sub_col3|
+ * +-------+----------+----------+----------+
+ * |123 |1 |2 |3 |
+ * +-------+----------+----------+----------+
+ *
+ * +-------+----------+----------+----------+
+ * |id |sub_col1 |sub_col2 |sub_col3 |
+ * +-------+----------+----------+----------+
+ * |123 |1 |2 |3 |
+ * +-------+----------+----------+----------+
+ *
+ * @param df DataFrame to operate on
+ * @param colName column name for a column of type StructType
+ * @return DataFrame representing the data after the operation
+ */
def flatten(df: DataFrame, colName: String): DataFrame = {
- assert(df.schema(colName).dataType.isInstanceOf[StructType], s"Column $colName must be of type Struct")
+ assert(df.schema(colName).dataType.isInstanceOf[StructType],
+ s"Column $colName must be of type Struct")
val outerFields = df.schema.fields.map(_.name).toSet
- val flattenFields = df.schema(colName).dataType.asInstanceOf[StructType].fields.filter(f => !outerFields.contains(f.name)).map("`" + colName + "`.`" + _.name + "`")
+ val flattenFields = df
+ .schema(colName)
+ .dataType
+ .asInstanceOf[StructType]
+ .fields
+ .filter(f => !outerFields.contains(f.name))
+ .map("`" + colName + "`.`" + _.name + "`")
df.selectExpr("*" +: flattenFields: _*).drop(colName)
}
/**
- * Returns a DataFrame with the column names renamed to the column names in the new schema
- *
- * @param df DataFrame to operate on
- * @param newScheme new column names
- * @return DataFrame representing the data after the operation
- */
+ * Returns a DataFrame with the column names renamed to the column names in the new schema
+ *
+ * @param df DataFrame to operate on
+ * @param newScheme new column names
+ * @return DataFrame representing the data after the operation
+ */
def changeSchema(df: DataFrame, newScheme: String*): DataFrame =
- df.select(df.columns.zip(newScheme).map {case (oldCol: String, newCol: String) => col(oldCol).as(newCol)}: _*)
+ df.select(df.columns.zip(newScheme).map {
+ case (oldCol: String, newCol: String) => col(oldCol).as(newCol)
+ }: _*)
/**
- * Used to perform a join when the right df is relatively small but still too big to fit in memory to perform map side broadcast join.
+ * Used to perform a join when the right df is relatively small
+ * but still too big to fit in memory to perform map side broadcast join.
* Use cases:
* a. excluding keys that might be skewed from a medium size list.
* b. join a big skewed table with a table that has small number of very large rows.
@@ -189,19 +285,31 @@ object SparkDFUtils {
* @param joinType join type
* @return joined DataFrame
*/
- def joinSkewed(dfLeft: DataFrame, dfRight: DataFrame, joinExprs: Column, numShards: Int = 10, joinType: String = "inner"): DataFrame = {
+ def joinSkewed(dfLeft: DataFrame,
+ dfRight: DataFrame,
+ joinExprs: Column,
+ numShards: Int = 10,
+ joinType: String = "inner"): DataFrame = {
// skew join based on salting
- // salts the left DF by adding another random column and join with the right DF after duplicating it
+ // salts the left DF by adding another random column and join with the right DF after
+ // duplicating it
val ss = dfLeft.sparkSession
import ss.implicits._
val shards = 1.to(numShards).toDF("shard")
- dfLeft.withColumn("randLeft", ceil(rand() * numShards)).join(dfRight.crossJoin(shards), joinExprs and $"randLeft" === $"shard", joinType).drop($"randLeft").drop($"shard")
+ dfLeft
+ .withColumn("randLeft", ceil(rand() * numShards))
+ .join(dfRight.crossJoin(shards),
+ joinExprs and $"randLeft" === $"shard",
+ joinType)
+ .drop($"randLeft")
+ .drop($"shard")
}
/**
* Suitable to perform a join in cases when one DF is skewed and the other is not skewed.
* splits both of the DFs to two parts according to the skewed keys.
- * 1. Map-join: broadcasts the skewed-keys part of the not skewed DF to the skewed-keys part of the skewed DF
+ * 1. Map-join: broadcasts the skewed-keys part of the not skewed DF to the skewed-keys
+ * part of the skewed DF
* 2. Regular join: between the remaining two parts.
*
* @param notSkewed not skewed DataFrame
@@ -210,35 +318,55 @@ object SparkDFUtils {
* @param numRowsToBroadcast num of rows to broadcast
* @return DataFrame representing the data after the operation
*/
- def broadcastJoinSkewed(notSkewed: DataFrame, skewed: DataFrame, joinCol: String, numRowsToBroadcast: Int): DataFrame = {
+ def broadcastJoinSkewed(notSkewed: DataFrame,
+ skewed: DataFrame,
+ joinCol: String,
+ numRowsToBroadcast: Int): DataFrame = {
val ss = notSkewed.sparkSession
import ss.implicits._
- val skewedKeys = skewed.groupBy(joinCol).count().sort($"count".desc).limit(numRowsToBroadcast).drop("count")
- .withColumnRenamed(joinCol, "skew_join_key").cache()
-
- val notSkewedWithSkewIndicator = notSkewed.join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
- .withColumn("is_skewed_record", col("skew_join_key").isNotNull).drop("skew_join_key").persist(StorageLevel.DISK_ONLY)
+ val skewedKeys = skewed
+ .groupBy(joinCol)
+ .count()
+ .sort($"count".desc)
+ .limit(numRowsToBroadcast)
+ .drop("count")
+ .withColumnRenamed(joinCol, "skew_join_key")
+ .cache()
+
+ val notSkewedWithSkewIndicator = notSkewed
+ .join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
+ .withColumn("is_skewed_record", col("skew_join_key").isNotNull)
+ .drop("skew_join_key")
+ .persist(StorageLevel.DISK_ONLY)
// broadcast map-join, sending the notSkewed data
- val bigRecordsJnd = broadcast(notSkewedWithSkewIndicator.filter("is_skewed_record"))
- .join(skewed, joinCol)
+ val bigRecordsJnd =
+ broadcast(notSkewedWithSkewIndicator.filter("is_skewed_record"))
+ .join(skewed, joinCol)
// regular join for the rest
- val skewedWithoutSkewedKeys = skewed.join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
- .where("skew_join_key is null").drop("skew_join_key")
- val smallRecordsJnd = notSkewedWithSkewIndicator.filter("not is_skewed_record")
+ val skewedWithoutSkewedKeys = skewed
+ .join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
+ .where("skew_join_key is null")
+ .drop("skew_join_key")
+ val smallRecordsJnd = notSkewedWithSkewIndicator
+ .filter("not is_skewed_record")
.join(skewedWithoutSkewedKeys, joinCol)
- smallRecordsJnd.union(bigRecordsJnd).drop("is_skewed_record", "skew_join_key")
+ smallRecordsJnd
+ .union(bigRecordsJnd)
+ .drop("is_skewed_record", "skew_join_key")
}
/**
* Helper function to join a table with point column to a table with range column.
- * For example, join a table that contains specific time in minutes with a table that contains time ranges.
- * The main problem this function addresses is that doing naive explode on the ranges can result in a huge table.
+ * For example, join a table that contains specific time in minutes with a table that
+ * contains time ranges.
+ * The main problem this function addresses is that doing naive explode on the ranges can result
+ * in a huge table.
* requires:
- * 1. point table needs to be distinct on the point column. there could be a few corresponding ranges to each point,
- * so we choose the minimal range.
+ * 1. point table needs to be distinct on the point column. there could be a few corresponding
+ * ranges to each point, so we choose the minimal range.
* 2. the range and point columns need to be numeric.
*
* TIMES:
@@ -266,33 +394,61 @@ object SparkDFUtils {
* |11:55 |11:50 |12:15 | lunch |
* +-------+----------+---------+---------+
*
- * @param dfSingle - DataFrame that contains the point column
- * @param colSingle - the point column's name
- * @param dfRange - DataFrame that contains the range column
- * @param colRangeStart - the start range column's name
- * @param colRangeEnd - the end range column's name
- * @param DECREASE_FACTOR - resolution factor. instead of exploding the range column directly, we first decrease its resolution by this factor
+ * @param dfSingle DataFrame that contains the point column
+ * @param colSingle the point column's name
+ * @param dfRange DataFrame that contains the range column
+ * @param colRangeStart the start range column's name
+ * @param colRangeEnd the end range column's name
+ * @param DECREASE_FACTOR resolution factor. instead of exploding the range column directly,
+ * we first decrease its resolution by this factor
* @return
*/
- def joinWithRange(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
- val dfJoined = joinWithRangeInternal(dfSingle, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
- dfJoined.drop("range_start", "range_end", "decreased_range_single", "single", "decreased_single", "range_size")
+ def joinWithRange(dfSingle: DataFrame,
+ colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long): DataFrame = {
+ val dfJoined = joinWithRangeInternal(dfSingle,
+ colSingle,
+ dfRange,
+ colRangeStart,
+ colRangeEnd,
+ DECREASE_FACTOR)
+ dfJoined.drop("range_start",
+ "range_end",
+ "decreased_range_single",
+ "single",
+ "decreased_single",
+ "range_size")
}
- private def joinWithRangeInternal(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
+ private def joinWithRangeInternal(dfSingle: DataFrame,
+ colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long): DataFrame = {
import org.apache.spark.sql.functions.udf
val rangeUDF = udf((start: Long, end: Long) => (start to end).toArray)
- val dfRange_exploded = dfRange.withColumn("range_start", col(colRangeStart).cast(LongType))
- .withColumn("range_end" , col(colRangeEnd).cast(LongType))
- .withColumn("decreased_range_single", explode(rangeUDF(col("range_start")/lit(DECREASE_FACTOR),
- col("range_end") /lit(DECREASE_FACTOR))))
-
- dfSingle.withColumn("single", floor(col(colSingle).cast(LongType)))
- .withColumn("decreased_single", floor(col(colSingle).cast(LongType)/lit(DECREASE_FACTOR)))
- .join(dfRange_exploded, col("decreased_single") === col("decreased_range_single"), "left_outer")
- .withColumn("range_size", expr("(range_end - range_start + 1)"))
- .filter("single>=range_start and single<=range_end")
+ val dfRange_exploded = dfRange
+ .withColumn("range_start", col(colRangeStart).cast(LongType))
+ .withColumn("range_end", col(colRangeEnd).cast(LongType))
+ .withColumn("decreased_range_single",
+ explode(
+ rangeUDF(col("range_start") / lit(DECREASE_FACTOR),
+ col("range_end") / lit(DECREASE_FACTOR))))
+
+ dfSingle
+ .withColumn("single", floor(col(colSingle).cast(LongType)))
+ .withColumn("decreased_single",
+ floor(col(colSingle).cast(LongType) / lit(DECREASE_FACTOR)))
+ .join(dfRange_exploded,
+ col("decreased_single") === col("decreased_range_single"),
+ "left_outer")
+ .withColumn("range_size", expr("(range_end - range_start + 1)"))
+ .filter("single>=range_start and single<=range_end")
}
/**
@@ -315,17 +471,39 @@ object SparkDFUtils {
* +-------+----------+---------+---------+
*
*/
- def joinWithRangeAndDedup(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long, dedupSmallRange: Boolean): DataFrame = {
-
- val dfJoined = joinWithRangeInternal(dfSingle, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
+ def joinWithRangeAndDedup(dfSingle: DataFrame,
+ colSingle: String,
+ dfRange: DataFrame,
+ colRangeStart: String,
+ colRangeEnd: String,
+ DECREASE_FACTOR: Long,
+ dedupSmallRange: Boolean): DataFrame = {
+
+ val dfJoined = joinWithRangeInternal(dfSingle,
+ colSingle,
+ dfRange,
+ colRangeStart,
+ colRangeEnd,
+ DECREASE_FACTOR)
// "range_start" is here for consistency
val dfDeduped = if (dedupSmallRange) {
- dedup2(dfJoined, col(colSingle), struct("range_size", "range_start"), desc = false)
+ dedup2(dfJoined,
+ col(colSingle),
+ struct("range_size", "range_start"),
+ desc = false)
} else {
- dedup2(dfJoined, col(colSingle), struct(expr("-range_size"), col("range_start")), desc = true)
+ dedup2(dfJoined,
+ col(colSingle),
+ struct(expr("-range_size"), col("range_start")),
+ desc = true)
}
- dfDeduped.drop("range_start", "range_end", "decreased_range_single", "single", "decreased_single", "range_size")
+ dfDeduped.drop("range_start",
+ "range_end",
+ "decreased_range_single",
+ "single",
+ "decreased_single",
+ "range_size")
}
}
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala b/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
index fe95ffb..3e0810b 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
@@ -18,11 +18,12 @@
*/
package datafu.spark
+import scala.collection.{mutable, Map}
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types.{ArrayType, _}
-import scala.collection.{Map, mutable}
// @TODO: add documentation and tests to all the functions. maybe also expose in python.
@@ -36,36 +37,44 @@ object SparkUDAFs {
def inputSchema: StructType = new StructType().add("key", StringType)
- def bufferSchema = new StructType().add("mp", MapType(StringType, IntegerType))
+ def bufferSchema: StructType =
+ new StructType().add("mp", MapType(StringType, IntegerType))
def dataType: DataType = MapType(StringType, IntegerType, false)
- def deterministic = true
+ def deterministic: Boolean = true
// This function is called whenever key changes
- def initialize(buffer: MutableAggregationBuffer) = {
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
buffer(0) = mutable.Map()
}
// Iterate over each entry of a group
- def update(buffer: MutableAggregationBuffer, input: Row) = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val key = input.getString(0)
- if (key != null)
- buffer(0) = buffer.getMap(0) + (key -> (buffer.getMap(0).getOrElse(key, 0) + 1))
+ if (key != null) {
+ buffer(0) = buffer.getMap(0) + (key -> (buffer
+ .getMap(0)
+ .getOrElse(key, 0) + 1))
+ }
}
// Merge two partial aggregates
- def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
val mp = mutable.Map[String, Int]() ++= buffer1.getMap(0)
- buffer2.getMap(0).keys.foreach((key: String) =>
- if (key != null)
- mp.put(key, mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
- )
+ buffer2
+ .getMap(0)
+ .keys
+ .foreach((key: String) =>
+ if (key != null) {
+ mp.put(key,
+ mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
+ })
buffer1(0) = mp
}
// Called after all the entries are exhausted.
- def evaluate(buffer: Row) = {
+ def evaluate(buffer: Row): Any = {
buffer(0)
}
@@ -75,23 +84,24 @@ object SparkUDAFs {
* Essentially the same as MultiSet, but gets an Array for input.
* There is an extra option to limit the number of keys (like CountDistinctUpTo)
*/
- class MultiArraySet[T : Ordering](dt : DataType = StringType, maxKeys: Int = -1) extends UserDefinedAggregateFunction {
+ class MultiArraySet[T: Ordering](dt: DataType = StringType, maxKeys: Int = -1)
+ extends UserDefinedAggregateFunction {
def inputSchema: StructType = new StructType().add("key", ArrayType(dt))
- def bufferSchema = new StructType().add("mp", dataType)
+ def bufferSchema: StructType = new StructType().add("mp", dataType)
def dataType: DataType = MapType(dt, IntegerType, false)
- def deterministic = true
+ def deterministic: Boolean = true
// This function is called whenever key changes
- def initialize(buffer: MutableAggregationBuffer) = {
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
buffer(0) = mutable.Map()
}
// Iterate over each entry of a group
- def update(buffer: MutableAggregationBuffer, input: Row) = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val mp = mutable.Map[T, Int]() ++= buffer.getMap(0)
val keyArr: Seq[T] = Option(input.getAs[Seq[T]](0)).getOrElse(Nil)
for (key <- keyArr; if key != null)
@@ -101,12 +111,16 @@ object SparkUDAFs {
}
// Merge two partial aggregates
- def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
val mp = mutable.Map[T, Int]() ++= buffer1.getMap(0)
- buffer2.getMap(0).keys.foreach((key: T) =>
- if (key != null)
- mp.put(key, mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
- )
+ buffer2
+ .getMap(0)
+ .keys
+ .foreach((key: T) =>
+ if (key != null) {
+ mp.put(key,
+ mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
+ })
buffer1(0) = limitKeys(mp, 3)
}
@@ -114,24 +128,29 @@ object SparkUDAFs {
private def limitKeys(mp: Map[T, Int], factor: Int = 1): Map[T, Int] = {
if (maxKeys > 0 && maxKeys * factor < mp.size) {
val k = mp.toList.map(_.swap).sorted.reverse(maxKeys - 1)._1
- var mp2 = mutable.Map[T, Int]() ++= mp.filter((t: (T, Int)) => t._2 >= k)
+ var mp2 = mutable.Map[T, Int]() ++= mp.filter((t: (T, Int)) =>
+ t._2 >= k)
var toRemove = mp2.size - maxKeys
- if (toRemove > 0)
+ if (toRemove > 0) {
mp2 = mp2.filter((t: (T, Int)) => {
- if (t._2 > k)
+ if (t._2 > k) {
true
- else {
- if (toRemove >= 0)
+ } else {
+ if (toRemove >= 0) {
toRemove = toRemove - 1
+ }
toRemove < 0
}
})
+ }
mp2
- } else mp
+ } else {
+ mp
+ }
}
// Called after all the entries are exhausted.
- def evaluate(buffer: Row) = {
+ def evaluate(buffer: Row): Map[T, Int] = {
limitKeys(buffer.getMap(0).asInstanceOf[Map[T, Int]])
}
@@ -144,97 +163,116 @@ object SparkUDAFs {
def inputSchema: StructType = new StructType().add("key", dataType)
- def bufferSchema = inputSchema
+ def bufferSchema: StructType = inputSchema
def dataType: DataType = MapType(StringType, ArrayType(StringType))
- def deterministic = true
+ def deterministic: Boolean = true
// This function is called whenever key changes
- def initialize(buffer: MutableAggregationBuffer) = {
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
buffer(0) = mutable.Map()
}
// Iterate over each entry of a group
- def update(buffer: MutableAggregationBuffer, input: Row) = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val mp0 = input.getMap(0)
if (mp0 != null) {
- val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= input.getMap(0)
- buffer(0) = merge(mp, buffer.getMap[String, mutable.WrappedArray[String]](0))
+ val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= input
+ .getMap(0)
+ buffer(0) =
+ merge(mp, buffer.getMap[String, mutable.WrappedArray[String]](0))
}
}
// Merge two partial aggregates
- def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
- val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= buffer1.getMap(0)
- buffer1(0) = merge(mp, buffer2.getMap[String, mutable.WrappedArray[String]](0))
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= buffer1
+ .getMap(0)
+ buffer1(0) =
+ merge(mp, buffer2.getMap[String, mutable.WrappedArray[String]](0))
}
- def merge(mpBuffer: mutable.Map[String, mutable.WrappedArray[String]], mp: Map[String, mutable.WrappedArray[String]]): mutable.Map[String, mutable.WrappedArray[String]] = {
- if (mp != null)
+ def merge(mpBuffer: mutable.Map[String, mutable.WrappedArray[String]],
+ mp: Map[String, mutable.WrappedArray[String]])
+ : mutable.Map[String, mutable.WrappedArray[String]] = {
+ if (mp != null) {
mp.keys.foreach((key: String) => {
- val blah1: mutable.WrappedArray[String] = mpBuffer.getOrElse(key, mutable.WrappedArray.empty)
- val blah2: mutable.WrappedArray[String] = mp.getOrElse(key, mutable.WrappedArray.empty)
- mpBuffer.put(key, mutable.WrappedArray.make((Option(blah1).getOrElse(mutable.WrappedArray.empty) ++ Option(blah2).getOrElse(mutable.WrappedArray.empty)).toSet.toArray) )
+ val blah1: mutable.WrappedArray[String] =
+ mpBuffer.getOrElse(key, mutable.WrappedArray.empty)
+ val blah2: mutable.WrappedArray[String] =
+ mp.getOrElse(key, mutable.WrappedArray.empty)
+ mpBuffer.put(
+ key,
+ mutable.WrappedArray.make(
+ (Option(blah1).getOrElse(mutable.WrappedArray.empty) ++ Option(
+ blah2).getOrElse(mutable.WrappedArray.empty)).toSet.toArray)
+ )
})
+ }
mpBuffer
}
// Called after all the entries are exhausted.
- def evaluate(buffer: Row) = {
+ def evaluate(buffer: Row): Any = {
buffer(0)
}
}
/**
- * Counts number of distinct records, but only up to a preset amount - more efficient than an unbounded count
- */
- class CountDistinctUpTo(maxItems: Int = -1) extends UserDefinedAggregateFunction {
+ * Counts number of distinct records, but only up to a preset amount -
+ * more efficient than an unbounded count
+ */
+ class CountDistinctUpTo(maxItems: Int = -1)
+ extends UserDefinedAggregateFunction {
def inputSchema: StructType = new StructType().add("key", StringType)
- def bufferSchema = new StructType().add("mp", MapType(StringType, BooleanType))
+ def bufferSchema: StructType =
+ new StructType().add("mp", MapType(StringType, BooleanType))
def dataType: DataType = IntegerType
- def deterministic = true
+ def deterministic: Boolean = true
// This function is called whenever key changes
- def initialize(buffer: MutableAggregationBuffer) = {
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
buffer(0) = mutable.Map()
}
// Iterate over each entry of a group
- def update(buffer: MutableAggregationBuffer, input: Row) = {
- if(buffer.getMap(0).size < maxItems)
- {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ if (buffer.getMap(0).size < maxItems) {
val key = input.getString(0)
- if (key != null)
+ if (key != null) {
buffer(0) = buffer.getMap(0) + (key -> true)
+ }
}
}
// Merge two partial aggregates
- def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
- if(buffer1.getMap(0).size < maxItems)
- {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ if (buffer1.getMap(0).size < maxItems) {
val mp = mutable.Map[String, Boolean]() ++= buffer1.getMap(0)
- buffer2.getMap(0).keys.foreach((key: String) =>
- if (key != null)
- mp.put(key,true)
- )
+ buffer2
+ .getMap(0)
+ .keys
+ .foreach((key: String) =>
+ if (key != null) {
+ mp.put(key, true)
+ })
buffer1(0) = mp
}
}
// Called after all the entries are exhausted.
- def evaluate(buffer: Row) = {
- math.min(buffer.getMap(0).size,maxItems)
+ def evaluate(buffer: Row): Int = {
+ math.min(buffer.getMap(0).size, maxItems)
}
}
-}
\ No newline at end of file
+}
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
index 5949b62..b1b81b8 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
@@ -20,19 +20,28 @@ package org.apache.spark.sql.types
import org.apache.spark.sql.Column
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryComparison, ExpectsInputTypes, Expression, GreaterThan, If, IsNull, LessThan, Literal}
import org.apache.spark.sql.catalyst.util.TypeUtils
object SparkOverwriteUDAFs {
- def minValueByKey(key: Column, value: Column): Column = Column(MinValueByKey(key.expr, value.expr).toAggregateExpression(false))
- def maxValueByKey(key: Column, value: Column): Column = Column(MaxValueByKey(key.expr, value.expr).toAggregateExpression(false))
+ def minValueByKey(key: Column, value: Column): Column =
+ Column(MinValueByKey(key.expr, value.expr).toAggregateExpression(false))
+ def maxValueByKey(key: Column, value: Column): Column =
+ Column(MaxValueByKey(key.expr, value.expr).toAggregateExpression(false))
}
-case class MinValueByKey(child1: Expression, child2: Expression) extends ExtramumValueByKey(child1,child2, LessThan)
-case class MaxValueByKey(child1: Expression, child2: Expression) extends ExtramumValueByKey(child1,child2, GreaterThan)
+case class MinValueByKey(child1: Expression, child2: Expression)
+ extends ExtramumValueByKey(child1, child2, LessThan)
+case class MaxValueByKey(child1: Expression, child2: Expression)
+ extends ExtramumValueByKey(child1, child2, GreaterThan)
-abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp: (Expression, Expression) => BinaryComparison) extends DeclarativeAggregate with ExpectsInputTypes {
+abstract class ExtramumValueByKey(
+ child1: Expression,
+ child2: Expression,
+ bComp: (Expression, Expression) => BinaryComparison)
+ extends DeclarativeAggregate
+ with ExpectsInputTypes {
override def children: Seq[Expression] = child1 :: child2 :: Nil
@@ -50,7 +59,8 @@ abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp:
private lazy val minmax = AttributeReference("minmax", child1.dataType)()
private lazy val data = AttributeReference("data", child2.dataType)()
- override lazy val aggBufferAttributes: Seq[AttributeReference] = minmax :: data :: Nil
+ override lazy val aggBufferAttributes
+ : Seq[AttributeReference] = minmax :: data :: Nil
override lazy val initialValues: Seq[Expression] = Seq(
Literal.create(null, child1.dataType),
@@ -63,11 +73,17 @@ abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp:
override lazy val mergeExpressions: Seq[Expression] =
chooseKeyValue(minmax.left, data.left, minmax.right, data.right)
- def chooseKeyValue(key1:Expression, value1: Expression, key2:Expression, value2: Expression) = Seq(
- If(IsNull(key1), key2, If(IsNull(key2), key1, If(bComp(key1, key2), key1, key2))),
- If(IsNull(key1), value2, If(IsNull(key2), value1, If(bComp(key1, key2), value1, value2)))
+ private def chooseKeyValue(key1: Expression,
+ value1: Expression,
+ key2: Expression,
+ value2: Expression) = Seq(
+ If(IsNull(key1),
+ key2,
+ If(IsNull(key2), key1, If(bComp(key1, key2), key1, key2))),
+ If(IsNull(key1),
+ value2,
+ If(IsNull(key2), value1, If(bComp(key1, key2), value1, value2)))
)
override lazy val evaluateExpression: AttributeReference = data
}
-
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
index 3eed62c..c904587 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
@@ -18,23 +18,25 @@ package org.apache.spark.deploy
import java.io._
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
import datafu.spark.ScalaPythonBridge
import org.apache.log4j.Logger
+
import org.apache.spark.api.python.PythonUtils
import org.apache.spark.util.Utils
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
/**
- * We wrap Spark's PythonRunner because we failed on premature python process closing.
- * in PythonRunner the python process exits immediately when finished to read the filename,
- * this caused us to Accumulators Exceptions when the driver tries to get accumulation data
- * from the python gateway.
- * Instead, like in Zeppelin, we create an "interactive" python process, feed it the python
- * script and not closing the gateway.
- */
-case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()) {
+ * We wrap Spark's PythonRunner because we failed on premature python process closing.
+ * in PythonRunner the python process exits immediately when finished to read the filename,
+ * this caused us to Accumulators Exceptions when the driver tries to get accumulation data
+ * from the python gateway.
+ * Instead, like in Zeppelin, we create an "interactive" python process, feed it the python
+ * script and not closing the gateway.
+ */
+case class SparkPythonRunner(pyPaths: String,
+ otherArgs: Array[String] = Array()) {
val logger: Logger = Logger.getLogger(getClass)
val (reader, writer, process) = initPythonEnv()
@@ -49,7 +51,8 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
private def initPythonEnv(): (BufferedReader, BufferedWriter, Process) = {
val pythonExec =
- sys.env.getOrElse("PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python"))
+ sys.env.getOrElse("PYSPARK_DRIVER_PYTHON",
+ sys.env.getOrElse("PYSPARK_PYTHON", "python"))
// Format python filename paths before adding them to the PYTHONPATH
val formattedPyFiles = PythonRunner.formatPaths(pyPaths)
@@ -79,39 +82,33 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
pathElements += PythonUtils.sparkPythonPath
pathElements += sys.env.getOrElse("PYTHONPATH", "")
val pythonPath = PythonUtils.mergePythonPaths(pathElements: _*)
- logger.info(s"Running python with PYTHONPATH:\n\t${formattedPyFiles.mkString(",")}")
+ logger.info(
+ s"Running python with PYTHONPATH:\n\t${formattedPyFiles.mkString(",")}")
// Launch Python process
- //val builder = new ProcessBuilder((Seq(pythonExec, formattedPythonFile) ++ otherArgs).asJava)
- val builder = new ProcessBuilder((Seq(pythonExec, "-iu") ++ otherArgs).asJava)
+ val builder = new ProcessBuilder(
+ (Seq(pythonExec, "-iu") ++ otherArgs).asJava)
val env = builder.environment()
env.put("PYTHONPATH", pythonPath)
// This is equivalent to setting the -u flag; we use it because ipython doesn't support -u:
env.put("PYTHONUNBUFFERED", "YES") // value is needed to be set to a non-empty string
env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
builder.redirectErrorStream(true) // Ugly but needed for stdout and stderr to synchronize
- //try {
- val process = builder.start()
- //new RedirectThread(process.getInputStream, System.out, "redirect output").start()
- val writer = new BufferedWriter(new OutputStreamWriter(process.getOutputStream))
- val reader = new BufferedReader(new InputStreamReader(process.getInputStream))
-
- // val exitCode = process.waitFor()
- // if (exitCode != 0) {
- // throw new SparkUserAppException(exitCode)
- // }
- //} finally {
- // gatewayServer.shutdown()
- //}
+ val process = builder.start()
+ val writer = new BufferedWriter(
+ new OutputStreamWriter(process.getOutputStream))
+ val reader = new BufferedReader(
+ new InputStreamReader(process.getInputStream))
(reader, writer, process)
}
-
- private def execFile(filename: String, writer: BufferedWriter, reader: BufferedReader): String = {
+ private def execFile(filename: String,
+ writer: BufferedWriter,
+ reader: BufferedReader): String = {
writer.write("import traceback\n")
writer.write("try:\n")
- writer.write(" execfile('"+filename+"')\n")
+ writer.write(" execfile('" + filename + "')\n")
writer.write(" print (\"*!?flush reader!?*\")\n")
writer.write("except Exception as e:\n")
writer.write(" traceback.print_exc()\n")
@@ -120,19 +117,19 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
writer.flush()
var output = ""
var line: String = reader.readLine
- while (!line.contains("*!?flush reader!?*") && !line.contains("*!?flush error reader!?*")) {
- //logger.debug("Read line from python shell : " + line)
- System.out.println(line)
+ while (!line.contains("*!?flush reader!?*") && !line.contains(
+ "*!?flush error reader!?*")) {
+ logger.info(line)
if (line == "...") {
- //logger.warn("Syntax error ! ")
output += "Syntax error ! "
}
output += "\r" + line + "\n"
line = reader.readLine
}
- if (line.contains("*!?flush error reader!?*"))
+ if (line.contains("*!?flush error reader!?*")) {
throw new RuntimeException("python bridge error: " + output)
+ }
output
}
diff --git a/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource b/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
index 714c69c..072cde9 100644
--- a/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
+++ b/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
@@ -17,4 +17,4 @@
datafu.spark.ExampleFiles
datafu.spark.Py4JResource
-datafu.spark.PysparkResource
\ No newline at end of file
+datafu.spark.PysparkResource
diff --git a/datafu-spark/src/test/resources/example_tests/pyfromscala.py b/datafu-spark/src/test/resources/example_tests/pyfromscala.py
index c2c3c6f..040854b 100644
--- a/datafu-spark/src/test/resources/example_tests/pyfromscala.py
+++ b/datafu-spark/src/test/resources/example_tests/pyfromscala.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+
+# Some usage exmaples of python-Scala functionality
+
# print the PYTHONPATH
import sys
from pprint import pprint as p
diff --git a/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala b/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
index 93c9c5f..2ee8bde 100644
--- a/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
@@ -44,6 +44,10 @@ object PathsResolver {
val py4jVersion = py4js.getOrElse(sparkVersion, "0.10.6") // our default
- val pyspark = ResourceCloning.cloneResource(getClass.getResource("/built_in_pyspark_lib/pyspark-" + sparkVersion + ".zip"), "pyspark_cloned.zip").getPath
- val py4j = ResourceCloning.cloneResource(getClass.getResource("/built_in_pyspark_lib/py4j-" + py4jVersion + "-src.zip"), "py4j_cloned.zip").getPath
+ val pyspark = ResourceCloning.cloneResource(
+ getClass.getResource("/built_in_pyspark_lib/pyspark-" + sparkVersion + ".zip"),
+ "pyspark_cloned.zip").getPath
+ val py4j = ResourceCloning.cloneResource(
+ getClass.getResource("/built_in_pyspark_lib/py4j-" + py4jVersion + "-src.zip"),
+ "py4j_cloned.zip").getPath
}
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
index f3bea97..1408e2c 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
@@ -20,18 +20,22 @@ package datafu.spark
import java.io.File
+import scala.util.Try
+
import com.holdenkarau.spark.testing.Utils
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.SparkSession
import org.junit._
+import org.junit.runner.RunWith
import org.scalatest.FunSuite
import org.scalatest.junit.JUnitRunner
-import org.junit.runner.RunWith
+import org.slf4j.LoggerFactory
-import scala.util.Try
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
object TestScalaPythonBridge {
+ val logger = LoggerFactory.getLogger(this.getClass)
+
def getNewRunner(): ScalaPythonBridgeRunner = {
val runner = ScalaPythonBridgeRunner()
runner.runPythonFile("pyspark_utils/init_spark_context.py")
@@ -43,15 +47,16 @@ object TestScalaPythonBridge {
val tempDir = Utils.createTempDir()
val localMetastorePath = new File(tempDir, "metastore").getCanonicalPath
val localWarehousePath = new File(tempDir, "wharehouse").getCanonicalPath
- val pythonPath = PythonPathsManager.getAbsolutePaths().mkString(File.pathSeparator)
- println("Creating SparkConf with PYTHONPATH: " + pythonPath)
+ val pythonPath =
+ PythonPathsManager.getAbsolutePaths().mkString(File.pathSeparator)
+ logger.info("Creating SparkConf with PYTHONPATH: " + pythonPath)
val sparkConf = new SparkConf()
- .setMaster("local[1]")
- .set("spark.sql.warehouse.dir", localWarehousePath)
- .set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=$localMetastorePath;create=true")
- //.set("datanucleus.rdbms.datastoreAdapterClassName", "org.datanucleus.store.rdbms.adapter.DerbyAdapter")
- .setExecutorEnv(Seq(("PYTHONPATH", pythonPath)))
- .setAppName("Spark Unit Test")
+ .setMaster("local[1]")
+ .set("spark.sql.warehouse.dir", localWarehousePath)
+ .set("javax.jdo.option.ConnectionURL",
+ s"jdbc:derby:;databaseName=$localMetastorePath;create=true")
+ .setExecutorEnv(Seq(("PYTHONPATH", pythonPath)))
+ .setAppName("Spark Unit Test")
val builder = SparkSession.builder().config(sparkConf).enableHiveSupport()
val spark = builder.getOrCreate()
@@ -67,8 +72,9 @@ class TestScalaPythonBridge extends FunSuite {
private lazy val runner = TestScalaPythonBridge.getNewRunner()
def assertTable(tableName: String, expected: String): Unit =
- Assert.assertEquals(expected, spark.table(tableName).collect().sortBy(_.toString).mkString(", "))
-
+ Assert.assertEquals(
+ expected,
+ spark.table(tableName).collect().sortBy(_.toString).mkString(", "))
test("pyfromscala.py") {
@@ -80,11 +86,16 @@ class TestScalaPythonBridge extends FunSuite {
runner.runPythonFile("example_tests/pyfromscala.py")
// try to invoke python udf from scala code
- assert(spark.sql("select magic('python_udf')").collect().mkString(",") == "[python_udf magic]")
-
- //spark.sql("select * from dfout").registerTempTable("output")
- assertTable("dfout", "[10], [12], [14], [16], [18], [20], [2], [4], [6], [8]")
- assertTable("dfout2", "[16], [24], [32], [40], [48], [56], [64], [72], [80], [8]")
+ assert(
+ spark
+ .sql("select magic('python_udf')")
+ .collect()
+ .mkString(",") == "[python_udf magic]")
+
+ assertTable("dfout",
+ "[10], [12], [14], [16], [18], [20], [2], [4], [6], [8]")
+ assertTable("dfout2",
+ "[16], [24], [32], [40], [48], [56], [64], [72], [80], [8]")
assertTable("stats", "[a,0.1], [b,2.0]")
}
@@ -93,17 +104,24 @@ class TestScalaPythonBridge extends FunSuite {
assert(t.isFailure)
assert(t.failed.get.isInstanceOf[RuntimeException])
}
-
+
test("SparkDFUtilsBridge") {
runner.runPythonFile("example_tests/df_utils_tests.py")
assertTable("dedup", "[a,Alice,34], [b,Bob,36], [c,Zoey,36]")
- assertTable("dedupTopN", "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,Fanny,36], [c,Zoey,36]")
+ assertTable(
+ "dedupTopN",
+ "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,Fanny,36], [c,Zoey,36]")
assertTable("dedup2", "[a,34], [b,36], [c,36]")
- assertTable("changeSchema", "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], [c,Fanny,36], [c,Zoey,36]")
+ assertTable(
+ "changeSchema",
+ "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], " +
+ "[c,Fanny,36], [c,Zoey,36]")
assertTable("joinSkewed", "[a,Laura,34,a,1], [a,Stephani,33,a,1]")
assertTable("broadcastJoinSkewed", "[a,Laura,34,1], [a,Stephani,33,1]")
- assertTable("joinWithRange", "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
- assertTable("joinWithRangeAndDedup", "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
+ assertTable("joinWithRange",
+ "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
+ assertTable("joinWithRangeAndDedup",
+ "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
}
}
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala b/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
index 988e206..8c7453e 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
@@ -18,36 +18,28 @@
*/
package datafu.spark
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.Row
-
-import org.apache.spark.sql.functions._
-
import com.holdenkarau.spark.testing.DataFrameSuiteBase
-
+import org.junit.runner.RunWith
import org.scalatest.FunSuite
-
import org.scalatest.junit.JUnitRunner
-import org.junit.runner.RunWith
-
-import scala.collection.mutable.WrappedArray
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
-import org.apache.spark.rdd.RDD
@RunWith(classOf[JUnitRunner])
class DataFrameOpsTests extends FunSuite with DataFrameSuiteBase {
-
+
import DataFrameOps._
-
- import spark.implicits._
- val inputSchema = List(
- StructField("col_grp", StringType, true),
- StructField("col_ord", IntegerType, false),
- StructField("col_str", StringType, true)
+ import spark.implicits._
+
+ val inputSchema = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, false),
+ StructField("col_str", StringType, true)
)
val dedupSchema = List(
@@ -55,230 +47,332 @@ class DataFrameOpsTests extends FunSuite with DataFrameSuiteBase {
StructField("col_ord", IntegerType, false)
)
- lazy val inputRDD = sc.parallelize(Seq(Row("a", 1, "asd1"), Row("a", 2, "asd2"), Row("a", 3, "asd3"), Row("b", 1, "asd4")))
-
- lazy val inputDataFrame = sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
+ lazy val inputRDD = sc.parallelize(
+ Seq(Row("a", 1, "asd1"),
+ Row("a", 2, "asd2"),
+ Row("a", 3, "asd3"),
+ Row("b", 1, "asd4")))
- test("dedup") {
- val expected : DataFrame = sqlContext.createDataFrame(sc.parallelize(Seq(Row("b",1),Row("a",3))), StructType(dedupSchema))
+ lazy val inputDataFrame =
+ sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
- assertDataFrameEquals(expected, inputDataFrame.dedup($"col_grp", $"col_ord".desc).select($"col_grp", $"col_ord"))
+ test("dedup") {
+ val expected: DataFrame =
+ sqlContext.createDataFrame(sc.parallelize(Seq(Row("b", 1), Row("a", 3))),
+ StructType(dedupSchema))
+
+ assertDataFrameEquals(expected,
+ inputDataFrame
+ .dedup($"col_grp", $"col_ord".desc)
+ .select($"col_grp", $"col_ord"))
}
- case class dedup_exp(col2: String, col_grp:String, col_ord:Option[Int], col_str:String)
+ case class dedupExp(col2: String,
+ col_grp: String,
+ col_ord: Option[Int],
+ col_str: String)
test("dedup2_by_int") {
- val expectedByIntDf : DataFrame = sqlContext.createDataFrame(List(dedup_exp("asd4","b",Option(1),"asd4"),dedup_exp("asd1","a",Option(3),"asd3")))
+ val expectedByIntDf: DataFrame = sqlContext.createDataFrame(
+ List(dedupExp("asd4", "b", Option(1), "asd4"),
+ dedupExp("asd1", "a", Option(3), "asd3")))
- val actual = inputDataFrame.dedup2($"col_grp", $"col_ord", moreAggFunctions = Seq(min($"col_str")))
+ val actual = inputDataFrame.dedup2($"col_grp",
+ $"col_ord",
+ moreAggFunctions = Seq(min($"col_str")))
assertDataFrameEquals(expectedByIntDf, actual)
}
- case class dedup_exp2(col_grp:String, col_ord:Option[Int], col_str:String)
+ case class dedupExp2(col_grp: String, col_ord: Option[Int], col_str: String)
- test("dedup2_by_string_asc") {
+ test("dedup2_by_string_asc") {
- val actual = inputDataFrame.dedup2($"col_grp", $"col_str", desc = false)
-
- val expectedByStringDf : DataFrame = sqlContext.createDataFrame(List(dedup_exp2("b",Option(1),"asd4"),dedup_exp2("a",Option(1),"asd1")))
+ val actual = inputDataFrame.dedup2($"col_grp", $"col_str", desc = false)
- assertDataFrameEquals(expectedByStringDf, actual)
- }
+ val expectedByStringDf: DataFrame = sqlContext.createDataFrame(
+ List(dedupExp2("b", Option(1), "asd4"),
+ dedupExp2("a", Option(1), "asd1")))
- test("test_dedup2_by_complex_column") {
+ assertDataFrameEquals(expectedByStringDf, actual)
+ }
- val actual = inputDataFrame.dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"), desc = false)
-
- val expectedComplex : DataFrame = sqlContext.createDataFrame(List(dedup_exp2("b",Option(1),"asd4"),dedup_exp2("a",Option(3),"asd3")))
+ test("test_dedup2_by_complex_column") {
- assertDataFrameEquals(expectedComplex, actual)
- }
+ val actual = inputDataFrame.dedup2($"col_grp",
+ expr("cast(concat('-',col_ord) as int)"),
+ desc = false)
- case class inner(col_grp:String, col_ord:Int)
-
- case class expComplex(
- col_grp: String,
- col_ord: Option[Int],
- col_str: String,
- arr_col: Array[String],
- struct_col: inner,
- map_col: Map[String, Int]
- )
-
- test("test_dedup2_with_other_complex_column") {
-
- val actual = inputDataFrame.withColumn("arr_col", expr("array(col_grp, col_ord)"))
- .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
- .withColumn("map_col", expr("map(col_grp, col_ord)"))
- .withColumn("map_col_blah", expr("map(col_grp, col_ord)"))
- .dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"))
- .drop("map_col_blah")
-
- val expected : DataFrame = sqlContext.createDataFrame(List(
- expComplex("b", Option(1), "asd4", Array("b","1"), inner("b",1), Map("b" -> 1)),
- expComplex("a", Option(1), "asd1", Array("a","1"), inner("a",1), Map("a" -> 1))
+ val expectedComplex: DataFrame = sqlContext.createDataFrame(
+ List(dedupExp2("b", Option(1), "asd4"),
+ dedupExp2("a", Option(3), "asd3")))
+
+ assertDataFrameEquals(expectedComplex, actual)
+ }
+
+ case class Inner(colGrp: String, colOrd: Int)
+
+ case class expComplex(
+ col_grp: String,
+ col_ord: Option[Int],
+ col_str: String,
+ arr_col: Array[String],
+ struct_col: Inner,
+ map_col: Map[String, Int]
+ )
+
+ test("test_dedup2_with_other_complex_column") {
+
+ val actual = inputDataFrame
+ .withColumn("arr_col", expr("array(col_grp, col_ord)"))
+ .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
+ .withColumn("map_col", expr("map(col_grp, col_ord)"))
+ .withColumn("map_col_blah", expr("map(col_grp, col_ord)"))
+ .dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"))
+ .drop("map_col_blah")
+
+ val expected: DataFrame = sqlContext.createDataFrame(
+ List(
+ expComplex("b",
+ Option(1),
+ "asd4",
+ Array("b", "1"),
+ Inner("b", 1),
+ Map("b" -> 1)),
+ expComplex("a",
+ Option(1),
+ "asd1",
+ Array("a", "1"),
+ Inner("a", 1),
+ Map("a" -> 1))
))
-
- assertDataFrameEquals(expected, actual)
- }
- val dedupTopNExpectedSchema = List(
- StructField("col_grp", StringType, true),
- StructField("col_ord", IntegerType, false)
- )
+ assertDataFrameEquals(expected, actual)
+ }
- test("test_dedup_top_n") {
- val actual = inputDataFrame.dedupTopN(2, $"col_grp", $"col_ord".desc).select($"col_grp", $"col_ord")
-
- val expected = sqlContext.createDataFrame(sc.parallelize(Seq(Row("b",1), Row("a",3), Row("a",2))), StructType(dedupTopNExpectedSchema))
-
- assertDataFrameEquals(expected, actual)
- }
-
- val schema2 = List(
- StructField("start", IntegerType, false),
- StructField("end", IntegerType, false),
- StructField("desc", StringType, true)
- )
+ val dedupTopNExpectedSchema = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, false)
+ )
- val expectedSchemaRangedJoin = List(
- StructField("col_grp",StringType,true),
- StructField("col_ord",IntegerType,false),
- StructField("col_str",StringType,true),
- StructField("start",IntegerType,true),
- StructField("end",IntegerType,true),
- StructField("desc",StringType,true)
+ test("test_dedup_top_n") {
+ val actual = inputDataFrame
+ .dedupTopN(2, $"col_grp", $"col_ord".desc)
+ .select($"col_grp", $"col_ord")
+
+ val expected = sqlContext.createDataFrame(
+ sc.parallelize(Seq(Row("b", 1), Row("a", 3), Row("a", 2))),
+ StructType(dedupTopNExpectedSchema))
+
+ assertDataFrameEquals(expected, actual)
+ }
+
+ val schema2 = List(
+ StructField("start", IntegerType, false),
+ StructField("end", IntegerType, false),
+ StructField("desc", StringType, true)
+ )
+
+ val expectedSchemaRangedJoin = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, false),
+ StructField("col_str", StringType, true),
+ StructField("start", IntegerType, true),
+ StructField("end", IntegerType, true),
+ StructField("desc", StringType, true)
+ )
+
+ test("join_with_range") {
+ val joinWithRangeDataFrame =
+ sqlContext.createDataFrame(sc.parallelize(
+ Seq(Row(1, 2, "asd1"),
+ Row(1, 4, "asd2"),
+ Row(3, 5, "asd3"),
+ Row(3, 10, "asd4"))),
+ StructType(schema2))
+
+ val expected = sqlContext.createDataFrame(
+ sc.parallelize(
+ Seq(
+ Row("b", 1, "asd4", 1, 2, "asd1"),
+ Row("a", 2, "asd2", 1, 2, "asd1"),
+ Row("a", 1, "asd1", 1, 2, "asd1"),
+ Row("b", 1, "asd4", 1, 4, "asd2"),
+ Row("a", 3, "asd3", 1, 4, "asd2"),
+ Row("a", 2, "asd2", 1, 4, "asd2"),
+ Row("a", 1, "asd1", 1, 4, "asd2"),
+ Row("a", 3, "asd3", 3, 5, "asd3"),
+ Row("a", 3, "asd3", 3, 10, "asd4")
+ )),
+ StructType(expectedSchemaRangedJoin)
)
-
- test("join_with_range") {
- val joinWithRangeDataFrame = sqlContext.createDataFrame(sc.parallelize(Seq(Row(1, 2,"asd1"), Row(1, 4, "asd2"), Row(3, 5,"asd3"), Row(3, 10,"asd4"))),StructType(schema2))
-
- val expected = sqlContext.createDataFrame(sc.parallelize(Seq(
- Row("b",1,"asd4",1,2,"asd1"),
- Row("a",2,"asd2",1,2,"asd1"),
- Row("a",1,"asd1",1,2,"asd1"),
- Row("b",1,"asd4",1,4,"asd2"),
- Row("a",3,"asd3",1,4,"asd2"),
- Row("a",2,"asd2",1,4,"asd2"),
- Row("a",1,"asd1",1,4,"asd2"),
- Row("a",3,"asd3",3,5,"asd3"),
- Row("a",3,"asd3",3,10,"asd4")
- )), StructType(expectedSchemaRangedJoin))
-
- val actual = inputDataFrame.joinWithRange("col_ord", joinWithRangeDataFrame, "start", "end")
-
- assertDataFrameEquals(expected, actual)
+
+ val actual = inputDataFrame.joinWithRange("col_ord",
+ joinWithRangeDataFrame,
+ "start",
+ "end")
+
+ assertDataFrameEquals(expected, actual)
}
- val expectedSchemaRangedJoinWithDedup = List(
- StructField("col_grp",StringType,true),
- StructField("col_ord",IntegerType,true),
- StructField("col_str",StringType,true),
- StructField("start",IntegerType,true),
- StructField("end",IntegerType,true),
- StructField("desc",StringType,true)
+ val expectedSchemaRangedJoinWithDedup = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, true),
+ StructField("col_str", StringType, true),
+ StructField("start", IntegerType, true),
+ StructField("end", IntegerType, true),
+ StructField("desc", StringType, true)
+ )
+
+ test("join_with_range_and_dedup") {
+ val df = sc
+ .parallelize(
+ List(("a", 1, "asd1"),
+ ("a", 2, "asd2"),
+ ("a", 3, "asd3"),
+ ("b", 1, "asd4")))
+ .toDF("col_grp", "col_ord", "col_str")
+ val dfr = sc
+ .parallelize(
+ List((1, 2, "asd1"), (1, 4, "asd2"), (3, 5, "asd3"), (3, 10, "asd4")))
+ .toDF("start", "end", "desc")
+
+ val expected = sqlContext.createDataFrame(
+ sc.parallelize(
+ Seq(
+ Row("b", 1, "asd4", 1, 2, "asd1"),
+ Row("a", 3, "asd3", 3, 5, "asd3"),
+ Row("a", 2, "asd2", 1, 2, "asd1")
+ )),
+ StructType(expectedSchemaRangedJoinWithDedup)
)
- test("join_with_range_and_dedup") {
- val df = sc.parallelize(List(("a", 1, "asd1"), ("a", 2, "asd2"), ("a", 3, "asd3"), ("b", 1, "asd4"))).toDF("col_grp", "col_ord", "col_str")
- val dfr = sc.parallelize(List((1, 2,"asd1"), (1, 4, "asd2"), (3, 5,"asd3"), (3, 10,"asd4"))).toDF("start", "end", "desc")
-
- val expected = sqlContext.createDataFrame(sc.parallelize(Seq(
- Row("b",1,"asd4",1,2,"asd1"),
- Row("a",3,"asd3",3,5,"asd3"),
- Row("a",2,"asd2",1,2,"asd1")
- )), StructType(expectedSchemaRangedJoinWithDedup))
-
-
- val actual = df.joinWithRangeAndDedup("col_ord", dfr, "start", "end")
-
- assertDataFrameEquals(expected, actual)
- }
-
- test("broadcastJoinSkewed") {
- val skewedList = List(("1", "a"), ("1", "b"), ("1", "c"), ("1", "d"), ("1", "e"),("2", "k"),("0", "k"))
- val skewed = sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
- val notSkewed = sqlContext.createDataFrame((1 to 10).map(i => (i.toString, s"str$i"))).toDF("key", "val")
-
- val expected = sqlContext.createDataFrame(List(
- ("1","str1", "a"),
- ("1","str1", "b"),
- ("1","str1", "c"),
- ("1","str1", "d"),
- ("1","str1", "e"),
- ("2","str2", "k")
- )).toDF("key","val","val_skewed")
-
- val actual1 = notSkewed.broadcastJoinSkewed(skewed,"key", 1)
-
- assertDataFrameEquals(expected, actual1.sort($"val_skewed"))
-
- val actual2 = notSkewed.broadcastJoinSkewed(skewed,"key", 2)
-
- assertDataFrameEquals(expected, actual2.sort($"val_skewed"))
- }
-
- // because of nulls in expected data, an actual schema needs to be used
- case class expJoinSkewed(str1:String, str2:String, str3:String, str4:String )
-
- test("joinSkewed") {
- val skewedList = List(("1", "a"), ("1", "b"), ("1", "c"), ("1", "d"), ("1", "e"),("2", "k"),("0", "k"))
- val skewed = sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
- val notSkewed = sqlContext.createDataFrame((1 to 10).map(i => (i.toString, s"str$i"))).toDF("key", "val")
-
- val actual1 = skewed.as("a").joinSkewed(notSkewed.as("b"),expr("a.key = b.key"), 3)
-
- val expected1 = sqlContext.createDataFrame(List(
- ("1","a","1","str1"),
- ("1","b","1","str1"),
- ("1","c","1","str1"),
- ("1","d","1","str1"),
- ("1","e","1","str1"),
- ("2","k","2","str2")
- )).toDF("key","val_skewed","key","val")
-
- assertDataFrameEquals(expected1, actual1.sort($"val_skewed")) // assertDataFrameEquals cares about order but we don't
-
- val actual2 = skewed.as("a").joinSkewed(notSkewed.as("b"),expr("a.key = b.key"), 3, "left_outer")
-
- val expected2 = sqlContext.createDataFrame(List(
- expJoinSkewed("1","a","1","str1"),
- expJoinSkewed("1","b","1","str1"),
- expJoinSkewed("1","c","1","str1"),
- expJoinSkewed("1","d","1","str1"),
- expJoinSkewed("1","e","1","str1"),
- expJoinSkewed("2","k","2","str2"),
- expJoinSkewed("0","k",null,null)
- )).toDF("key","val_skewed","key","val")
-
- assertDataFrameEquals(expected2, actual2.sort($"val_skewed")) // assertDataFrameEquals cares about order but we don't
- }
-
- val changedSchema = List(
- StructField("fld1", StringType, true),
- StructField("fld2", IntegerType, false),
- StructField("fld3", StringType, true)
+ val actual = df.joinWithRangeAndDedup("col_ord", dfr, "start", "end")
+
+ assertDataFrameEquals(expected, actual)
+ }
+
+ test("broadcastJoinSkewed") {
+ val skewedList = List(("1", "a"),
+ ("1", "b"),
+ ("1", "c"),
+ ("1", "d"),
+ ("1", "e"),
+ ("2", "k"),
+ ("0", "k"))
+ val skewed =
+ sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
+ val notSkewed = sqlContext
+ .createDataFrame((1 to 10).map(i => (i.toString, s"str$i")))
+ .toDF("key", "val")
+
+ val expected = sqlContext
+ .createDataFrame(
+ List(
+ ("1", "str1", "a"),
+ ("1", "str1", "b"),
+ ("1", "str1", "c"),
+ ("1", "str1", "d"),
+ ("1", "str1", "e"),
+ ("2", "str2", "k")
+ ))
+ .toDF("key", "val", "val_skewed")
+
+ val actual1 = notSkewed.broadcastJoinSkewed(skewed, "key", 1)
+
+ assertDataFrameEquals(expected, actual1.sort($"val_skewed"))
+
+ val actual2 = notSkewed.broadcastJoinSkewed(skewed, "key", 2)
+
+ assertDataFrameEquals(expected, actual2.sort($"val_skewed"))
+ }
+
+ // because of nulls in expected data, an actual schema needs to be used
+ case class expJoinSkewed(str1: String,
+ str2: String,
+ str3: String,
+ str4: String)
+
+ test("joinSkewed") {
+ val skewedList = List(("1", "a"),
+ ("1", "b"),
+ ("1", "c"),
+ ("1", "d"),
+ ("1", "e"),
+ ("2", "k"),
+ ("0", "k"))
+ val skewed =
+ sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
+ val notSkewed = sqlContext
+ .createDataFrame((1 to 10).map(i => (i.toString, s"str$i")))
+ .toDF("key", "val")
+
+ val actual1 =
+ skewed.as("a").joinSkewed(notSkewed.as("b"), expr("a.key = b.key"), 3)
+
+ val expected1 = sqlContext
+ .createDataFrame(
+ List(
+ ("1", "a", "1", "str1"),
+ ("1", "b", "1", "str1"),
+ ("1", "c", "1", "str1"),
+ ("1", "d", "1", "str1"),
+ ("1", "e", "1", "str1"),
+ ("2", "k", "2", "str2")
+ ))
+ .toDF("key", "val_skewed", "key", "val")
+
+ // assertDataFrameEquals cares about order but we don't
+ assertDataFrameEquals(expected1, actual1.sort($"val_skewed"))
+
+ val actual2 = skewed
+ .as("a")
+ .joinSkewed(notSkewed.as("b"), expr("a.key = b.key"), 3, "left_outer")
+
+ val expected2 = sqlContext
+ .createDataFrame(
+ List(
+ expJoinSkewed("1", "a", "1", "str1"),
+ expJoinSkewed("1", "b", "1", "str1"),
+ expJoinSkewed("1", "c", "1", "str1"),
+ expJoinSkewed("1", "d", "1", "str1"),
+ expJoinSkewed("1", "e", "1", "str1"),
+ expJoinSkewed("2", "k", "2", "str2"),
+ expJoinSkewed("0", "k", null, null)
+ ))
+ .toDF("key", "val_skewed", "key", "val")
+
+ // assertDataFrameEquals cares about order but we don't
+ assertDataFrameEquals(expected2, actual2.sort($"val_skewed"))
+ }
+
+ val changedSchema = List(
+ StructField("fld1", StringType, true),
+ StructField("fld2", IntegerType, false),
+ StructField("fld3", StringType, true)
)
-
- test("test_changeSchema") {
-
- val actual = inputDataFrame.changeSchema("fld1","fld2", "fld3")
-
- val expected = sqlContext.createDataFrame(inputRDD, StructType(changedSchema))
-
- assertDataFrameEquals(expected, actual)
- }
-
- test("test_flatten") {
-
- val input = inputDataFrame.withColumn("struct_col", expr("struct(col_grp, col_ord)")).select("struct_col")
-
- val expected : DataFrame = inputDataFrame.select("col_grp", "col_ord")
-
- val actual = input.flatten("struct_col")
-
- assertDataFrameEquals(expected, actual)
- }
-}
\ No newline at end of file
+
+ test("test_changeSchema") {
+
+ val actual = inputDataFrame.changeSchema("fld1", "fld2", "fld3")
+
+ val expected =
+ sqlContext.createDataFrame(inputRDD, StructType(changedSchema))
+
+ assertDataFrameEquals(expected, actual)
+ }
+
+ test("test_flatten") {
+
+ val input = inputDataFrame
+ .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
+ .select("struct_col")
+
+ val expected: DataFrame = inputDataFrame.select("col_grp", "col_ord")
+
+ val actual = input.flatten("struct_col")
+
+ assertDataFrameEquals(expected, actual)
+ }
+}
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala b/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
index 4a4be37..6fe906c 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
@@ -19,19 +19,18 @@
package datafu.spark
import com.holdenkarau.spark.testing.DataFrameSuiteBase
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.expressions.Window
-import org.apache.spark.sql.types.SparkOverwriteUDAFs
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.Row
import org.junit.Assert
import org.junit.runner.RunWith
import org.scalatest.FunSuite
import org.scalatest.junit.JUnitRunner
import org.slf4j.LoggerFactory
+
import org.apache.spark.SparkConf
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
-import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.types._
@RunWith(classOf[JUnitRunner])
class UdafTests extends FunSuite with DataFrameSuiteBase {
@@ -39,30 +38,37 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
import spark.implicits._
/**
- * taken from https://github.com/holdenk/spark-testing-base/issues/234#issuecomment-390150835
- *
- * Solves problem with Hive in Spark 2.3.0 in spark-testing-base
- */
- override def conf: SparkConf = super.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
-
+ * taken from https://github.com/holdenk/spark-testing-base/issues/234#issuecomment-390150835
+ *
+ * Solves problem with Hive in Spark 2.3.0 in spark-testing-base
+ */
+ override def conf: SparkConf =
+ super.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
+
val logger = LoggerFactory.getLogger(this.getClass)
- val inputSchema = List(
- StructField("col_grp", StringType, true),
- StructField("col_ord", IntegerType, false),
- StructField("col_str", StringType, true)
+ val inputSchema = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, false),
+ StructField("col_str", StringType, true)
)
- lazy val inputRDD = sc.parallelize(Seq(Row("a", 1, "asd1"), Row("a", 2, "asd2"), Row("a", 3, "asd3"), Row("b", 1, "asd4")))
-
- lazy val df = sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
+ lazy val inputRDD = sc.parallelize(
+ Seq(Row("a", 1, "asd1"),
+ Row("a", 2, "asd2"),
+ Row("a", 3, "asd3"),
+ Row("b", 1, "asd4")))
+
+ lazy val df =
+ sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
case class mapExp(map_col: Map[String, Int])
case class mapArrExp(map_col: Map[String, Array[String]])
test("test multiset simple") {
val ms = new SparkUDAFs.MultiSet()
- val expected : DataFrame = sqlContext.createDataFrame(List(mapExp(Map("b" -> 1, "a" -> 3))))
+ val expected: DataFrame =
+ sqlContext.createDataFrame(List(mapExp(Map("b" -> 1, "a" -> 3))))
assertDataFrameEquals(expected, df.agg(ms($"col_grp").as("map_col")))
}
@@ -71,23 +77,34 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
test("test multiarrayset simple") {
assertDataFrameEquals(
sqlContext.createDataFrame(List(mapExp(Map("tre" -> 1, "asd" -> 2)))),
- spark.sql("select array('asd','tre','asd') arr").groupBy().agg(mas($"arr").as("map_col")))
+ spark
+ .sql("select array('asd','tre','asd') arr")
+ .groupBy()
+ .agg(mas($"arr").as("map_col"))
+ )
}
test("test multiarrayset all nulls") {
// end case
spark.sql("drop table if exists mas_table")
spark.sql("create table mas_table (arr array<string>)")
- spark.sql("insert overwrite table mas_table select case when 1=2 then array('asd') end from (select 1)z")
- spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
- spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
- spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
- spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+ spark.sql(
+ "insert overwrite table mas_table select case when 1=2 then array('asd') end " +
+ "from (select 1)z")
+ spark.sql(
+ "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+ spark.sql(
+ "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+ spark.sql(
+ "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+ spark.sql(
+ "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
val expected = sqlContext.createDataFrame(List(mapExp(Map())))
-
- val actual = spark.table("mas_table").groupBy().agg(mas($"arr").as("map_col"))
-
+
+ val actual =
+ spark.table("mas_table").groupBy().agg(mas($"arr").as("map_col"))
+
assertDataFrameEquals(expected, actual)
}
@@ -95,12 +112,18 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
// max keys case
spark.sql("drop table if exists mas_table2")
spark.sql("create table mas_table2 (arr array<string>)")
- spark.sql("insert overwrite table mas_table2 select array('asd','dsa') from (select 1)z")
- spark.sql("insert into table mas_table2 select array('asd','abc') from (select 1)z")
- spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
- spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
- spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
- spark.sql("insert into table mas_table2 select array('asd2') from (select 1)z")
+ spark.sql(
+ "insert overwrite table mas_table2 select array('asd','dsa') from (select 1)z")
+ spark.sql(
+ "insert into table mas_table2 select array('asd','abc') from (select 1)z")
+ spark.sql(
+ "insert into table mas_table2 select array('asd') from (select 1)z")
+ spark.sql(
+ "insert into table mas_table2 select array('asd') from (select 1)z")
+ spark.sql(
+ "insert into table mas_table2 select array('asd') from (select 1)z")
+ spark.sql(
+ "insert into table mas_table2 select array('asd2') from (select 1)z")
val mas2 = new SparkUDAFs.MultiArraySet[String](maxKeys = 2)
@@ -116,15 +139,23 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
test("test multiarrayset big input") {
val N = 100000
- val blah = spark.sparkContext.parallelize(1 to N, 20).toDF("num").selectExpr("array('asd',concat('dsa',num)) as arr")
+ val blah = spark.sparkContext
+ .parallelize(1 to N, 20)
+ .toDF("num")
+ .selectExpr("array('asd',concat('dsa',num)) as arr")
val mas = new SparkUDAFs.MultiArraySet[String](maxKeys = 3)
val time1 = System.currentTimeMillis()
- val mp = blah.groupBy().agg(mas($"arr")).collect().map(_.getMap[String,Int](0)).head
+ val mp = blah
+ .groupBy()
+ .agg(mas($"arr"))
+ .collect()
+ .map(_.getMap[String, Int](0))
+ .head
Assert.assertEquals(3, mp.size)
- Assert.assertEquals("asd", mp.maxBy(_._2)._1 )
- Assert.assertEquals(N, mp.maxBy(_._2)._2 )
+ Assert.assertEquals("asd", mp.maxBy(_._2)._1)
+ Assert.assertEquals(N, mp.maxBy(_._2)._2)
val time2 = System.currentTimeMillis()
- logger.info("time took: " + (time2-time1)/1000 + " secs")
+ logger.info("time took: " + (time2 - time1) / 1000 + " secs")
}
test("test mapmerge") {
@@ -132,72 +163,99 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
spark.sql("drop table if exists mapmerge_table")
spark.sql("create table mapmerge_table (c map<string, array<string>>)")
- spark.sql("insert overwrite table mapmerge_table select map('k1', array('v1')) from (select 1) z")
- spark.sql("insert into table mapmerge_table select map('k1', array('v1')) from (select 1) z")
- spark.sql("insert into table mapmerge_table select map('k2', array('v3')) from (select 1) z")
+ spark.sql(
+ "insert overwrite table mapmerge_table select map('k1', array('v1')) from (select 1) z")
+ spark.sql(
+ "insert into table mapmerge_table select map('k1', array('v1')) from (select 1) z")
+ spark.sql(
+ "insert into table mapmerge_table select map('k2', array('v3')) from (select 1) z")
assertDataFrameEquals(
- sqlContext.createDataFrame(List(mapArrExp(Map("k1" -> Array("v1"), "k2" -> Array("v3"))))),
- spark.table("mapmerge_table").groupBy().agg(mapMerge($"c").as("map_col")))
+ sqlContext.createDataFrame(
+ List(mapArrExp(Map("k1" -> Array("v1"), "k2" -> Array("v3"))))),
+ spark.table("mapmerge_table").groupBy().agg(mapMerge($"c").as("map_col"))
+ )
}
test("minKeyValue") {
assertDataFrameEquals(
sqlContext.createDataFrame(List(("b", "asd4"), ("a", "asd1"))),
- df.groupBy($"col_grp".as("_1")).agg(SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").as("_2")))
+ df.groupBy($"col_grp".as("_1"))
+ .agg(SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").as("_2"))
+ )
}
- case class exp4(col_grp: String, col_ord: Int, col_str: String, asd: String)
+ case class Exp4(colGrp: String, colOrd: Int, colStr: String, asd: String)
- val minKeyValueWindowExpectedSchema = List(
- StructField("col_grp",StringType,true),
- StructField("col_ord",IntegerType,false),
- StructField("col_str",StringType,true),
- StructField("asd",StringType,true)
- )
+ val minKeyValueWindowExpectedSchema = List(
+ StructField("col_grp", StringType, true),
+ StructField("col_ord", IntegerType, false),
+ StructField("col_str", StringType, true),
+ StructField("asd", StringType, true)
+ )
test("minKeyValue window") {
assertDataFrameEquals(
- sqlContext.createDataFrame(sc.parallelize(Seq(
- Row("b", 1, "asd4", "asd4"),
- Row("a", 1, "asd1", "asd1"),
- Row("a", 2, "asd2", "asd1"),
- Row("a", 3, "asd3", "asd1")
- )), StructType(minKeyValueWindowExpectedSchema)),
- df.withColumn("asd", SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").over(Window.partitionBy("col_grp"))))
+ sqlContext.createDataFrame(
+ sc.parallelize(
+ Seq(
+ Row("b", 1, "asd4", "asd4"),
+ Row("a", 1, "asd1", "asd1"),
+ Row("a", 2, "asd2", "asd1"),
+ Row("a", 3, "asd3", "asd1")
+ )),
+ StructType(minKeyValueWindowExpectedSchema)
+ ),
+ df.withColumn("asd",
+ SparkOverwriteUDAFs
+ .minValueByKey($"col_ord", $"col_str")
+ .over(Window.partitionBy("col_grp")))
+ )
}
- case class exp5(col_grp: String, col_ord: Option[Int])
-
+ case class Exp5(colGrp: String, colOrd: Option[Int])
+
test("countDistinctUpTo") {
import datafu.spark.SparkUDAFs.CountDistinctUpTo
-
+
val countDistinctUpTo3 = new CountDistinctUpTo(3)
val countDistinctUpTo6 = new CountDistinctUpTo(6)
-
- val inputDF = sqlContext.createDataFrame(List(
- exp5("b", Option(1)),
- exp5("a", Option(1)),
- exp5("a", Option(2)),
- exp5("a", Option(3)),
- exp5("a", Option(4))
+
+ val inputDF = sqlContext.createDataFrame(
+ List(
+ Exp5("b", Option(1)),
+ Exp5("a", Option(1)),
+ Exp5("a", Option(2)),
+ Exp5("a", Option(3)),
+ Exp5("a", Option(4))
))
- val results3DF = sqlContext.createDataFrame(List(
- exp5("b", Option(1)),
- exp5("a", Option(3))
- ))
-
- val results6DF = sqlContext.createDataFrame(List(
- exp5("b", Option(1)),
- exp5("a", Option(4))
- ))
-
- inputDF.groupBy("col_grp").agg(countDistinctUpTo3($"col_ord").as("col_ord")).show
-
- assertDataFrameEquals(results3DF, inputDF.groupBy("col_grp").agg(countDistinctUpTo3($"col_ord").as("col_ord")))
-
- assertDataFrameEquals(results6DF, inputDF.groupBy("col_grp").agg(countDistinctUpTo6($"col_ord").as("col_ord")))
+ val results3DF = sqlContext.createDataFrame(
+ List(
+ Exp5("b", Option(1)),
+ Exp5("a", Option(3))
+ ))
+
+ val results6DF = sqlContext.createDataFrame(
+ List(
+ Exp5("b", Option(1)),
+ Exp5("a", Option(4))
+ ))
+
+ inputDF
+ .groupBy("col_grp")
+ .agg(countDistinctUpTo3($"col_ord").as("col_ord"))
+ .show
+
+ assertDataFrameEquals(results3DF,
+ inputDF
+ .groupBy("col_grp")
+ .agg(countDistinctUpTo3($"col_ord").as("col_ord")))
+
+ assertDataFrameEquals(results6DF,
+ inputDF
+ .groupBy("col_grp")
+ .agg(countDistinctUpTo6($"col_ord").as("col_ord")))
}
-
+
}