You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by ey...@apache.org on 2019/05/15 14:13:36 UTC

[datafu] 01/02: code reformatting according to Spark's scalastyle

This is an automated email from the ASF dual-hosted git repository.

eyal pushed a commit to branch spark-tmp
in repository https://gitbox.apache.org/repos/asf/datafu.git

commit 8a5ccc89a1b38df4abd581d2a58c8f572d105df9
Author: oraviv <or...@paypal.com>
AuthorDate: Wed May 15 13:40:28 2019 +0300

    code reformatting according to Spark's scalastyle
---
 .../META-INF/services/datafu.spark.PythonResource  |   2 +-
 .../src/main/scala/datafu/spark/DataFrameOps.scala |  65 ++-
 .../scala/datafu/spark/PythonPathsManager.scala    | 141 +++---
 .../scala/datafu/spark/ScalaPythonBridge.scala     |  58 ++-
 .../src/main/scala/datafu/spark/SparkDFUtils.scala | 408 +++++++++++-----
 .../src/main/scala/datafu/spark/SparkUDAFs.scala   | 168 ++++---
 .../utils/overwrites/SparkOverwriteUDAFs.scala     |  38 +-
 .../spark/utils/overwrites/SparkPythonRunner.scala |  69 ++-
 .../META-INF/services/datafu.spark.PythonResource  |   2 +-
 .../test/resources/example_tests/pyfromscala.py    |   4 +-
 .../datafu/spark/PySparkLibTestResources.scala     |   8 +-
 .../scala/datafu/spark/TestScalaPythonBridge.scala |  66 ++-
 .../test/scala/datafu/spark/TestSparkDFUtils.scala | 528 ++++++++++++---------
 .../test/scala/datafu/spark/TestSparkUDAFs.scala   | 230 +++++----
 14 files changed, 1134 insertions(+), 653 deletions(-)

diff --git a/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource b/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
index d43f740..ee69a0d 100644
--- a/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
+++ b/datafu-spark/src/main/resources/META-INF/services/datafu.spark.PythonResource
@@ -15,4 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-datafu.spark.CoreBridgeDirectory
\ No newline at end of file
+datafu.spark.CoreBridgeDirectory
diff --git a/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala b/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
index f1ac607..110f13f 100644
--- a/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/DataFrameOps.scala
@@ -30,26 +30,63 @@ object DataFrameOps {
     def dedupTopN(n: Int, groupCol: Column, orderCols: Column*): DataFrame =
       SparkDFUtils.dedupTopN(df, n, groupCol, orderCols: _*)
 
-    def dedup2(groupCol: Column, orderByCol: Column, desc: Boolean = true, moreAggFunctions: Seq[Column] = Nil, columnsFilter: Seq[String] = Nil, columnsFilterKeep: Boolean = true): DataFrame =
-      SparkDFUtils.dedup2(df, groupCol, orderByCol, desc, moreAggFunctions, columnsFilter, columnsFilterKeep)
+    def dedup2(groupCol: Column,
+               orderByCol: Column,
+               desc: Boolean = true,
+               moreAggFunctions: Seq[Column] = Nil,
+               columnsFilter: Seq[String] = Nil,
+               columnsFilterKeep: Boolean = true): DataFrame =
+      SparkDFUtils.dedup2(df,
+                          groupCol,
+                          orderByCol,
+                          desc,
+                          moreAggFunctions,
+                          columnsFilter,
+                          columnsFilterKeep)
 
     def flatten(colName: String): DataFrame = SparkDFUtils.flatten(df, colName)
 
-    def changeSchema(newScheme: String*): DataFrame = SparkDFUtils.changeSchema(df, newScheme: _*)
+    def changeSchema(newScheme: String*): DataFrame =
+      SparkDFUtils.changeSchema(df, newScheme: _*)
 
-    def joinWithRange(colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long = 2^8) =
-      SparkDFUtils.joinWithRange(df, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
+    def joinWithRange(colSingle: String,
+                      dfRange: DataFrame,
+                      colRangeStart: String,
+                      colRangeEnd: String,
+                      DECREASE_FACTOR: Long = 2 ^ 8): DataFrame =
+      SparkDFUtils.joinWithRange(df,
+                                 colSingle,
+                                 dfRange,
+                                 colRangeStart,
+                                 colRangeEnd,
+                                 DECREASE_FACTOR)
 
-    def joinWithRangeAndDedup(colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long = 2^8, dedupSmallRange: Boolean = true): DataFrame = {
-      SparkDFUtils.joinWithRangeAndDedup(df, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR, dedupSmallRange)
-    }
+    def joinWithRangeAndDedup(colSingle: String,
+                              dfRange: DataFrame,
+                              colRangeStart: String,
+                              colRangeEnd: String,
+                              DECREASE_FACTOR: Long = 2 ^ 8,
+                              dedupSmallRange: Boolean = true): DataFrame =
+      SparkDFUtils.joinWithRangeAndDedup(df,
+                                         colSingle,
+                                         dfRange,
+                                         colRangeStart,
+                                         colRangeEnd,
+                                         DECREASE_FACTOR,
+                                         dedupSmallRange)
 
-    def broadcastJoinSkewed(skewed: DataFrame, joinCol: String, numberCustsToBroadcast: Int): DataFrame = {
-      SparkDFUtils.broadcastJoinSkewed(df, skewed, joinCol, numberCustsToBroadcast)
-    }
+    def broadcastJoinSkewed(skewed: DataFrame,
+                            joinCol: String,
+                            numberCustsToBroadcast: Int): DataFrame =
+      SparkDFUtils.broadcastJoinSkewed(df,
+                                       skewed,
+                                       joinCol,
+                                       numberCustsToBroadcast)
 
-    def joinSkewed(notSkewed: DataFrame, joinExprs: Column, numShards: Int = 1000, joinType: String = "inner") : DataFrame = {
+    def joinSkewed(notSkewed: DataFrame,
+                   joinExprs: Column,
+                   numShards: Int = 1000,
+                   joinType: String = "inner"): DataFrame =
       SparkDFUtils.joinSkewed(df, notSkewed, joinExprs, numShards, joinType)
-    }
   }
-}
\ No newline at end of file
+}
diff --git a/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala b/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
index 9c16898..26377b8 100644
--- a/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/PythonPathsManager.scala
@@ -21,105 +21,131 @@ package datafu.spark
 import java.io.{File, IOException}
 import java.net.JarURLConnection
 import java.nio.file.Paths
+import java.util
 import java.util.{MissingResourceException, ServiceLoader}
 
+import scala.collection.JavaConverters._
+
 import org.apache.log4j.Logger
 
-import scala.collection.JavaConverters._
 
 /**
-  * Represents a resource that needs to be added to PYTHONPATH used by ScalaPythonBridge.
-  *
-  * To ensure your python resources (modules, files, etc.) are properly added to the bridge, do the following:
-  * 1)  Put all the resource under some root directory with a unique name x, and make sure path/to/x
-  * is visible to the class loader (usually just use src/main/resources/x).
-  * 2)  Extend this class like this:
-  * class MyResource extends PythonResource("x")
-  * This assumes x is under src/main/resources/x
-  * 3)  (since we use ServiceLoader) Add a file to your jar/project:
-  * META-INF/services/spark.utils.PythonResource
-  * with a single line containing the full name (including package) of MyResource.
-  *
-  * This process involves scanning the entire jar and copying files from the jar to some temporary location,
-  * so if your jar is really big consider putting the resources in a smaller jar.
-  *
-  * @param resourcePath   Path to the resource, will be loaded via getClass.getClassLoader.getResource()
-  * @param isAbsolutePath Set to true if the resource is in some absolute path rather than in jar (try to avoid that).
-  */
-abstract class PythonResource(val resourcePath: String, val isAbsolutePath: Boolean = false)
+ * Represents a resource that needs to be added to PYTHONPATH used by ScalaPythonBridge.
+ *
+ * To ensure your python resources (modules, files, etc.) are properly added to the bridge,
+ * do the following:
+ * 1)  Put all the resource under some root directory with a unique name x, and make sure path/to/x
+ * is visible to the class loader (usually just use src/main/resources/x).
+ * 2)  Extend this class like this:
+ * class MyResource extends PythonResource("x")
+ * This assumes x is under src/main/resources/x
+ * 3)  (since we use ServiceLoader) Add a file to your jar/project:
+ * META-INF/services/spark.utils.PythonResource
+ * with a single line containing the full name (including package) of MyResource.
+ *
+ * This process involves scanning the entire jar and copying files from the jar to some temporary
+ * location, so if your jar is really big consider putting the resources in a smaller jar.
+ *
+ * @param resourcePath   Path to the resource, will be loaded via
+ *                       getClass.getClassLoader.getResource()
+ * @param isAbsolutePath Set to true if the resource is in some absolute path rather than in jar
+ *                       (try to avoid that).
+ */
+abstract class PythonResource(val resourcePath: String,
+                              val isAbsolutePath: Boolean = false)
 
 /**
-  * There are two phases of resolving python files path:
-  *
-  * 1) When launching spark:
-  *   the files need to be added to spark.executorEnv.PYTHONPATH
-  *
-  * 2) When executing python file via bridge:
-  *   the files need to be added to the process PYTHONPATH. This is different than the previous phase because
-  *   this python process is spawned by datafu-spark, not by spark, and always on the driver.
-  */
+ * There are two phases of resolving python files path:
+ *
+ * 1) When launching spark:
+ *   the files need to be added to spark.executorEnv.PYTHONPATH
+ *
+ * 2) When executing python file via bridge:
+ *   the files need to be added to the process PYTHONPATH.
+ *   This is different than the previous phase because
+ *   this python process is spawned by datafu-spark, not by spark, and always on the driver.
+ */
 object PythonPathsManager {
 
-  case class ResolvedResource(resource: PythonResource, resolvedLocation: String)
+  case class ResolvedResource(resource: PythonResource,
+                              resolvedLocation: String)
 
   private val logger: Logger = Logger.getLogger(getClass)
 
   val resources: Seq[ResolvedResource] =
-    ServiceLoader.load(classOf[PythonResource])
+    ServiceLoader
+      .load(classOf[PythonResource])
       .asScala
       .map(p => ResolvedResource(p, resolveDependencyLocation(p)))
       .toSeq
 
   logResolved
 
-  def getAbsolutePaths() = resources.map(_.resolvedLocation).distinct
-  def getAbsolutePathsForJava() = resources.map(_.resolvedLocation).distinct.asJava
+  def getAbsolutePaths(): Seq[String] = resources.map(_.resolvedLocation).distinct
+  def getAbsolutePathsForJava(): util.List[String] =
+    resources.map(_.resolvedLocation).distinct.asJava
 
   def getPYTHONPATH(): String =
     resources
       .map(_.resolvedLocation)
       .map(p => new File(p))
-      .map(_.getName) //get just the name of the file
+      .map(_.getName) // get just the name of the file
       .mkString(":")
 
   private def resolveDependencyLocation(resource: PythonResource): String =
     if (resource.isAbsolutePath) {
-      if (!new File(resource.resourcePath).exists())
-        throw new IOException("Could not find resource in absolute path: " + resource.resourcePath)
-      else {
+      if (!new File(resource.resourcePath).exists()) {
+        throw new IOException(
+          "Could not find resource in absolute path: " + resource.resourcePath)
+      } else {
         logger.info("Using file absolute path: " + resource.resourcePath)
         resource.resourcePath
       }
     } else {
       Option(getClass.getClassLoader.getResource(resource.resourcePath)) match {
         case None =>
-          logger.error("Didn't find resource in classpath! resource path: " + resource.resourcePath)
-          throw new MissingResourceException("Didn't find resource in classpath!", resource.getClass.getName, resource.resourcePath)
+          logger.error(
+            "Didn't find resource in classpath! resource path: " + resource.resourcePath)
+          throw new MissingResourceException(
+            "Didn't find resource in classpath!",
+            resource.getClass.getName,
+            resource.resourcePath)
         case Some(p) =>
           p.toURI.getScheme match {
             case "jar" =>
-              //if dependency is inside jar file, use jar file path:
-              val jarPath = new File(p.openConnection().asInstanceOf[JarURLConnection].getJarFileURL.toURI).getPath
-              logger.info(s"Dependency ${resource.resourcePath} found inside jar: " + jarPath)
+              // if dependency is inside jar file, use jar file path:
+              val jarPath = new File(
+                p.openConnection()
+                  .asInstanceOf[JarURLConnection]
+                  .getJarFileURL
+                  .toURI).getPath
+              logger.info(
+                s"Dependency ${resource.resourcePath} found inside jar: " + jarPath)
               jarPath
             case "file" =>
               val file = new File(p.getFile)
               if (!file.exists()) {
                 logger.warn("Dependency not found, skipping: " + file.getPath)
                 null
-              }
-              else {
-                if (file.isDirectory) {       
+              } else {
+                if (file.isDirectory) {
                   val t_path =
-                    if (System.getProperty("os.name").toLowerCase().contains("win")&& p.getPath().startsWith("/")) {
+                    if (System
+                          .getProperty("os.name")
+                          .toLowerCase()
+                          .contains("win") && p.getPath().startsWith("/")) {
                       val path = p.getPath.substring(1)
-                      logger.warn(s"Fixing path for windows operating system! converted ${p.getPath} to $path")
+                      logger.warn(
+                        s"Fixing path for windows operating system! " +
+                          s"converted ${p.getPath} to $path")
                       path
-                    }
-                    else
+                    } else {
                       p.getPath
+                    }
                   val path = Paths.get(t_path)
-                  logger.info(s"Dependency found as directory: ${t_path}\n\tusing parent path: ${path.getParent}")
+                  logger.info(
+                    s"Dependency found as directory: ${t_path}\n\tusing " +
+                      s"parent path: ${path.getParent}")
                   path.getParent.toString
                 } else {
                   logger.info("Dependency found as a file: " + p.getPath)
@@ -131,15 +157,16 @@ object PythonPathsManager {
     }
 
   private def logResolved = {
-    logger.info(
-      s"Discovered ${resources.size} python paths:\n" +
-        resources
-          .map(p => s"className: ${p.resource.getClass.getName}\n\tresource: ${p.resource.resourcePath}\n\tlocation: ${p.resolvedLocation}")
-          .mkString("\n")) + "\n\n"
+    logger.info(s"Discovered ${resources.size} python paths:\n" +
+      resources
+        .map(p =>
+          s"className: ${p.resource.getClass.getName}\n\tresource: " +
+            s"${p.resource.resourcePath}\n\tlocation: ${p.resolvedLocation}")
+        .mkString("\n")) + "\n\n"
   }
 }
 
 /**
-  * Contains all python files needed by the bridge itself
-  */
+ * Contains all python files needed by the bridge itself
+ */
 class CoreBridgeDirectory extends PythonResource("pyspark_utils")
diff --git a/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala b/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
index 8727f82..4e0fc77 100644
--- a/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/ScalaPythonBridge.scala
@@ -23,22 +23,27 @@ import java.net.URL
 import java.nio.file.Files
 import java.util.UUID
 
+import org.slf4j.LoggerFactory
+
 import org.apache.spark.SparkConf
 import org.apache.spark.api.java.JavaSparkContext
 import org.apache.spark.deploy.SparkPythonRunner
 import org.apache.spark.sql.SparkSession
-import org.slf4j.LoggerFactory
-
 
 case class ScalaPythonBridgeRunner(extraPath: String = "") {
 
   val logger = LoggerFactory.getLogger(this.getClass)
-  //for the bridge we take the full resolved location, since this runs on the driver where the files are local:
+  // for the bridge we take the full resolved location,
+  // since this runs on the driver where the files are local:
   logger.info("constructing PYTHONPATH")
-  
+
   // we include multiple options for py4j because on any given cluster only one should be found
   val pythonPath = (PythonPathsManager.getAbsolutePaths() ++
-    Array("pyspark.zip", "py4j-0.10.4-src.zip", "py4j-0.10.6-src.zip", "py4j-0.10.7-src.zip", "py4j-0.10.8.1-src.zip") ++
+    Array("pyspark.zip",
+          "py4j-0.10.4-src.zip",
+          "py4j-0.10.6-src.zip",
+          "py4j-0.10.7-src.zip",
+          "py4j-0.10.8.1-src.zip") ++
     Option(extraPath).getOrElse("").split(",")).distinct
 
   logger.info("Bridge PYTHONPATH: " + pythonPath.mkString(":"))
@@ -47,28 +52,29 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
 
   def runPythonFile(filename: String): String = {
     val pyScript = resolveRunnableScript(filename)
-    //    this.getClass.getClassLoader.asInstanceOf[URLClassLoader].getURLs.foreach(s => logger.info("class: " + s))
     logger.info(s"Running python file $pyScript")
     runner.runPyFile(pyScript)
   }
 
   def runPythonString(str: String): String = {
     val tmpFile = writeToTempFile(str, "pyspark-tmp-file-", ".py")
-    logger.info("Running tmp PySpark file: " + tmpFile.getAbsolutePath + " with content:\n" + str)
+    logger.info(
+      "Running tmp PySpark file: " + tmpFile.getAbsolutePath + " with content:\n" + str)
     runner.runPyFile(tmpFile.getAbsolutePath)
   }
 
   private def resolveRunnableScript(path: String): String = {
     logger.info("Resolving python script location for: " + path)
 
-    val res: String = Option(this.getClass.getClassLoader.getResource(path)) match {
+    val res
+      : String = Option(this.getClass.getClassLoader.getResource(path)) match {
       case None =>
         logger.info("Didn't find script via classLoader, using as is: " + path)
         path
       case Some(resource) =>
         resource.toURI.getScheme match {
           case "jar" =>
-            //if inside jar, extract it and return cloned file:
+            // if inside jar, extract it and return cloned file:
             logger.info("Script found inside jar, extracting...")
             val outputFile = ResourceCloning.cloneResource(resource, path)
             logger.info("Extracted file path: " + outputFile.getPath)
@@ -81,7 +87,9 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
     res
   }
 
-  private def writeToTempFile(contents: String, prefix: String, suffix: String): File = {
+  private def writeToTempFile(contents: String,
+                              prefix: String,
+                              suffix: String): File = {
     val tempFi = File.createTempFile(prefix, suffix)
     tempFi.deleteOnExit()
     val bw = new BufferedWriter(new FileWriter(tempFi))
@@ -93,12 +101,13 @@ case class ScalaPythonBridgeRunner(extraPath: String = "") {
 }
 
 /**
-  * Do not instantiate this class! Use the companion object instead.
-  * This class should only be used by python
-  */
-object ScalaPythonBridge { //need empty ctor for py4j gateway
+ * Do not instantiate this class! Use the companion object instead.
+ * This class should only be used by python
+ */
+object ScalaPythonBridge { // need empty ctor for py4j gateway
 
-  /** members used to allow python script share context with main Scala program calling it.
+  /**
+    * members used to allow python script share context with main Scala program calling it.
     * Python script calls :
     * sc, sqlContext, spark = utils.get_contexts()
     * our Python util function get_contexts
@@ -106,20 +115,22 @@ object ScalaPythonBridge { //need empty ctor for py4j gateway
     */
   // Called by python util get_contexts()
   def pyGetSparkSession(): SparkSession = SparkSession.builder().getOrCreate()
-  def pyGetJSparkContext(sparkSession: SparkSession): JavaSparkContext = new JavaSparkContext(sparkSession.sparkContext)
+  def pyGetJSparkContext(sparkSession: SparkSession): JavaSparkContext =
+    new JavaSparkContext(sparkSession.sparkContext)
   def pyGetSparkConf(jsc: JavaSparkContext): SparkConf = jsc.getConf
 
 }
 
 /**
-  * Utility for extracting resource from a jar and copy it to a temporary location
-  */
+ * Utility for extracting resource from a jar and copy it to a temporary location
+ */
 object ResourceCloning {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
   val uuid = UUID.randomUUID().toString.substring(6)
-  val outputTempDir = new File(System.getProperty("java.io.tmpdir"), s"risk_tmp/$uuid/cloned_resources/")
+  val outputTempDir = new File(System.getProperty("java.io.tmpdir"),
+                               s"risk_tmp/$uuid/cloned_resources/")
   forceMkdirs(outputTempDir)
 
   def cloneResource(resource: URL, outputFileName: String): File = {
@@ -129,7 +140,8 @@ object ResourceCloning {
       outputTmpFile
     } else {
       logger.info("cloning resource: " + resource)
-      if (!outputTmpFile.exists()) { //it is possible that the file was already extracted in the session
+      if (!outputTmpFile.exists()) {
+        // it is possible that the file was already extracted in the session
         forceMkdirs(outputTmpFile.getParentFile)
         val inputStream = resource.openStream()
         streamToFile(outputTmpFile, inputStream)
@@ -139,14 +151,14 @@ object ResourceCloning {
   }
 
   private def forceMkdirs(dir: File) =
-    if (!dir.exists() && !dir.mkdirs())
+    if (!dir.exists() && !dir.mkdirs()) {
       throw new IOException("Failed to create " + dir.getPath)
+    }
 
   private def streamToFile(outputFile: File, inputStream: InputStream) = {
     try {
       Files.copy(inputStream, outputFile.toPath)
-    }
-    finally {
+    } finally {
       inputStream.close()
       assert(outputFile.exists())
     }
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
index 61386bd..2b48c52 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkDFUtils.scala
@@ -20,30 +20,51 @@ package datafu.spark
 
 import java.util.{List => JavaList}
 
+import org.apache.spark.sql.{Column, DataFrame}
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{LongType, SparkOverwriteUDAFs, StructType}
-import org.apache.spark.sql.{Column, DataFrame}
 import org.apache.spark.storage.StorageLevel
 
 /**
-  * class definition so we could expose this functionality in PySpark
-  */
+ * class definition so we could expose this functionality in PySpark
+ */
 class SparkDFUtilsBridge {
 
-  def dedup(df: DataFrame, groupCol: Column, orderCols: JavaList[Column]): DataFrame = {
+  def dedup(df: DataFrame,
+            groupCol: Column,
+            orderCols: JavaList[Column]): DataFrame = {
     val converted = convertJavaListToSeq(orderCols)
     SparkDFUtils.dedup(df = df, groupCol = groupCol, orderCols = converted: _*)
   }
 
-  def dedupTopN(df: DataFrame, n: Int, groupCol: Column, orderCols: JavaList[Column]): DataFrame = {
+  def dedupTopN(df: DataFrame,
+                n: Int,
+                groupCol: Column,
+                orderCols: JavaList[Column]): DataFrame = {
     val converted = convertJavaListToSeq(orderCols)
-    SparkDFUtils.dedupTopN(df = df, n = n, groupCol = groupCol, orderCols = converted: _*)
+    SparkDFUtils.dedupTopN(df = df,
+                           n = n,
+                           groupCol = groupCol,
+                           orderCols = converted: _*)
   }
 
-  def dedup2(df: DataFrame, groupCol: Column, orderByCol: Column, desc: Boolean, columnsFilter: JavaList[String], columnsFilterKeep: Boolean): DataFrame = {
+  def dedup2(df: DataFrame,
+             groupCol: Column,
+             orderByCol: Column,
+             desc: Boolean,
+             columnsFilter: JavaList[String],
+             columnsFilterKeep: Boolean): DataFrame = {
     val columnsFilter_converted = convertJavaListToSeq(columnsFilter)
-    SparkDFUtils.dedup2(df = df, groupCol = groupCol, orderByCol = orderByCol, desc = desc, moreAggFunctions = Nil, columnsFilter = columnsFilter_converted, columnsFilterKeep = columnsFilterKeep)
+    SparkDFUtils.dedup2(
+      df = df,
+      groupCol = groupCol,
+      orderByCol = orderByCol,
+      desc = desc,
+      moreAggFunctions = Nil,
+      columnsFilter = columnsFilter_converted,
+      columnsFilterKeep = columnsFilterKeep
+    )
   }
 
   def changeSchema(df: DataFrame, newScheme: JavaList[String]): DataFrame = {
@@ -51,31 +72,73 @@ class SparkDFUtilsBridge {
     SparkDFUtils.changeSchema(df = df, newScheme = newScheme_converted: _*)
   }
 
-  def joinSkewed(dfLeft: DataFrame, dfRight: DataFrame, joinExprs: Column, numShards: Int, joinType: String): DataFrame = {
-    SparkDFUtils.joinSkewed(dfLeft = dfLeft, dfRight = dfRight, joinExprs = joinExprs, numShards = numShards, joinType = joinType)
+  def joinSkewed(dfLeft: DataFrame,
+                 dfRight: DataFrame,
+                 joinExprs: Column,
+                 numShards: Int,
+                 joinType: String): DataFrame = {
+    SparkDFUtils.joinSkewed(dfLeft = dfLeft,
+                            dfRight = dfRight,
+                            joinExprs = joinExprs,
+                            numShards = numShards,
+                            joinType = joinType)
   }
 
-  def broadcastJoinSkewed(notSkewed: DataFrame, skewed: DataFrame, joinCol: String, numRowsToBroadcast: Int): DataFrame = {
-    SparkDFUtils.broadcastJoinSkewed(notSkewed = notSkewed, skewed = skewed, joinCol = joinCol, numRowsToBroadcast = numRowsToBroadcast)
+  def broadcastJoinSkewed(notSkewed: DataFrame,
+                          skewed: DataFrame,
+                          joinCol: String,
+                          numRowsToBroadcast: Int): DataFrame = {
+    SparkDFUtils.broadcastJoinSkewed(notSkewed = notSkewed,
+                                     skewed = skewed,
+                                     joinCol = joinCol,
+                                     numRowsToBroadcast = numRowsToBroadcast)
   }
 
-  def joinWithRange(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
-    SparkDFUtils.joinWithRange(dfSingle = dfSingle, colSingle = colSingle, dfRange = dfRange, colRangeStart = colRangeStart, colRangeEnd = colRangeEnd, DECREASE_FACTOR = DECREASE_FACTOR)
+  def joinWithRange(dfSingle: DataFrame,
+                    colSingle: String,
+                    dfRange: DataFrame,
+                    colRangeStart: String,
+                    colRangeEnd: String,
+                    DECREASE_FACTOR: Long): DataFrame = {
+    SparkDFUtils.joinWithRange(dfSingle = dfSingle,
+                               colSingle = colSingle,
+                               dfRange = dfRange,
+                               colRangeStart = colRangeStart,
+                               colRangeEnd = colRangeEnd,
+                               DECREASE_FACTOR = DECREASE_FACTOR)
   }
 
-  def joinWithRangeAndDedup(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long, dedupSmallRange: Boolean): DataFrame = {
-    SparkDFUtils.joinWithRangeAndDedup(dfSingle = dfSingle, colSingle = colSingle, dfRange = dfRange, colRangeStart = colRangeStart, colRangeEnd = colRangeEnd, DECREASE_FACTOR = DECREASE_FACTOR, dedupSmallRange = dedupSmallRange)
+  def joinWithRangeAndDedup(dfSingle: DataFrame,
+                            colSingle: String,
+                            dfRange: DataFrame,
+                            colRangeStart: String,
+                            colRangeEnd: String,
+                            DECREASE_FACTOR: Long,
+                            dedupSmallRange: Boolean): DataFrame = {
+    SparkDFUtils.joinWithRangeAndDedup(
+      dfSingle = dfSingle,
+      colSingle = colSingle,
+      dfRange = dfRange,
+      colRangeStart = colRangeStart,
+      colRangeEnd = colRangeEnd,
+      DECREASE_FACTOR = DECREASE_FACTOR,
+      dedupSmallRange = dedupSmallRange
+    )
   }
 
   private def convertJavaListToSeq[T](list: JavaList[T]): Seq[T] = {
-    scala.collection.JavaConverters.asScalaIteratorConverter(list.iterator()).asScala.toList
+    scala.collection.JavaConverters
+      .asScalaIteratorConverter(list.iterator())
+      .asScala
+      .toList
   }
 }
 
 object SparkDFUtils {
 
   /**
-    * Used to get the 'latest' record (after ordering according to the provided order columns) in each group.
+    * Used to get the 'latest' record (after ordering according to the provided order columns)
+    * in each group.
     * Different from {@link org.apache.spark.sql.Dataset#dropDuplicates} because order matters.
     *
     * @param df DataFrame to operate on
@@ -89,7 +152,8 @@ object SparkDFUtils {
   }
 
   /**
-    * Used get the top N records (after ordering according to the provided order columns) in each group.
+    * Used get the top N records (after ordering according to the provided order columns)
+    * in each group.
     *
     * @param df DataFrame to operate on
     * @param n number of records to return from each group
@@ -97,14 +161,19 @@ object SparkDFUtils {
     * @param orderCols columns to order the records according to
     * @return DataFrame representing the data after the operation
     */
-  def dedupTopN(df: DataFrame, n: Int, groupCol: Column, orderCols: Column*): DataFrame = {
+  def dedupTopN(df: DataFrame,
+                n: Int,
+                groupCol: Column,
+                orderCols: Column*): DataFrame = {
     val w = Window.partitionBy(groupCol).orderBy(orderCols: _*)
     df.withColumn("rn", row_number.over(w)).where(col("rn") <= n).drop("rn")
   }
 
   /**
-    * Used to get the 'latest' record (after ordering according to the provided order columns) in each group.
-    * the same functionality as {@link #dedup} but implemented using UDAF to utilize map side aggregation.
+    * Used to get the 'latest' record (after ordering according to the provided order columns)
+    * in each group.
+    * the same functionality as {@link #dedup} but implemented using UDAF to utilize
+    * map side aggregation.
     *
     * @param df DataFrame to operate on
     * @param groupCol column to group by the records
@@ -112,72 +181,99 @@ object SparkDFUtils {
     * @param desc have the order as desc
     * @param moreAggFunctions more aggregate functions
     * @param columnsFilter columns to filter
-    * @param columnsFilterKeep indicates whether we should filter the selected columns 'out' or alternatively have only
-    *                          those columns in the result
+    * @param columnsFilterKeep indicates whether we should filter the selected columns 'out'
+    *                          or alternatively have only those columns in the result
     * @return DataFrame representing the data after the operation
     */
-  def dedup2(df: DataFrame, groupCol: Column, orderByCol: Column, desc: Boolean = true, moreAggFunctions: Seq[Column] = Nil, columnsFilter: Seq[String] = Nil, columnsFilterKeep: Boolean = true): DataFrame = {
-    val newDF = if (columnsFilter == Nil)
-      df.withColumn("sort_by_column", orderByCol)
-    else {
-      if(columnsFilterKeep)
-        df.withColumn("sort_by_column", orderByCol).select("sort_by_column", columnsFilter: _*)
-      else
-        df.select(df.columns.filter(colName => !columnsFilter.contains(colName)).map(colName => new Column(colName)):_*).withColumn("sort_by_column", orderByCol)
-    }
-
-    val aggFunc = if (desc) SparkOverwriteUDAFs.maxValueByKey(_:Column, _:Column)
-                  else      SparkOverwriteUDAFs.minValueByKey(_:Column, _:Column)
+  def dedup2(df: DataFrame,
+             groupCol: Column,
+             orderByCol: Column,
+             desc: Boolean = true,
+             moreAggFunctions: Seq[Column] = Nil,
+             columnsFilter: Seq[String] = Nil,
+             columnsFilterKeep: Boolean = true): DataFrame = {
+    val newDF =
+      if (columnsFilter == Nil) {
+        df.withColumn("sort_by_column", orderByCol)
+      } else {
+        if (columnsFilterKeep) {
+          df.withColumn("sort_by_column", orderByCol)
+            .select("sort_by_column", columnsFilter: _*)
+        } else {
+          df.select(
+            df.columns
+              .filter(colName => !columnsFilter.contains(colName))
+              .map(colName => new Column(colName)): _*)
+            .withColumn("sort_by_column", orderByCol)
+        }
+      }
+
+    val aggFunc =
+      if (desc) SparkOverwriteUDAFs.maxValueByKey(_: Column, _: Column)
+      else SparkOverwriteUDAFs.minValueByKey(_: Column, _: Column)
 
     val df2 = newDF
       .groupBy(groupCol.as("group_by_col"))
-      .agg(aggFunc(expr("sort_by_column"), expr("struct(sort_by_column, *)")).as("h1"),
-           struct(lit(1).as("lit_placeholder_col") +: moreAggFunctions: _*).as("h2"))
-      .selectExpr("h2.*","h1.*")
+      .agg(aggFunc(expr("sort_by_column"), expr("struct(sort_by_column, *)"))
+             .as("h1"),
+           struct(lit(1).as("lit_placeholder_col") +: moreAggFunctions: _*)
+             .as("h2"))
+      .selectExpr("h2.*", "h1.*")
       .drop("lit_placeholder_col")
       .drop("sort_by_column")
     df2
   }
 
- /**
-  * Returns a DataFrame with the given column (should be a StructType) replaced by its inner fields.
-  * This method only flattens a single level of nesting.
-  *
-  * +-------+----------+----------+----------+
-  * |id     |s.sub_col1|s.sub_col2|s.sub_col3|
-  * +-------+----------+----------+----------+
-  * |123    |1         |2         |3         |
-  * +-------+----------+----------+----------+
-  *
-  * +-------+----------+----------+----------+
-  * |id     |sub_col1  |sub_col2  |sub_col3  |
-  * +-------+----------+----------+----------+
-  * |123    |1         |2         |3         |
-  * +-------+----------+----------+----------+
-  *
-  * @param df DataFrame to operate on
-  * @param colName column name for a column of type StructType
-  * @return DataFrame representing the data after the operation
-  */
+  /**
+    * Returns a DataFrame with the given column (should be a StructType)
+    * replaced by its inner fields.
+    * This method only flattens a single level of nesting.
+    *
+    * +-------+----------+----------+----------+
+    * |id     |s.sub_col1|s.sub_col2|s.sub_col3|
+    * +-------+----------+----------+----------+
+    * |123    |1         |2         |3         |
+    * +-------+----------+----------+----------+
+    *
+    * +-------+----------+----------+----------+
+    * |id     |sub_col1  |sub_col2  |sub_col3  |
+    * +-------+----------+----------+----------+
+    * |123    |1         |2         |3         |
+    * +-------+----------+----------+----------+
+    *
+    * @param df DataFrame to operate on
+    * @param colName column name for a column of type StructType
+    * @return DataFrame representing the data after the operation
+    */
   def flatten(df: DataFrame, colName: String): DataFrame = {
-    assert(df.schema(colName).dataType.isInstanceOf[StructType], s"Column $colName must be of type Struct")
+    assert(df.schema(colName).dataType.isInstanceOf[StructType],
+           s"Column $colName must be of type Struct")
     val outerFields = df.schema.fields.map(_.name).toSet
-    val flattenFields = df.schema(colName).dataType.asInstanceOf[StructType].fields.filter(f => !outerFields.contains(f.name)).map("`" + colName + "`.`" + _.name + "`")
+    val flattenFields = df
+      .schema(colName)
+      .dataType
+      .asInstanceOf[StructType]
+      .fields
+      .filter(f => !outerFields.contains(f.name))
+      .map("`" + colName + "`.`" + _.name + "`")
     df.selectExpr("*" +: flattenFields: _*).drop(colName)
   }
 
   /**
-  * Returns a DataFrame with the column names renamed to the column names in the new schema
-  *
-  * @param df DataFrame to operate on
-  * @param newScheme new column names
-  * @return DataFrame representing the data after the operation
-  */
+    * Returns a DataFrame with the column names renamed to the column names in the new schema
+    *
+    * @param df DataFrame to operate on
+    * @param newScheme new column names
+    * @return DataFrame representing the data after the operation
+    */
   def changeSchema(df: DataFrame, newScheme: String*): DataFrame =
-    df.select(df.columns.zip(newScheme).map {case (oldCol: String, newCol: String) => col(oldCol).as(newCol)}: _*)
+    df.select(df.columns.zip(newScheme).map {
+      case (oldCol: String, newCol: String) => col(oldCol).as(newCol)
+    }: _*)
 
   /**
-    * Used to perform a join when the right df is relatively small but still too big to fit in memory to perform map side broadcast join.
+    * Used to perform a join when the right df is relatively small
+    * but still too big to fit in memory to perform map side broadcast join.
     * Use cases:
     * a. excluding keys that might be skewed from a medium size list.
     * b. join a big skewed table with a table that has small number of very large rows.
@@ -189,19 +285,31 @@ object SparkDFUtils {
     * @param joinType join type
     * @return joined DataFrame
     */
-  def joinSkewed(dfLeft: DataFrame, dfRight: DataFrame, joinExprs: Column, numShards: Int = 10, joinType: String = "inner"): DataFrame = {
+  def joinSkewed(dfLeft: DataFrame,
+                 dfRight: DataFrame,
+                 joinExprs: Column,
+                 numShards: Int = 10,
+                 joinType: String = "inner"): DataFrame = {
     // skew join based on salting
-    // salts the left DF by adding another random column and join with the right DF after duplicating it
+    // salts the left DF by adding another random column and join with the right DF after
+    // duplicating it
     val ss = dfLeft.sparkSession
     import ss.implicits._
     val shards = 1.to(numShards).toDF("shard")
-    dfLeft.withColumn("randLeft", ceil(rand() * numShards)).join(dfRight.crossJoin(shards), joinExprs and $"randLeft" === $"shard", joinType).drop($"randLeft").drop($"shard")
+    dfLeft
+      .withColumn("randLeft", ceil(rand() * numShards))
+      .join(dfRight.crossJoin(shards),
+            joinExprs and $"randLeft" === $"shard",
+            joinType)
+      .drop($"randLeft")
+      .drop($"shard")
   }
 
   /**
     * Suitable to perform a join in cases when one DF is skewed and the other is not skewed.
     * splits both of the DFs to two parts according to the skewed keys.
-    * 1. Map-join: broadcasts the skewed-keys part of the not skewed DF to the skewed-keys part of the skewed DF
+    * 1. Map-join: broadcasts the skewed-keys part of the not skewed DF to the skewed-keys
+    *    part of the skewed DF
     * 2. Regular join: between the remaining two parts.
     *
     * @param notSkewed not skewed DataFrame
@@ -210,35 +318,55 @@ object SparkDFUtils {
     * @param numRowsToBroadcast num of rows to broadcast
     * @return DataFrame representing the data after the operation
     */
-  def broadcastJoinSkewed(notSkewed: DataFrame, skewed: DataFrame, joinCol: String, numRowsToBroadcast: Int): DataFrame = {
+  def broadcastJoinSkewed(notSkewed: DataFrame,
+                          skewed: DataFrame,
+                          joinCol: String,
+                          numRowsToBroadcast: Int): DataFrame = {
     val ss = notSkewed.sparkSession
     import ss.implicits._
-    val skewedKeys = skewed.groupBy(joinCol).count().sort($"count".desc).limit(numRowsToBroadcast).drop("count")
-                           .withColumnRenamed(joinCol, "skew_join_key").cache()
-
-    val notSkewedWithSkewIndicator = notSkewed.join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
-      .withColumn("is_skewed_record", col("skew_join_key").isNotNull).drop("skew_join_key").persist(StorageLevel.DISK_ONLY)
+    val skewedKeys = skewed
+      .groupBy(joinCol)
+      .count()
+      .sort($"count".desc)
+      .limit(numRowsToBroadcast)
+      .drop("count")
+      .withColumnRenamed(joinCol, "skew_join_key")
+      .cache()
+
+    val notSkewedWithSkewIndicator = notSkewed
+      .join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
+      .withColumn("is_skewed_record", col("skew_join_key").isNotNull)
+      .drop("skew_join_key")
+      .persist(StorageLevel.DISK_ONLY)
 
     // broadcast map-join, sending the notSkewed data
-    val bigRecordsJnd = broadcast(notSkewedWithSkewIndicator.filter("is_skewed_record"))
-      .join(skewed, joinCol)
+    val bigRecordsJnd =
+      broadcast(notSkewedWithSkewIndicator.filter("is_skewed_record"))
+        .join(skewed, joinCol)
 
     // regular join for the rest
-    val skewedWithoutSkewedKeys = skewed.join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
-      .where("skew_join_key is null").drop("skew_join_key")
-    val smallRecordsJnd = notSkewedWithSkewIndicator.filter("not is_skewed_record")
+    val skewedWithoutSkewedKeys = skewed
+      .join(broadcast(skewedKeys), $"skew_join_key" === col(joinCol), "left")
+      .where("skew_join_key is null")
+      .drop("skew_join_key")
+    val smallRecordsJnd = notSkewedWithSkewIndicator
+      .filter("not is_skewed_record")
       .join(skewedWithoutSkewedKeys, joinCol)
 
-    smallRecordsJnd.union(bigRecordsJnd).drop("is_skewed_record", "skew_join_key")
+    smallRecordsJnd
+      .union(bigRecordsJnd)
+      .drop("is_skewed_record", "skew_join_key")
   }
 
   /**
     * Helper function to join a table with point column to a table with range column.
-    * For example, join a table that contains specific time in minutes with a table that contains time ranges.
-    * The main problem this function addresses is that doing naive explode on the ranges can result in a huge table.
+    * For example, join a table that contains specific time in minutes with a table that
+    * contains time ranges.
+    * The main problem this function addresses is that doing naive explode on the ranges can result
+    * in a huge table.
     * requires:
-    * 1. point table needs to be distinct on the point column. there could be a few corresponding ranges to each point,
-    *    so we choose the minimal range.
+    * 1. point table needs to be distinct on the point column. there could be a few corresponding
+    *    ranges to each point, so we choose the minimal range.
     * 2. the range and point columns need to be numeric.
     *
     * TIMES:
@@ -266,33 +394,61 @@ object SparkDFUtils {
     * |11:55  |11:50     |12:15    | lunch   |
     * +-------+----------+---------+---------+
     *
-    * @param dfSingle - DataFrame that contains the point column
-    * @param colSingle - the point column's name
-    * @param dfRange - DataFrame that contains the range column
-    * @param colRangeStart - the start range column's name
-    * @param colRangeEnd - the end range column's name
-    * @param DECREASE_FACTOR - resolution factor. instead of exploding the range column directly, we first decrease its resolution by this factor
+    * @param dfSingle DataFrame that contains the point column
+    * @param colSingle the point column's name
+    * @param dfRange DataFrame that contains the range column
+    * @param colRangeStart the start range column's name
+    * @param colRangeEnd the end range column's name
+    * @param DECREASE_FACTOR resolution factor. instead of exploding the range column directly,
+    *                        we first decrease its resolution by this factor
     * @return
     */
-  def joinWithRange(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
-    val dfJoined = joinWithRangeInternal(dfSingle, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
-    dfJoined.drop("range_start", "range_end", "decreased_range_single", "single", "decreased_single", "range_size")
+  def joinWithRange(dfSingle: DataFrame,
+                    colSingle: String,
+                    dfRange: DataFrame,
+                    colRangeStart: String,
+                    colRangeEnd: String,
+                    DECREASE_FACTOR: Long): DataFrame = {
+    val dfJoined = joinWithRangeInternal(dfSingle,
+                                         colSingle,
+                                         dfRange,
+                                         colRangeStart,
+                                         colRangeEnd,
+                                         DECREASE_FACTOR)
+    dfJoined.drop("range_start",
+                  "range_end",
+                  "decreased_range_single",
+                  "single",
+                  "decreased_single",
+                  "range_size")
   }
 
-  private def joinWithRangeInternal(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long): DataFrame = {
+  private def joinWithRangeInternal(dfSingle: DataFrame,
+                                    colSingle: String,
+                                    dfRange: DataFrame,
+                                    colRangeStart: String,
+                                    colRangeEnd: String,
+                                    DECREASE_FACTOR: Long): DataFrame = {
 
     import org.apache.spark.sql.functions.udf
     val rangeUDF = udf((start: Long, end: Long) => (start to end).toArray)
-    val dfRange_exploded = dfRange.withColumn("range_start", col(colRangeStart).cast(LongType))
-                                  .withColumn("range_end"  , col(colRangeEnd).cast(LongType))
-                                  .withColumn("decreased_range_single", explode(rangeUDF(col("range_start")/lit(DECREASE_FACTOR),
-                                                                                                   col("range_end")  /lit(DECREASE_FACTOR))))
-
-    dfSingle.withColumn("single", floor(col(colSingle).cast(LongType)))
-            .withColumn("decreased_single", floor(col(colSingle).cast(LongType)/lit(DECREASE_FACTOR)))
-            .join(dfRange_exploded, col("decreased_single") === col("decreased_range_single"), "left_outer")
-            .withColumn("range_size", expr("(range_end - range_start + 1)"))
-            .filter("single>=range_start and single<=range_end")
+    val dfRange_exploded = dfRange
+      .withColumn("range_start", col(colRangeStart).cast(LongType))
+      .withColumn("range_end", col(colRangeEnd).cast(LongType))
+      .withColumn("decreased_range_single",
+                  explode(
+                    rangeUDF(col("range_start") / lit(DECREASE_FACTOR),
+                             col("range_end") / lit(DECREASE_FACTOR))))
+
+    dfSingle
+      .withColumn("single", floor(col(colSingle).cast(LongType)))
+      .withColumn("decreased_single",
+                  floor(col(colSingle).cast(LongType) / lit(DECREASE_FACTOR)))
+      .join(dfRange_exploded,
+            col("decreased_single") === col("decreased_range_single"),
+            "left_outer")
+      .withColumn("range_size", expr("(range_end - range_start + 1)"))
+      .filter("single>=range_start and single<=range_end")
   }
 
   /**
@@ -315,17 +471,39 @@ object SparkDFUtils {
     * +-------+----------+---------+---------+
     *
     */
-  def joinWithRangeAndDedup(dfSingle: DataFrame, colSingle: String, dfRange: DataFrame, colRangeStart: String, colRangeEnd: String, DECREASE_FACTOR: Long, dedupSmallRange: Boolean): DataFrame = {
-
-    val dfJoined = joinWithRangeInternal(dfSingle, colSingle, dfRange, colRangeStart, colRangeEnd, DECREASE_FACTOR)
+  def joinWithRangeAndDedup(dfSingle: DataFrame,
+                            colSingle: String,
+                            dfRange: DataFrame,
+                            colRangeStart: String,
+                            colRangeEnd: String,
+                            DECREASE_FACTOR: Long,
+                            dedupSmallRange: Boolean): DataFrame = {
+
+    val dfJoined = joinWithRangeInternal(dfSingle,
+                                         colSingle,
+                                         dfRange,
+                                         colRangeStart,
+                                         colRangeEnd,
+                                         DECREASE_FACTOR)
 
     // "range_start" is here for consistency
     val dfDeduped = if (dedupSmallRange) {
-      dedup2(dfJoined, col(colSingle), struct("range_size", "range_start"), desc = false)
+      dedup2(dfJoined,
+             col(colSingle),
+             struct("range_size", "range_start"),
+             desc = false)
     } else {
-      dedup2(dfJoined, col(colSingle), struct(expr("-range_size"), col("range_start")), desc = true)
+      dedup2(dfJoined,
+             col(colSingle),
+             struct(expr("-range_size"), col("range_start")),
+             desc = true)
     }
 
-    dfDeduped.drop("range_start", "range_end", "decreased_range_single", "single", "decreased_single", "range_size")
+    dfDeduped.drop("range_start",
+                   "range_end",
+                   "decreased_range_single",
+                   "single",
+                   "decreased_single",
+                   "range_size")
   }
 }
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala b/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
index fe95ffb..3e0810b 100644
--- a/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
+++ b/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
@@ -18,11 +18,12 @@
  */
 package datafu.spark
 
+import scala.collection.{mutable, Map}
+
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
 import org.apache.spark.sql.types.{ArrayType, _}
 
-import scala.collection.{Map, mutable}
 
 // @TODO: add documentation and tests to all the functions. maybe also expose in python.
 
@@ -36,36 +37,44 @@ object SparkUDAFs {
 
     def inputSchema: StructType = new StructType().add("key", StringType)
 
-    def bufferSchema = new StructType().add("mp", MapType(StringType, IntegerType))
+    def bufferSchema: StructType =
+      new StructType().add("mp", MapType(StringType, IntegerType))
 
     def dataType: DataType = MapType(StringType, IntegerType, false)
 
-    def deterministic = true
+    def deterministic: Boolean = true
 
     // This function is called whenever key changes
-    def initialize(buffer: MutableAggregationBuffer) = {
+    def initialize(buffer: MutableAggregationBuffer): Unit = {
       buffer(0) = mutable.Map()
     }
 
     // Iterate over each entry of a group
-    def update(buffer: MutableAggregationBuffer, input: Row) = {
+    def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
       val key = input.getString(0)
-      if (key != null)
-        buffer(0) = buffer.getMap(0) + (key -> (buffer.getMap(0).getOrElse(key, 0) + 1))
+      if (key != null) {
+        buffer(0) = buffer.getMap(0) + (key -> (buffer
+          .getMap(0)
+          .getOrElse(key, 0) + 1))
+      }
     }
 
     // Merge two partial aggregates
-    def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
+    def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
       val mp = mutable.Map[String, Int]() ++= buffer1.getMap(0)
-      buffer2.getMap(0).keys.foreach((key: String) =>
-        if (key != null)
-          mp.put(key, mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
-      )
+      buffer2
+        .getMap(0)
+        .keys
+        .foreach((key: String) =>
+          if (key != null) {
+            mp.put(key,
+              mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
+          })
       buffer1(0) = mp
     }
 
     // Called after all the entries are exhausted.
-    def evaluate(buffer: Row) = {
+    def evaluate(buffer: Row): Any = {
       buffer(0)
     }
 
@@ -75,23 +84,24 @@ object SparkUDAFs {
     * Essentially the same as MultiSet, but gets an Array for input.
     * There is an extra option to limit the number of keys (like CountDistinctUpTo)
     */
-  class MultiArraySet[T : Ordering](dt : DataType = StringType, maxKeys: Int = -1) extends UserDefinedAggregateFunction {
+  class MultiArraySet[T: Ordering](dt: DataType = StringType, maxKeys: Int = -1)
+      extends UserDefinedAggregateFunction {
 
     def inputSchema: StructType = new StructType().add("key", ArrayType(dt))
 
-    def bufferSchema = new StructType().add("mp", dataType)
+    def bufferSchema: StructType = new StructType().add("mp", dataType)
 
     def dataType: DataType = MapType(dt, IntegerType, false)
 
-    def deterministic = true
+    def deterministic: Boolean = true
 
     // This function is called whenever key changes
-    def initialize(buffer: MutableAggregationBuffer) = {
+    def initialize(buffer: MutableAggregationBuffer): Unit = {
       buffer(0) = mutable.Map()
     }
 
     // Iterate over each entry of a group
-    def update(buffer: MutableAggregationBuffer, input: Row) = {
+    def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
       val mp = mutable.Map[T, Int]() ++= buffer.getMap(0)
       val keyArr: Seq[T] = Option(input.getAs[Seq[T]](0)).getOrElse(Nil)
       for (key <- keyArr; if key != null)
@@ -101,12 +111,16 @@ object SparkUDAFs {
     }
 
     // Merge two partial aggregates
-    def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
+    def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
       val mp = mutable.Map[T, Int]() ++= buffer1.getMap(0)
-      buffer2.getMap(0).keys.foreach((key: T) =>
-        if (key != null)
-          mp.put(key, mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
-      )
+      buffer2
+        .getMap(0)
+        .keys
+        .foreach((key: T) =>
+          if (key != null) {
+            mp.put(key,
+              mp.getOrElse(key, 0) + buffer2.getMap(0).getOrElse(key, 0))
+          })
 
       buffer1(0) = limitKeys(mp, 3)
     }
@@ -114,24 +128,29 @@ object SparkUDAFs {
     private def limitKeys(mp: Map[T, Int], factor: Int = 1): Map[T, Int] = {
       if (maxKeys > 0 && maxKeys * factor < mp.size) {
         val k = mp.toList.map(_.swap).sorted.reverse(maxKeys - 1)._1
-        var mp2 = mutable.Map[T, Int]() ++= mp.filter((t: (T, Int)) => t._2 >= k)
+        var mp2 = mutable.Map[T, Int]() ++= mp.filter((t: (T, Int)) =>
+          t._2 >= k)
         var toRemove = mp2.size - maxKeys
-        if (toRemove > 0)
+        if (toRemove > 0) {
           mp2 = mp2.filter((t: (T, Int)) => {
-            if (t._2 > k)
+            if (t._2 > k) {
               true
-            else {
-              if (toRemove >= 0)
+            } else {
+              if (toRemove >= 0) {
                 toRemove = toRemove - 1
+              }
               toRemove < 0
             }
           })
+        }
         mp2
-      } else mp
+      } else {
+        mp
+      }
     }
 
     // Called after all the entries are exhausted.
-    def evaluate(buffer: Row) = {
+    def evaluate(buffer: Row): Map[T, Int] = {
       limitKeys(buffer.getMap(0).asInstanceOf[Map[T, Int]])
     }
 
@@ -144,97 +163,116 @@ object SparkUDAFs {
 
     def inputSchema: StructType = new StructType().add("key", dataType)
 
-    def bufferSchema = inputSchema
+    def bufferSchema: StructType = inputSchema
 
     def dataType: DataType = MapType(StringType, ArrayType(StringType))
 
-    def deterministic = true
+    def deterministic: Boolean = true
 
     // This function is called whenever key changes
-    def initialize(buffer: MutableAggregationBuffer) = {
+    def initialize(buffer: MutableAggregationBuffer): Unit = {
       buffer(0) = mutable.Map()
     }
 
     // Iterate over each entry of a group
-    def update(buffer: MutableAggregationBuffer, input: Row) = {
+    def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
       val mp0 = input.getMap(0)
       if (mp0 != null) {
-        val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= input.getMap(0)
-        buffer(0) = merge(mp, buffer.getMap[String, mutable.WrappedArray[String]](0))
+        val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= input
+          .getMap(0)
+        buffer(0) =
+          merge(mp, buffer.getMap[String, mutable.WrappedArray[String]](0))
       }
     }
 
     // Merge two partial aggregates
-    def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
-      val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= buffer1.getMap(0)
-      buffer1(0) = merge(mp, buffer2.getMap[String, mutable.WrappedArray[String]](0))
+    def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+      val mp = mutable.Map[String, mutable.WrappedArray[String]]() ++= buffer1
+        .getMap(0)
+      buffer1(0) =
+        merge(mp, buffer2.getMap[String, mutable.WrappedArray[String]](0))
     }
 
-    def merge(mpBuffer: mutable.Map[String, mutable.WrappedArray[String]], mp: Map[String, mutable.WrappedArray[String]]): mutable.Map[String, mutable.WrappedArray[String]] = {
-      if (mp != null)
+    def merge(mpBuffer: mutable.Map[String, mutable.WrappedArray[String]],
+              mp: Map[String, mutable.WrappedArray[String]])
+      : mutable.Map[String, mutable.WrappedArray[String]] = {
+      if (mp != null) {
         mp.keys.foreach((key: String) => {
-          val blah1: mutable.WrappedArray[String] = mpBuffer.getOrElse(key, mutable.WrappedArray.empty)
-          val blah2: mutable.WrappedArray[String] = mp.getOrElse(key, mutable.WrappedArray.empty)
-          mpBuffer.put(key, mutable.WrappedArray.make((Option(blah1).getOrElse(mutable.WrappedArray.empty) ++ Option(blah2).getOrElse(mutable.WrappedArray.empty)).toSet.toArray) )
+          val blah1: mutable.WrappedArray[String] =
+            mpBuffer.getOrElse(key, mutable.WrappedArray.empty)
+          val blah2: mutable.WrappedArray[String] =
+            mp.getOrElse(key, mutable.WrappedArray.empty)
+          mpBuffer.put(
+            key,
+            mutable.WrappedArray.make(
+              (Option(blah1).getOrElse(mutable.WrappedArray.empty) ++ Option(
+                blah2).getOrElse(mutable.WrappedArray.empty)).toSet.toArray)
+          )
         })
+      }
 
       mpBuffer
     }
 
     // Called after all the entries are exhausted.
-    def evaluate(buffer: Row) = {
+    def evaluate(buffer: Row): Any = {
       buffer(0)
     }
 
   }
 
   /**
-   * Counts number of distinct records, but only up to a preset amount - more efficient than an unbounded count
-   */
-  class CountDistinctUpTo(maxItems: Int = -1) extends UserDefinedAggregateFunction {
+    * Counts number of distinct records, but only up to a preset amount -
+    * more efficient than an unbounded count
+    */
+  class CountDistinctUpTo(maxItems: Int = -1)
+      extends UserDefinedAggregateFunction {
 
     def inputSchema: StructType = new StructType().add("key", StringType)
 
-    def bufferSchema = new StructType().add("mp", MapType(StringType, BooleanType))
+    def bufferSchema: StructType =
+      new StructType().add("mp", MapType(StringType, BooleanType))
 
     def dataType: DataType = IntegerType
 
-    def deterministic = true
+    def deterministic: Boolean = true
 
     // This function is called whenever key changes
-    def initialize(buffer: MutableAggregationBuffer) = {
+    def initialize(buffer: MutableAggregationBuffer): Unit = {
       buffer(0) = mutable.Map()
     }
 
     // Iterate over each entry of a group
-    def update(buffer: MutableAggregationBuffer, input: Row) = {
-      if(buffer.getMap(0).size < maxItems)
-      {
+    def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+      if (buffer.getMap(0).size < maxItems) {
         val key = input.getString(0)
-        if (key != null)
+        if (key != null) {
           buffer(0) = buffer.getMap(0) + (key -> true)
+        }
       }
     }
 
     // Merge two partial aggregates
-    def merge(buffer1: MutableAggregationBuffer, buffer2: Row) = {
-      if(buffer1.getMap(0).size < maxItems)
-      {
+    def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+      if (buffer1.getMap(0).size < maxItems) {
         val mp = mutable.Map[String, Boolean]() ++= buffer1.getMap(0)
-        buffer2.getMap(0).keys.foreach((key: String) =>
-          if (key != null)
-            mp.put(key,true)
-        )
+        buffer2
+          .getMap(0)
+          .keys
+          .foreach((key: String) =>
+            if (key != null) {
+              mp.put(key, true)
+            })
         buffer1(0) = mp
       }
 
     }
 
     // Called after all the entries are exhausted.
-    def evaluate(buffer: Row) = {
-      math.min(buffer.getMap(0).size,maxItems)
+    def evaluate(buffer: Row): Int = {
+      math.min(buffer.getMap(0).size, maxItems)
     }
 
   }
 
-}
\ No newline at end of file
+}
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
index 5949b62..b1b81b8 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkOverwriteUDAFs.scala
@@ -20,19 +20,28 @@ package org.apache.spark.sql.types
 
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryComparison, ExpectsInputTypes, Expression, GreaterThan, If, IsNull, LessThan, Literal}
 import org.apache.spark.sql.catalyst.util.TypeUtils
 
 object SparkOverwriteUDAFs {
-  def minValueByKey(key: Column, value: Column): Column = Column(MinValueByKey(key.expr, value.expr).toAggregateExpression(false))
-  def maxValueByKey(key: Column, value: Column): Column = Column(MaxValueByKey(key.expr, value.expr).toAggregateExpression(false))
+  def minValueByKey(key: Column, value: Column): Column =
+    Column(MinValueByKey(key.expr, value.expr).toAggregateExpression(false))
+  def maxValueByKey(key: Column, value: Column): Column =
+    Column(MaxValueByKey(key.expr, value.expr).toAggregateExpression(false))
 }
 
-case class MinValueByKey(child1: Expression, child2: Expression) extends ExtramumValueByKey(child1,child2, LessThan)
-case class MaxValueByKey(child1: Expression, child2: Expression) extends ExtramumValueByKey(child1,child2, GreaterThan)
+case class MinValueByKey(child1: Expression, child2: Expression)
+    extends ExtramumValueByKey(child1, child2, LessThan)
+case class MaxValueByKey(child1: Expression, child2: Expression)
+    extends ExtramumValueByKey(child1, child2, GreaterThan)
 
-abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp: (Expression, Expression) => BinaryComparison) extends DeclarativeAggregate with ExpectsInputTypes {
+abstract class ExtramumValueByKey(
+    child1: Expression,
+    child2: Expression,
+    bComp: (Expression, Expression) => BinaryComparison)
+    extends DeclarativeAggregate
+    with ExpectsInputTypes {
 
   override def children: Seq[Expression] = child1 :: child2 :: Nil
 
@@ -50,7 +59,8 @@ abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp:
   private lazy val minmax = AttributeReference("minmax", child1.dataType)()
   private lazy val data = AttributeReference("data", child2.dataType)()
 
-  override lazy val aggBufferAttributes: Seq[AttributeReference] = minmax :: data :: Nil
+  override lazy val aggBufferAttributes
+    : Seq[AttributeReference] = minmax :: data :: Nil
 
   override lazy val initialValues: Seq[Expression] = Seq(
     Literal.create(null, child1.dataType),
@@ -63,11 +73,17 @@ abstract class ExtramumValueByKey(child1: Expression, child2: Expression, bComp:
   override lazy val mergeExpressions: Seq[Expression] =
     chooseKeyValue(minmax.left, data.left, minmax.right, data.right)
 
-  def chooseKeyValue(key1:Expression, value1: Expression, key2:Expression, value2: Expression) = Seq(
-    If(IsNull(key1), key2,  If(IsNull(key2), key1,    If(bComp(key1, key2), key1, key2))),
-    If(IsNull(key1), value2, If(IsNull(key2), value1, If(bComp(key1, key2), value1, value2)))
+  private def chooseKeyValue(key1: Expression,
+                     value1: Expression,
+                     key2: Expression,
+                     value2: Expression) = Seq(
+    If(IsNull(key1),
+       key2,
+       If(IsNull(key2), key1, If(bComp(key1, key2), key1, key2))),
+    If(IsNull(key1),
+       value2,
+       If(IsNull(key2), value1, If(bComp(key1, key2), value1, value2)))
   )
 
   override lazy val evaluateExpression: AttributeReference = data
 }
-
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
index 3eed62c..c904587 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
@@ -18,23 +18,25 @@ package org.apache.spark.deploy
 
 import java.io._
 
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
 import datafu.spark.ScalaPythonBridge
 import org.apache.log4j.Logger
+
 import org.apache.spark.api.python.PythonUtils
 import org.apache.spark.util.Utils
 
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
 /**
-  * We wrap Spark's PythonRunner because we failed on premature python process closing.
-  * in PythonRunner the python process exits immediately when finished to read the filename,
-  * this caused us to Accumulators Exceptions when the driver tries to get accumulation data
-  * from the python gateway.
-  * Instead, like in Zeppelin, we create an "interactive" python process, feed it the python
-  * script and not closing the gateway.
-  */
-case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()) {
+ * We wrap Spark's PythonRunner because we failed on premature python process closing.
+ * in PythonRunner the python process exits immediately when finished to read the filename,
+ * this caused us to Accumulators Exceptions when the driver tries to get accumulation data
+ * from the python gateway.
+ * Instead, like in Zeppelin, we create an "interactive" python process, feed it the python
+ * script and not closing the gateway.
+ */
+case class SparkPythonRunner(pyPaths: String,
+                             otherArgs: Array[String] = Array()) {
 
   val logger: Logger = Logger.getLogger(getClass)
   val (reader, writer, process) = initPythonEnv()
@@ -49,7 +51,8 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
   private def initPythonEnv(): (BufferedReader, BufferedWriter, Process) = {
 
     val pythonExec =
-      sys.env.getOrElse("PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python"))
+      sys.env.getOrElse("PYSPARK_DRIVER_PYTHON",
+                        sys.env.getOrElse("PYSPARK_PYTHON", "python"))
 
     // Format python filename paths before adding them to the PYTHONPATH
     val formattedPyFiles = PythonRunner.formatPaths(pyPaths)
@@ -79,39 +82,33 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
     pathElements += PythonUtils.sparkPythonPath
     pathElements += sys.env.getOrElse("PYTHONPATH", "")
     val pythonPath = PythonUtils.mergePythonPaths(pathElements: _*)
-    logger.info(s"Running python with PYTHONPATH:\n\t${formattedPyFiles.mkString(",")}")
+    logger.info(
+      s"Running python with PYTHONPATH:\n\t${formattedPyFiles.mkString(",")}")
 
     // Launch Python process
-    //val builder = new ProcessBuilder((Seq(pythonExec, formattedPythonFile) ++ otherArgs).asJava)
-    val builder = new ProcessBuilder((Seq(pythonExec, "-iu") ++ otherArgs).asJava)
+    val builder = new ProcessBuilder(
+      (Seq(pythonExec, "-iu") ++ otherArgs).asJava)
     val env = builder.environment()
     env.put("PYTHONPATH", pythonPath)
     // This is equivalent to setting the -u flag; we use it because ipython doesn't support -u:
     env.put("PYTHONUNBUFFERED", "YES") // value is needed to be set to a non-empty string
     env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
     builder.redirectErrorStream(true) // Ugly but needed for stdout and stderr to synchronize
-    //try {
-      val process = builder.start()
-      //new RedirectThread(process.getInputStream, System.out, "redirect output").start()
-      val writer = new BufferedWriter(new OutputStreamWriter(process.getOutputStream))
-      val reader = new BufferedReader(new InputStreamReader(process.getInputStream))
-
-      //      val exitCode = process.waitFor()
-      //      if (exitCode != 0) {
-      //        throw new SparkUserAppException(exitCode)
-      //      }
-    //} finally {
-      //      gatewayServer.shutdown()
-    //}
+    val process = builder.start()
+    val writer = new BufferedWriter(
+      new OutputStreamWriter(process.getOutputStream))
+    val reader = new BufferedReader(
+      new InputStreamReader(process.getInputStream))
 
     (reader, writer, process)
   }
 
-
-  private def execFile(filename: String, writer: BufferedWriter, reader: BufferedReader): String = {
+  private def execFile(filename: String,
+                       writer: BufferedWriter,
+                       reader: BufferedReader): String = {
     writer.write("import traceback\n")
     writer.write("try:\n")
-    writer.write("    execfile('"+filename+"')\n")
+    writer.write("    execfile('" + filename + "')\n")
     writer.write("    print (\"*!?flush reader!?*\")\n")
     writer.write("except Exception as e:\n")
     writer.write("    traceback.print_exc()\n")
@@ -120,19 +117,19 @@ case class SparkPythonRunner(pyPaths: String, otherArgs: Array[String] = Array()
     writer.flush()
     var output = ""
     var line: String = reader.readLine
-    while (!line.contains("*!?flush reader!?*") && !line.contains("*!?flush error reader!?*")) {
-      //logger.debug("Read line from python shell : " + line)
-      System.out.println(line)
+    while (!line.contains("*!?flush reader!?*") && !line.contains(
+             "*!?flush error reader!?*")) {
+      logger.info(line)
       if (line == "...") {
-        //logger.warn("Syntax error ! ")
         output += "Syntax error ! "
       }
       output += "\r" + line + "\n"
       line = reader.readLine
     }
 
-    if (line.contains("*!?flush error reader!?*"))
+    if (line.contains("*!?flush error reader!?*")) {
       throw new RuntimeException("python bridge error: " + output)
+    }
 
     output
   }
diff --git a/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource b/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
index 714c69c..072cde9 100644
--- a/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
+++ b/datafu-spark/src/test/resources/META-INF/services/datafu.spark.PythonResource
@@ -17,4 +17,4 @@
 
 datafu.spark.ExampleFiles
 datafu.spark.Py4JResource
-datafu.spark.PysparkResource
\ No newline at end of file
+datafu.spark.PysparkResource
diff --git a/datafu-spark/src/test/resources/example_tests/pyfromscala.py b/datafu-spark/src/test/resources/example_tests/pyfromscala.py
index c2c3c6f..040854b 100644
--- a/datafu-spark/src/test/resources/example_tests/pyfromscala.py
+++ b/datafu-spark/src/test/resources/example_tests/pyfromscala.py
@@ -14,7 +14,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
- 
+
+# Some usage exmaples of python-Scala functionality
+
 # print the PYTHONPATH
 import sys
 from pprint import pprint as p
diff --git a/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala b/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
index 93c9c5f..2ee8bde 100644
--- a/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/PySparkLibTestResources.scala
@@ -44,6 +44,10 @@ object PathsResolver {
   
   val py4jVersion = py4js.getOrElse(sparkVersion, "0.10.6") // our default
   
-  val pyspark = ResourceCloning.cloneResource(getClass.getResource("/built_in_pyspark_lib/pyspark-" + sparkVersion + ".zip"), "pyspark_cloned.zip").getPath
-  val py4j = ResourceCloning.cloneResource(getClass.getResource("/built_in_pyspark_lib/py4j-" + py4jVersion + "-src.zip"), "py4j_cloned.zip").getPath
+  val pyspark = ResourceCloning.cloneResource(
+    getClass.getResource("/built_in_pyspark_lib/pyspark-" + sparkVersion + ".zip"),
+    "pyspark_cloned.zip").getPath
+  val py4j = ResourceCloning.cloneResource(
+    getClass.getResource("/built_in_pyspark_lib/py4j-" + py4jVersion + "-src.zip"),
+    "py4j_cloned.zip").getPath
 }
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
index f3bea97..1408e2c 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestScalaPythonBridge.scala
@@ -20,18 +20,22 @@ package datafu.spark
 
 import java.io.File
 
+import scala.util.Try
+
 import com.holdenkarau.spark.testing.Utils
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.SparkSession
 import org.junit._
+import org.junit.runner.RunWith
 import org.scalatest.FunSuite
 import org.scalatest.junit.JUnitRunner
-import org.junit.runner.RunWith
+import org.slf4j.LoggerFactory
 
-import scala.util.Try
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
 
 object TestScalaPythonBridge {
 
+  val logger = LoggerFactory.getLogger(this.getClass)
+
   def getNewRunner(): ScalaPythonBridgeRunner = {
     val runner = ScalaPythonBridgeRunner()
     runner.runPythonFile("pyspark_utils/init_spark_context.py")
@@ -43,15 +47,16 @@ object TestScalaPythonBridge {
     val tempDir = Utils.createTempDir()
     val localMetastorePath = new File(tempDir, "metastore").getCanonicalPath
     val localWarehousePath = new File(tempDir, "wharehouse").getCanonicalPath
-    val pythonPath = PythonPathsManager.getAbsolutePaths().mkString(File.pathSeparator)
-    println("Creating SparkConf with PYTHONPATH: " + pythonPath)
+    val pythonPath =
+      PythonPathsManager.getAbsolutePaths().mkString(File.pathSeparator)
+    logger.info("Creating SparkConf with PYTHONPATH: " + pythonPath)
     val sparkConf = new SparkConf()
-    .setMaster("local[1]")
-    .set("spark.sql.warehouse.dir", localWarehousePath)
-    .set("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=$localMetastorePath;create=true")
-    //.set("datanucleus.rdbms.datastoreAdapterClassName", "org.datanucleus.store.rdbms.adapter.DerbyAdapter")
-    .setExecutorEnv(Seq(("PYTHONPATH", pythonPath)))
-    .setAppName("Spark Unit Test")
+      .setMaster("local[1]")
+      .set("spark.sql.warehouse.dir", localWarehousePath)
+      .set("javax.jdo.option.ConnectionURL",
+           s"jdbc:derby:;databaseName=$localMetastorePath;create=true")
+      .setExecutorEnv(Seq(("PYTHONPATH", pythonPath)))
+      .setAppName("Spark Unit Test")
 
     val builder = SparkSession.builder().config(sparkConf).enableHiveSupport()
     val spark = builder.getOrCreate()
@@ -67,8 +72,9 @@ class TestScalaPythonBridge extends FunSuite {
   private lazy val runner = TestScalaPythonBridge.getNewRunner()
 
   def assertTable(tableName: String, expected: String): Unit =
-    Assert.assertEquals(expected, spark.table(tableName).collect().sortBy(_.toString).mkString(", "))
-
+    Assert.assertEquals(
+      expected,
+      spark.table(tableName).collect().sortBy(_.toString).mkString(", "))
 
   test("pyfromscala.py") {
 
@@ -80,11 +86,16 @@ class TestScalaPythonBridge extends FunSuite {
     runner.runPythonFile("example_tests/pyfromscala.py")
 
     // try to invoke python udf from scala code
-    assert(spark.sql("select magic('python_udf')").collect().mkString(",") == "[python_udf magic]")
-
-    //spark.sql("select * from dfout").registerTempTable("output")
-    assertTable("dfout", "[10], [12], [14], [16], [18], [20], [2], [4], [6], [8]")
-    assertTable("dfout2", "[16], [24], [32], [40], [48], [56], [64], [72], [80], [8]")
+    assert(
+      spark
+        .sql("select magic('python_udf')")
+        .collect()
+        .mkString(",") == "[python_udf magic]")
+
+    assertTable("dfout",
+                "[10], [12], [14], [16], [18], [20], [2], [4], [6], [8]")
+    assertTable("dfout2",
+                "[16], [24], [32], [40], [48], [56], [64], [72], [80], [8]")
     assertTable("stats", "[a,0.1], [b,2.0]")
   }
 
@@ -93,17 +104,24 @@ class TestScalaPythonBridge extends FunSuite {
     assert(t.isFailure)
     assert(t.failed.get.isInstanceOf[RuntimeException])
   }
-  
+
   test("SparkDFUtilsBridge") {
     runner.runPythonFile("example_tests/df_utils_tests.py")
     assertTable("dedup", "[a,Alice,34], [b,Bob,36], [c,Zoey,36]")
-    assertTable("dedupTopN", "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,Fanny,36], [c,Zoey,36]")
+    assertTable(
+      "dedupTopN",
+      "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,Fanny,36], [c,Zoey,36]")
     assertTable("dedup2", "[a,34], [b,36], [c,36]")
-    assertTable("changeSchema", "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], [c,Fanny,36], [c,Zoey,36]")
+    assertTable(
+      "changeSchema",
+      "[a,Alice,34], [a,Sara,33], [b,Bob,36], [b,Charlie,30], [c,David,29], [c,Esther,32], " +
+        "[c,Fanny,36], [c,Zoey,36]")
     assertTable("joinSkewed", "[a,Laura,34,a,1], [a,Stephani,33,a,1]")
     assertTable("broadcastJoinSkewed", "[a,Laura,34,1], [a,Stephani,33,1]")
-    assertTable("joinWithRange", "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
-    assertTable("joinWithRangeAndDedup", "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
+    assertTable("joinWithRange",
+                "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
+    assertTable("joinWithRangeAndDedup",
+                "[a,Laura,34,a,34,36], [b,Margaret,36,a,34,36]")
   }
 
 }
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala b/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
index 988e206..8c7453e 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestSparkDFUtils.scala
@@ -18,36 +18,28 @@
  */
 package datafu.spark
 
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.Row
-
-import org.apache.spark.sql.functions._
-
 import com.holdenkarau.spark.testing.DataFrameSuiteBase
-
+import org.junit.runner.RunWith
 import org.scalatest.FunSuite
-
 import org.scalatest.junit.JUnitRunner
-import org.junit.runner.RunWith
-
-import scala.collection.mutable.WrappedArray
 
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 
-import org.apache.spark.rdd.RDD
 
 @RunWith(classOf[JUnitRunner])
 class DataFrameOpsTests extends FunSuite with DataFrameSuiteBase {
-  
+
   import DataFrameOps._
-  
-	import spark.implicits._
 
-	val inputSchema = List(
-      StructField("col_grp", StringType, true),
-      StructField("col_ord", IntegerType, false),
-      StructField("col_str", StringType, true)
+  import spark.implicits._
+
+  val inputSchema = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, false),
+    StructField("col_str", StringType, true)
   )
 
   val dedupSchema = List(
@@ -55,230 +47,332 @@ class DataFrameOpsTests extends FunSuite with DataFrameSuiteBase {
     StructField("col_ord", IntegerType, false)
   )
 
-  lazy val inputRDD = sc.parallelize(Seq(Row("a", 1, "asd1"), Row("a", 2, "asd2"), Row("a", 3, "asd3"), Row("b", 1, "asd4")))
-  
-  lazy val inputDataFrame = sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
+  lazy val inputRDD = sc.parallelize(
+    Seq(Row("a", 1, "asd1"),
+        Row("a", 2, "asd2"),
+        Row("a", 3, "asd3"),
+        Row("b", 1, "asd4")))
 
-  test("dedup") {
-    val expected : DataFrame = sqlContext.createDataFrame(sc.parallelize(Seq(Row("b",1),Row("a",3))), StructType(dedupSchema))
+  lazy val inputDataFrame =
+    sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
 
-    assertDataFrameEquals(expected, inputDataFrame.dedup($"col_grp", $"col_ord".desc).select($"col_grp", $"col_ord"))
+  test("dedup") {
+    val expected: DataFrame =
+      sqlContext.createDataFrame(sc.parallelize(Seq(Row("b", 1), Row("a", 3))),
+                                 StructType(dedupSchema))
+
+    assertDataFrameEquals(expected,
+                          inputDataFrame
+                            .dedup($"col_grp", $"col_ord".desc)
+                            .select($"col_grp", $"col_ord"))
   }
 
- 	case class dedup_exp(col2: String, col_grp:String, col_ord:Option[Int], col_str:String)
+  case class dedupExp(col2: String,
+                       col_grp: String,
+                       col_ord: Option[Int],
+                       col_str: String)
 
   test("dedup2_by_int") {
 
-   	val expectedByIntDf : DataFrame = sqlContext.createDataFrame(List(dedup_exp("asd4","b",Option(1),"asd4"),dedup_exp("asd1","a",Option(3),"asd3")))
+    val expectedByIntDf: DataFrame = sqlContext.createDataFrame(
+      List(dedupExp("asd4", "b", Option(1), "asd4"),
+        dedupExp("asd1", "a", Option(3), "asd3")))
 
-    val actual = inputDataFrame.dedup2($"col_grp", $"col_ord", moreAggFunctions = Seq(min($"col_str")))
+    val actual = inputDataFrame.dedup2($"col_grp",
+                                       $"col_ord",
+                                       moreAggFunctions = Seq(min($"col_str")))
 
     assertDataFrameEquals(expectedByIntDf, actual)
   }
 
-  	case class dedup_exp2(col_grp:String, col_ord:Option[Int], col_str:String)
+  case class dedupExp2(col_grp: String, col_ord: Option[Int], col_str: String)
 
-    test("dedup2_by_string_asc") {
+  test("dedup2_by_string_asc") {
 
-      val actual = inputDataFrame.dedup2($"col_grp", $"col_str", desc = false)
-    
-      val expectedByStringDf : DataFrame = sqlContext.createDataFrame(List(dedup_exp2("b",Option(1),"asd4"),dedup_exp2("a",Option(1),"asd1")))
+    val actual = inputDataFrame.dedup2($"col_grp", $"col_str", desc = false)
 
-      assertDataFrameEquals(expectedByStringDf, actual)
-    }
+    val expectedByStringDf: DataFrame = sqlContext.createDataFrame(
+      List(dedupExp2("b", Option(1), "asd4"),
+        dedupExp2("a", Option(1), "asd1")))
 
-    test("test_dedup2_by_complex_column") {
+    assertDataFrameEquals(expectedByStringDf, actual)
+  }
 
-      val actual = inputDataFrame.dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"), desc = false)
-    
-      val expectedComplex : DataFrame = sqlContext.createDataFrame(List(dedup_exp2("b",Option(1),"asd4"),dedup_exp2("a",Option(3),"asd3")))
+  test("test_dedup2_by_complex_column") {
 
-      assertDataFrameEquals(expectedComplex, actual)
-    }
+    val actual = inputDataFrame.dedup2($"col_grp",
+                                       expr("cast(concat('-',col_ord) as int)"),
+                                       desc = false)
 
-    case class inner(col_grp:String, col_ord:Int)
-    
-    case class expComplex(
-        col_grp: String,
-        col_ord: Option[Int],
-        col_str: String,
-        arr_col: Array[String],
-        struct_col: inner,
-        map_col: Map[String, Int]
-    )
-    
-    test("test_dedup2_with_other_complex_column") {
-
-      val actual = inputDataFrame.withColumn("arr_col", expr("array(col_grp, col_ord)"))
-                  .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
-                  .withColumn("map_col", expr("map(col_grp, col_ord)"))
-                  .withColumn("map_col_blah", expr("map(col_grp, col_ord)"))
-                  .dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"))
-                  .drop("map_col_blah")
-
-      val expected : DataFrame = sqlContext.createDataFrame(List(
-          expComplex("b", Option(1), "asd4", Array("b","1"), inner("b",1), Map("b" -> 1)),
-          expComplex("a", Option(1), "asd1", Array("a","1"), inner("a",1), Map("a" -> 1))
+    val expectedComplex: DataFrame = sqlContext.createDataFrame(
+      List(dedupExp2("b", Option(1), "asd4"),
+        dedupExp2("a", Option(3), "asd3")))
+
+    assertDataFrameEquals(expectedComplex, actual)
+  }
+
+  case class Inner(colGrp: String, colOrd: Int)
+
+  case class expComplex(
+                         col_grp: String,
+                         col_ord: Option[Int],
+                         col_str: String,
+                         arr_col: Array[String],
+                         struct_col: Inner,
+                         map_col: Map[String, Int]
+  )
+
+  test("test_dedup2_with_other_complex_column") {
+
+    val actual = inputDataFrame
+      .withColumn("arr_col", expr("array(col_grp, col_ord)"))
+      .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
+      .withColumn("map_col", expr("map(col_grp, col_ord)"))
+      .withColumn("map_col_blah", expr("map(col_grp, col_ord)"))
+      .dedup2($"col_grp", expr("cast(concat('-',col_ord) as int)"))
+      .drop("map_col_blah")
+
+    val expected: DataFrame = sqlContext.createDataFrame(
+      List(
+        expComplex("b",
+                   Option(1),
+                   "asd4",
+                   Array("b", "1"),
+                   Inner("b", 1),
+                   Map("b" -> 1)),
+        expComplex("a",
+                   Option(1),
+                   "asd1",
+                   Array("a", "1"),
+                   Inner("a", 1),
+                   Map("a" -> 1))
       ))
-                  
-      assertDataFrameEquals(expected, actual)
-    }
 
-    val dedupTopNExpectedSchema = List(
-      StructField("col_grp", StringType, true),
-      StructField("col_ord", IntegerType, false)
-    )
+    assertDataFrameEquals(expected, actual)
+  }
 
-    test("test_dedup_top_n") {
-      val actual = inputDataFrame.dedupTopN(2, $"col_grp", $"col_ord".desc).select($"col_grp", $"col_ord")
-      
-      val expected = sqlContext.createDataFrame(sc.parallelize(Seq(Row("b",1), Row("a",3), Row("a",2))), StructType(dedupTopNExpectedSchema))
-      
-       assertDataFrameEquals(expected, actual)
-    }
-
-    val schema2 = List(
-      StructField("start", IntegerType, false),
-      StructField("end", IntegerType, false),
-      StructField("desc", StringType, true)
-    )
+  val dedupTopNExpectedSchema = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, false)
+  )
 
-    val expectedSchemaRangedJoin = List(
-        StructField("col_grp",StringType,true),
-        StructField("col_ord",IntegerType,false),
-        StructField("col_str",StringType,true),
-        StructField("start",IntegerType,true),
-        StructField("end",IntegerType,true),
-        StructField("desc",StringType,true)
+  test("test_dedup_top_n") {
+    val actual = inputDataFrame
+      .dedupTopN(2, $"col_grp", $"col_ord".desc)
+      .select($"col_grp", $"col_ord")
+
+    val expected = sqlContext.createDataFrame(
+      sc.parallelize(Seq(Row("b", 1), Row("a", 3), Row("a", 2))),
+      StructType(dedupTopNExpectedSchema))
+
+    assertDataFrameEquals(expected, actual)
+  }
+
+  val schema2 = List(
+    StructField("start", IntegerType, false),
+    StructField("end", IntegerType, false),
+    StructField("desc", StringType, true)
+  )
+
+  val expectedSchemaRangedJoin = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, false),
+    StructField("col_str", StringType, true),
+    StructField("start", IntegerType, true),
+    StructField("end", IntegerType, true),
+    StructField("desc", StringType, true)
+  )
+
+  test("join_with_range") {
+    val joinWithRangeDataFrame =
+      sqlContext.createDataFrame(sc.parallelize(
+                                   Seq(Row(1, 2, "asd1"),
+                                       Row(1, 4, "asd2"),
+                                       Row(3, 5, "asd3"),
+                                       Row(3, 10, "asd4"))),
+                                 StructType(schema2))
+
+    val expected = sqlContext.createDataFrame(
+      sc.parallelize(
+        Seq(
+          Row("b", 1, "asd4", 1, 2, "asd1"),
+          Row("a", 2, "asd2", 1, 2, "asd1"),
+          Row("a", 1, "asd1", 1, 2, "asd1"),
+          Row("b", 1, "asd4", 1, 4, "asd2"),
+          Row("a", 3, "asd3", 1, 4, "asd2"),
+          Row("a", 2, "asd2", 1, 4, "asd2"),
+          Row("a", 1, "asd1", 1, 4, "asd2"),
+          Row("a", 3, "asd3", 3, 5, "asd3"),
+          Row("a", 3, "asd3", 3, 10, "asd4")
+        )),
+      StructType(expectedSchemaRangedJoin)
     )
-    
-    test("join_with_range") {
-      val joinWithRangeDataFrame = sqlContext.createDataFrame(sc.parallelize(Seq(Row(1, 2,"asd1"), Row(1, 4, "asd2"), Row(3, 5,"asd3"), Row(3, 10,"asd4"))),StructType(schema2))
-  
-      val expected = sqlContext.createDataFrame(sc.parallelize(Seq(
-        Row("b",1,"asd4",1,2,"asd1"),
-        Row("a",2,"asd2",1,2,"asd1"),
-        Row("a",1,"asd1",1,2,"asd1"),
-        Row("b",1,"asd4",1,4,"asd2"),
-        Row("a",3,"asd3",1,4,"asd2"),
-        Row("a",2,"asd2",1,4,"asd2"),
-        Row("a",1,"asd1",1,4,"asd2"),
-        Row("a",3,"asd3",3,5,"asd3"),
-        Row("a",3,"asd3",3,10,"asd4")
-      )), StructType(expectedSchemaRangedJoin))
-      
-      val actual = inputDataFrame.joinWithRange("col_ord", joinWithRangeDataFrame, "start", "end")
-      
-      assertDataFrameEquals(expected, actual)
+
+    val actual = inputDataFrame.joinWithRange("col_ord",
+                                              joinWithRangeDataFrame,
+                                              "start",
+                                              "end")
+
+    assertDataFrameEquals(expected, actual)
   }
 
-      val expectedSchemaRangedJoinWithDedup = List(
-        StructField("col_grp",StringType,true),
-        StructField("col_ord",IntegerType,true),
-        StructField("col_str",StringType,true),
-        StructField("start",IntegerType,true),
-        StructField("end",IntegerType,true),
-        StructField("desc",StringType,true)
+  val expectedSchemaRangedJoinWithDedup = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, true),
+    StructField("col_str", StringType, true),
+    StructField("start", IntegerType, true),
+    StructField("end", IntegerType, true),
+    StructField("desc", StringType, true)
+  )
+
+  test("join_with_range_and_dedup") {
+    val df = sc
+      .parallelize(
+        List(("a", 1, "asd1"),
+             ("a", 2, "asd2"),
+             ("a", 3, "asd3"),
+             ("b", 1, "asd4")))
+      .toDF("col_grp", "col_ord", "col_str")
+    val dfr = sc
+      .parallelize(
+        List((1, 2, "asd1"), (1, 4, "asd2"), (3, 5, "asd3"), (3, 10, "asd4")))
+      .toDF("start", "end", "desc")
+
+    val expected = sqlContext.createDataFrame(
+      sc.parallelize(
+        Seq(
+          Row("b", 1, "asd4", 1, 2, "asd1"),
+          Row("a", 3, "asd3", 3, 5, "asd3"),
+          Row("a", 2, "asd2", 1, 2, "asd1")
+        )),
+      StructType(expectedSchemaRangedJoinWithDedup)
     )
 
-    test("join_with_range_and_dedup") {
-          val df = sc.parallelize(List(("a", 1, "asd1"), ("a", 2, "asd2"), ("a", 3, "asd3"), ("b", 1, "asd4"))).toDF("col_grp", "col_ord", "col_str")
-       	  val dfr = sc.parallelize(List((1, 2,"asd1"), (1, 4, "asd2"), (3, 5,"asd3"), (3, 10,"asd4"))).toDF("start", "end", "desc")
-    
-          val expected = sqlContext.createDataFrame(sc.parallelize(Seq(
-              Row("b",1,"asd4",1,2,"asd1"),
-              Row("a",3,"asd3",3,5,"asd3"),
-              Row("a",2,"asd2",1,2,"asd1")
-          )), StructType(expectedSchemaRangedJoinWithDedup))
-    
-          
-          val actual = df.joinWithRangeAndDedup("col_ord", dfr, "start", "end")
-    
-          assertDataFrameEquals(expected, actual)
-    }
-
-    test("broadcastJoinSkewed") {
-      val skewedList = List(("1", "a"), ("1", "b"), ("1", "c"), ("1", "d"), ("1", "e"),("2", "k"),("0", "k"))
-      val skewed = sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
-      val notSkewed = sqlContext.createDataFrame((1 to 10).map(i => (i.toString, s"str$i"))).toDF("key", "val")
-
-      val expected = sqlContext.createDataFrame(List(
-          ("1","str1", "a"),
-          ("1","str1", "b"),
-          ("1","str1", "c"),
-          ("1","str1", "d"),
-          ("1","str1", "e"),
-          ("2","str2", "k")
-      )).toDF("key","val","val_skewed")
-
-      val actual1 = notSkewed.broadcastJoinSkewed(skewed,"key", 1)
-
-      assertDataFrameEquals(expected, actual1.sort($"val_skewed"))
-      
-      val actual2 = notSkewed.broadcastJoinSkewed(skewed,"key", 2)
-
-      assertDataFrameEquals(expected, actual2.sort($"val_skewed"))
-    }
-  
-    // because of nulls in expected data, an actual schema needs to be used
-    case class expJoinSkewed(str1:String, str2:String, str3:String, str4:String )
-  
-    test("joinSkewed") {
-      val skewedList = List(("1", "a"), ("1", "b"), ("1", "c"), ("1", "d"), ("1", "e"),("2", "k"),("0", "k"))
-      val skewed = sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
-      val notSkewed = sqlContext.createDataFrame((1 to 10).map(i => (i.toString, s"str$i"))).toDF("key", "val")
-  
-      val actual1 = skewed.as("a").joinSkewed(notSkewed.as("b"),expr("a.key = b.key"), 3)
-  
-      val expected1 = sqlContext.createDataFrame(List(
-          ("1","a","1","str1"),
-          ("1","b","1","str1"),
-          ("1","c","1","str1"),
-          ("1","d","1","str1"),
-          ("1","e","1","str1"),
-          ("2","k","2","str2")
-      )).toDF("key","val_skewed","key","val")
-  
-      assertDataFrameEquals(expected1, actual1.sort($"val_skewed")) // assertDataFrameEquals cares about order but we don't
-      
-      val actual2 = skewed.as("a").joinSkewed(notSkewed.as("b"),expr("a.key = b.key"), 3, "left_outer")
-  
-      val expected2 = sqlContext.createDataFrame(List(
-          expJoinSkewed("1","a","1","str1"),
-          expJoinSkewed("1","b","1","str1"),
-          expJoinSkewed("1","c","1","str1"),
-          expJoinSkewed("1","d","1","str1"),
-          expJoinSkewed("1","e","1","str1"),
-          expJoinSkewed("2","k","2","str2"),
-          expJoinSkewed("0","k",null,null)
-      )).toDF("key","val_skewed","key","val")
-  
-      assertDataFrameEquals(expected2, actual2.sort($"val_skewed")) // assertDataFrameEquals cares about order but we don't
-    }
-
-    val changedSchema = List(
-      StructField("fld1", StringType, true),
-      StructField("fld2", IntegerType, false),
-      StructField("fld3", StringType, true)
+    val actual = df.joinWithRangeAndDedup("col_ord", dfr, "start", "end")
+
+    assertDataFrameEquals(expected, actual)
+  }
+
+  test("broadcastJoinSkewed") {
+    val skewedList = List(("1", "a"),
+                          ("1", "b"),
+                          ("1", "c"),
+                          ("1", "d"),
+                          ("1", "e"),
+                          ("2", "k"),
+                          ("0", "k"))
+    val skewed =
+      sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
+    val notSkewed = sqlContext
+      .createDataFrame((1 to 10).map(i => (i.toString, s"str$i")))
+      .toDF("key", "val")
+
+    val expected = sqlContext
+      .createDataFrame(
+        List(
+          ("1", "str1", "a"),
+          ("1", "str1", "b"),
+          ("1", "str1", "c"),
+          ("1", "str1", "d"),
+          ("1", "str1", "e"),
+          ("2", "str2", "k")
+        ))
+      .toDF("key", "val", "val_skewed")
+
+    val actual1 = notSkewed.broadcastJoinSkewed(skewed, "key", 1)
+
+    assertDataFrameEquals(expected, actual1.sort($"val_skewed"))
+
+    val actual2 = notSkewed.broadcastJoinSkewed(skewed, "key", 2)
+
+    assertDataFrameEquals(expected, actual2.sort($"val_skewed"))
+  }
+
+  // because of nulls in expected data, an actual schema needs to be used
+  case class expJoinSkewed(str1: String,
+                           str2: String,
+                           str3: String,
+                           str4: String)
+
+  test("joinSkewed") {
+    val skewedList = List(("1", "a"),
+                          ("1", "b"),
+                          ("1", "c"),
+                          ("1", "d"),
+                          ("1", "e"),
+                          ("2", "k"),
+                          ("0", "k"))
+    val skewed =
+      sqlContext.createDataFrame(skewedList).toDF("key", "val_skewed")
+    val notSkewed = sqlContext
+      .createDataFrame((1 to 10).map(i => (i.toString, s"str$i")))
+      .toDF("key", "val")
+
+    val actual1 =
+      skewed.as("a").joinSkewed(notSkewed.as("b"), expr("a.key = b.key"), 3)
+
+    val expected1 = sqlContext
+      .createDataFrame(
+        List(
+          ("1", "a", "1", "str1"),
+          ("1", "b", "1", "str1"),
+          ("1", "c", "1", "str1"),
+          ("1", "d", "1", "str1"),
+          ("1", "e", "1", "str1"),
+          ("2", "k", "2", "str2")
+        ))
+      .toDF("key", "val_skewed", "key", "val")
+
+    // assertDataFrameEquals cares about order but we don't
+    assertDataFrameEquals(expected1, actual1.sort($"val_skewed"))
+
+    val actual2 = skewed
+      .as("a")
+      .joinSkewed(notSkewed.as("b"), expr("a.key = b.key"), 3, "left_outer")
+
+    val expected2 = sqlContext
+      .createDataFrame(
+        List(
+          expJoinSkewed("1", "a", "1", "str1"),
+          expJoinSkewed("1", "b", "1", "str1"),
+          expJoinSkewed("1", "c", "1", "str1"),
+          expJoinSkewed("1", "d", "1", "str1"),
+          expJoinSkewed("1", "e", "1", "str1"),
+          expJoinSkewed("2", "k", "2", "str2"),
+          expJoinSkewed("0", "k", null, null)
+        ))
+      .toDF("key", "val_skewed", "key", "val")
+
+    // assertDataFrameEquals cares about order but we don't
+    assertDataFrameEquals(expected2, actual2.sort($"val_skewed"))
+  }
+
+  val changedSchema = List(
+    StructField("fld1", StringType, true),
+    StructField("fld2", IntegerType, false),
+    StructField("fld3", StringType, true)
   )
-  
-    test("test_changeSchema") {
-
-      val actual = inputDataFrame.changeSchema("fld1","fld2", "fld3")
-    
-      val expected = sqlContext.createDataFrame(inputRDD, StructType(changedSchema))
-      
-      assertDataFrameEquals(expected, actual)
-    }
-
-    test("test_flatten") {
-
-      val input = inputDataFrame.withColumn("struct_col", expr("struct(col_grp, col_ord)")).select("struct_col")
-    
-      val expected : DataFrame = inputDataFrame.select("col_grp", "col_ord")
-      
-      val actual = input.flatten("struct_col")
-      
-      assertDataFrameEquals(expected, actual)
-    }
-}
\ No newline at end of file
+
+  test("test_changeSchema") {
+
+    val actual = inputDataFrame.changeSchema("fld1", "fld2", "fld3")
+
+    val expected =
+      sqlContext.createDataFrame(inputRDD, StructType(changedSchema))
+
+    assertDataFrameEquals(expected, actual)
+  }
+
+  test("test_flatten") {
+
+    val input = inputDataFrame
+      .withColumn("struct_col", expr("struct(col_grp, col_ord)"))
+      .select("struct_col")
+
+    val expected: DataFrame = inputDataFrame.select("col_grp", "col_ord")
+
+    val actual = input.flatten("struct_col")
+
+    assertDataFrameEquals(expected, actual)
+  }
+}
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala b/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
index 4a4be37..6fe906c 100644
--- a/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
+++ b/datafu-spark/src/test/scala/datafu/spark/TestSparkUDAFs.scala
@@ -19,19 +19,18 @@
 package datafu.spark
 
 import com.holdenkarau.spark.testing.DataFrameSuiteBase
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.expressions.Window
-import org.apache.spark.sql.types.SparkOverwriteUDAFs
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.Row
 import org.junit.Assert
 import org.junit.runner.RunWith
 import org.scalatest.FunSuite
 import org.scalatest.junit.JUnitRunner
 import org.slf4j.LoggerFactory
+
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
-import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.types._
 
 @RunWith(classOf[JUnitRunner])
 class UdafTests extends FunSuite with DataFrameSuiteBase {
@@ -39,30 +38,37 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
   import spark.implicits._
 
   /**
-   * taken from https://github.com/holdenk/spark-testing-base/issues/234#issuecomment-390150835
-   * 
-   * Solves problem with Hive in Spark 2.3.0 in spark-testing-base
-   */
-  override def conf: SparkConf = super.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
-  
+    * taken from https://github.com/holdenk/spark-testing-base/issues/234#issuecomment-390150835
+    *
+    * Solves problem with Hive in Spark 2.3.0 in spark-testing-base
+    */
+  override def conf: SparkConf =
+    super.conf.set(CATALOG_IMPLEMENTATION.key, "hive")
+
   val logger = LoggerFactory.getLogger(this.getClass)
 
- 	val inputSchema = List(
-      StructField("col_grp", StringType, true),
-      StructField("col_ord", IntegerType, false),
-      StructField("col_str", StringType, true)
+  val inputSchema = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, false),
+    StructField("col_str", StringType, true)
   )
 
-  lazy val inputRDD = sc.parallelize(Seq(Row("a", 1, "asd1"), Row("a", 2, "asd2"), Row("a", 3, "asd3"), Row("b", 1, "asd4")))
-  
-  lazy val df = sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
+  lazy val inputRDD = sc.parallelize(
+    Seq(Row("a", 1, "asd1"),
+        Row("a", 2, "asd2"),
+        Row("a", 3, "asd3"),
+        Row("b", 1, "asd4")))
+
+  lazy val df =
+    sqlContext.createDataFrame(inputRDD, StructType(inputSchema)).cache
 
   case class mapExp(map_col: Map[String, Int])
   case class mapArrExp(map_col: Map[String, Array[String]])
 
   test("test multiset simple") {
     val ms = new SparkUDAFs.MultiSet()
-    val expected : DataFrame = sqlContext.createDataFrame(List(mapExp(Map("b" -> 1, "a" -> 3))))
+    val expected: DataFrame =
+      sqlContext.createDataFrame(List(mapExp(Map("b" -> 1, "a" -> 3))))
     assertDataFrameEquals(expected, df.agg(ms($"col_grp").as("map_col")))
   }
 
@@ -71,23 +77,34 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
   test("test multiarrayset simple") {
     assertDataFrameEquals(
       sqlContext.createDataFrame(List(mapExp(Map("tre" -> 1, "asd" -> 2)))),
-      spark.sql("select array('asd','tre','asd') arr").groupBy().agg(mas($"arr").as("map_col")))
+      spark
+        .sql("select array('asd','tre','asd') arr")
+        .groupBy()
+        .agg(mas($"arr").as("map_col"))
+    )
   }
 
   test("test multiarrayset all nulls") {
     // end case
     spark.sql("drop table if exists mas_table")
     spark.sql("create table mas_table (arr array<string>)")
-    spark.sql("insert overwrite table mas_table select case when 1=2 then array('asd') end from (select 1)z")
-    spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
-    spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
-    spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
-    spark.sql("insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+    spark.sql(
+      "insert overwrite table mas_table select case when 1=2 then array('asd') end " +
+        "from (select 1)z")
+    spark.sql(
+      "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+    spark.sql(
+      "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+    spark.sql(
+      "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
+    spark.sql(
+      "insert into table mas_table select case when 1=2 then array('asd') end from (select 1)z")
 
     val expected = sqlContext.createDataFrame(List(mapExp(Map())))
-    
-    val actual = spark.table("mas_table").groupBy().agg(mas($"arr").as("map_col"))
-    
+
+    val actual =
+      spark.table("mas_table").groupBy().agg(mas($"arr").as("map_col"))
+
     assertDataFrameEquals(expected, actual)
   }
 
@@ -95,12 +112,18 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
     // max keys case
     spark.sql("drop table if exists mas_table2")
     spark.sql("create table mas_table2 (arr array<string>)")
-    spark.sql("insert overwrite table mas_table2 select array('asd','dsa') from (select 1)z")
-    spark.sql("insert into table mas_table2 select array('asd','abc') from (select 1)z")
-    spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
-    spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
-    spark.sql("insert into table mas_table2 select array('asd') from (select 1)z")
-    spark.sql("insert into table mas_table2 select array('asd2') from (select 1)z")
+    spark.sql(
+      "insert overwrite table mas_table2 select array('asd','dsa') from (select 1)z")
+    spark.sql(
+      "insert into table mas_table2 select array('asd','abc') from (select 1)z")
+    spark.sql(
+      "insert into table mas_table2 select array('asd') from (select 1)z")
+    spark.sql(
+      "insert into table mas_table2 select array('asd') from (select 1)z")
+    spark.sql(
+      "insert into table mas_table2 select array('asd') from (select 1)z")
+    spark.sql(
+      "insert into table mas_table2 select array('asd2') from (select 1)z")
 
     val mas2 = new SparkUDAFs.MultiArraySet[String](maxKeys = 2)
 
@@ -116,15 +139,23 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
 
   test("test multiarrayset big input") {
     val N = 100000
-    val blah = spark.sparkContext.parallelize(1 to N, 20).toDF("num").selectExpr("array('asd',concat('dsa',num)) as arr")
+    val blah = spark.sparkContext
+      .parallelize(1 to N, 20)
+      .toDF("num")
+      .selectExpr("array('asd',concat('dsa',num)) as arr")
     val mas = new SparkUDAFs.MultiArraySet[String](maxKeys = 3)
     val time1 = System.currentTimeMillis()
-    val mp = blah.groupBy().agg(mas($"arr")).collect().map(_.getMap[String,Int](0)).head
+    val mp = blah
+      .groupBy()
+      .agg(mas($"arr"))
+      .collect()
+      .map(_.getMap[String, Int](0))
+      .head
     Assert.assertEquals(3, mp.size)
-    Assert.assertEquals("asd", mp.maxBy(_._2)._1 )
-    Assert.assertEquals(N, mp.maxBy(_._2)._2 )
+    Assert.assertEquals("asd", mp.maxBy(_._2)._1)
+    Assert.assertEquals(N, mp.maxBy(_._2)._2)
     val time2 = System.currentTimeMillis()
-    logger.info("time took: " + (time2-time1)/1000 + " secs")
+    logger.info("time took: " + (time2 - time1) / 1000 + " secs")
   }
 
   test("test mapmerge") {
@@ -132,72 +163,99 @@ class UdafTests extends FunSuite with DataFrameSuiteBase {
 
     spark.sql("drop table if exists mapmerge_table")
     spark.sql("create table mapmerge_table (c map<string, array<string>>)")
-    spark.sql("insert overwrite table mapmerge_table select map('k1', array('v1')) from (select 1) z")
-    spark.sql("insert into table mapmerge_table select map('k1', array('v1')) from (select 1) z")
-    spark.sql("insert into table mapmerge_table select map('k2', array('v3')) from (select 1) z")
+    spark.sql(
+      "insert overwrite table mapmerge_table select map('k1', array('v1')) from (select 1) z")
+    spark.sql(
+      "insert into table mapmerge_table select map('k1', array('v1')) from (select 1) z")
+    spark.sql(
+      "insert into table mapmerge_table select map('k2', array('v3')) from (select 1) z")
 
     assertDataFrameEquals(
-      sqlContext.createDataFrame(List(mapArrExp(Map("k1" -> Array("v1"), "k2" -> Array("v3"))))),
-      spark.table("mapmerge_table").groupBy().agg(mapMerge($"c").as("map_col")))
+      sqlContext.createDataFrame(
+        List(mapArrExp(Map("k1" -> Array("v1"), "k2" -> Array("v3"))))),
+      spark.table("mapmerge_table").groupBy().agg(mapMerge($"c").as("map_col"))
+    )
   }
 
   test("minKeyValue") {
     assertDataFrameEquals(
       sqlContext.createDataFrame(List(("b", "asd4"), ("a", "asd1"))),
-      df.groupBy($"col_grp".as("_1")).agg(SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").as("_2")))
+      df.groupBy($"col_grp".as("_1"))
+        .agg(SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").as("_2"))
+    )
   }
 
-  case class exp4(col_grp: String, col_ord: Int, col_str: String, asd: String)
+  case class Exp4(colGrp: String, colOrd: Int, colStr: String, asd: String)
 
-    val minKeyValueWindowExpectedSchema = List(
-      StructField("col_grp",StringType,true),
-      StructField("col_ord",IntegerType,false),
-      StructField("col_str",StringType,true),
-      StructField("asd",StringType,true)
-    )
+  val minKeyValueWindowExpectedSchema = List(
+    StructField("col_grp", StringType, true),
+    StructField("col_ord", IntegerType, false),
+    StructField("col_str", StringType, true),
+    StructField("asd", StringType, true)
+  )
 
   test("minKeyValue window") {
     assertDataFrameEquals(
-      sqlContext.createDataFrame(sc.parallelize(Seq(
-        Row("b", 1, "asd4", "asd4"),
-        Row("a", 1, "asd1", "asd1"),
-        Row("a", 2, "asd2", "asd1"),
-        Row("a", 3, "asd3", "asd1")
-      )), StructType(minKeyValueWindowExpectedSchema)),
-      df.withColumn("asd", SparkOverwriteUDAFs.minValueByKey($"col_ord", $"col_str").over(Window.partitionBy("col_grp"))))
+      sqlContext.createDataFrame(
+        sc.parallelize(
+          Seq(
+            Row("b", 1, "asd4", "asd4"),
+            Row("a", 1, "asd1", "asd1"),
+            Row("a", 2, "asd2", "asd1"),
+            Row("a", 3, "asd3", "asd1")
+          )),
+        StructType(minKeyValueWindowExpectedSchema)
+      ),
+      df.withColumn("asd",
+                    SparkOverwriteUDAFs
+                      .minValueByKey($"col_ord", $"col_str")
+                      .over(Window.partitionBy("col_grp")))
+    )
   }
 
-  case class exp5(col_grp: String, col_ord: Option[Int])
-  
+  case class Exp5(colGrp: String, colOrd: Option[Int])
+
   test("countDistinctUpTo") {
     import datafu.spark.SparkUDAFs.CountDistinctUpTo
-    
+
     val countDistinctUpTo3 = new CountDistinctUpTo(3)
     val countDistinctUpTo6 = new CountDistinctUpTo(6)
-    
-    val inputDF = sqlContext.createDataFrame(List(
-        exp5("b", Option(1)),
-        exp5("a", Option(1)),
-        exp5("a", Option(2)),
-        exp5("a", Option(3)),
-        exp5("a", Option(4))
+
+    val inputDF = sqlContext.createDataFrame(
+      List(
+        Exp5("b", Option(1)),
+        Exp5("a", Option(1)),
+        Exp5("a", Option(2)),
+        Exp5("a", Option(3)),
+        Exp5("a", Option(4))
       ))
 
-    val results3DF = sqlContext.createDataFrame(List(
-        exp5("b", Option(1)),
-        exp5("a", Option(3))
-    ))
-
-    val results6DF = sqlContext.createDataFrame(List(
-        exp5("b", Option(1)),
-        exp5("a", Option(4))
-    ))
-    
-    inputDF.groupBy("col_grp").agg(countDistinctUpTo3($"col_ord").as("col_ord")).show
-    
-    assertDataFrameEquals(results3DF, inputDF.groupBy("col_grp").agg(countDistinctUpTo3($"col_ord").as("col_ord")))
-    
-    assertDataFrameEquals(results6DF, inputDF.groupBy("col_grp").agg(countDistinctUpTo6($"col_ord").as("col_ord")))
+    val results3DF = sqlContext.createDataFrame(
+      List(
+        Exp5("b", Option(1)),
+        Exp5("a", Option(3))
+      ))
+
+    val results6DF = sqlContext.createDataFrame(
+      List(
+        Exp5("b", Option(1)),
+        Exp5("a", Option(4))
+      ))
+
+    inputDF
+      .groupBy("col_grp")
+      .agg(countDistinctUpTo3($"col_ord").as("col_ord"))
+      .show
+
+    assertDataFrameEquals(results3DF,
+                          inputDF
+                            .groupBy("col_grp")
+                            .agg(countDistinctUpTo3($"col_ord").as("col_ord")))
+
+    assertDataFrameEquals(results6DF,
+                          inputDF
+                            .groupBy("col_grp")
+                            .agg(countDistinctUpTo6($"col_ord").as("col_ord")))
   }
-  
+
 }