You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/21 21:55:55 UTC

[2/3] incubator-spot git commit: SPOT-166: Missing scaladocs were added. Unit testing added to improve coverage, current lines covered 84%.

SPOT-166: Missing scaladocs were added.
Unit testing added to improve coverage, current lines covered 84%.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/53d00d5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/53d00d5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/53d00d5c

Branch: refs/heads/master
Commit: 53d00d5ccf71f55ac8d69e0c38be098eb23f2deb
Parents: 2ef6d06
Author: Ricardo Barona <ri...@intel.com>
Authored: Wed Jun 21 13:10:09 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Wed Jun 21 13:10:09 2017 -0500

----------------------------------------------------------------------
 .../scala/org/apache/spark/sql/WideUDFs.scala   | 16 +++++
 .../org/apache/spot/lda/SpotLDAWrapper.scala    | 40 ++++++++++++
 .../apache/spot/lda/SpotLDAWrapperSchema.scala  |  3 +
 .../FlowSuspiciousConnectsAnalysis.scala        | 13 ++++
 .../apache/spot/proxy/ProxyWordCreation.scala   | 43 +++++++++++--
 .../org/apache/spot/utilities/TopDomains.scala  |  4 +-
 .../utilities/data/InputOutputDataHandler.scala |  7 +++
 .../SuspiciousConnectsArgumentParserTest.scala  | 64 ++++++++++++++++++++
 .../dns/DNSSuspiciousConnectsAnalysisTest.scala | 15 +++--
 .../dns/DNSSuspiciousConnectsModelTest.scala    | 40 ++++++++++++
 .../FlowSuspiciousConnectsAnalysisTest.scala    | 15 ++---
 .../ProxySuspiciousConnectsAnalysisTest.scala   | 15 +++--
 12 files changed, 250 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala b/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
index c5e4dec..c60db0d 100644
--- a/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
+++ b/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
@@ -23,8 +23,17 @@ import scala.language.implicitConversions
 import scala.reflect.runtime.universe.{TypeTag, typeTag}
 import scala.util.Try
 
+/**
+  * Defines custom udf implementations to accept 10 to 12 parameters.
+  */
 object WideUDFs {
 
+  /**
+    * Defines a user-defined function of 10 arguments as user-defined function (UDF).
+    * The data types are automatically inferred based on the function's signature.
+    *
+    * @group udf_funcs
+    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag,
   A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]):
   UserDefinedFunction = {
@@ -66,6 +75,13 @@ object WideUDFs {
       :: Nil).getOrElse(Nil)
     UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Option(inputTypes))
   }
+
+  /**
+    * Defines a user-defined function of 12 arguments as user-defined function (UDF).
+    * The data types are automatically inferred based on the function's signature.
+    *
+    * @group udf_funcs
+    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag,
   A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]):
   UserDefinedFunction = {

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
index 7697acc..2d62817 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
@@ -39,6 +39,21 @@ import scala.collection.immutable.Map
 
 object SpotLDAWrapper {
 
+  /**
+    * Runs Spark LDA and returns a new model.
+    *
+    * @param sparkSession       the SparkSession
+    * @param docWordCount       RDD with document list and the word count for each document (corpus)
+    * @param topicCount         number of topics to find
+    * @param logger             application logger
+    * @param ldaSeed            LDA seed
+    * @param ldaAlpha           LDA alpha parameter
+    * @param ldaBeta            LDA beta parameter
+    * @param ldaOptimizerOption LDA optimizer, em or online
+    * @param maxIterations      number of maximum iteration LDA will run
+    * @param precisionUtility   FloatPointPrecisionUtility implementation based on user configuration (64 or 32 bit)
+    * @return
+    */
   def runLDA(sparkSession: SparkSession,
              docWordCount: RDD[SpotLDAInput],
              topicCount: Int,
@@ -165,6 +180,15 @@ object SpotLDAWrapper {
     SpotLDAOutput(docToTopicMixDF, wordResults)
   }
 
+  /**
+    * Formats input data for LDA algorithm
+    *
+    * @param docWordCount       RDD with document list and the word count for each document (corpus)
+    * @param documentDictionary DataFrame with a distinct list of documents and its id
+    * @param wordDictionary     immutable Map with distinct list of word and its id
+    * @param sparkSession       the SparkSession
+    * @return
+    */
   def formatSparkLDAInput(docWordCount: RDD[SpotLDAInput],
                           documentDictionary: DataFrame,
                           wordDictionary: Map[String, Int],
@@ -203,6 +227,13 @@ object SpotLDAWrapper {
     ldaInput
   }
 
+  /**
+    * Format LDA output topicMatrix for spot-ml scoring
+    *
+    * @param wordTopMat LDA model topicMatrix
+    * @param wordMap    immutable Map with distinct list of word and its id
+    * @return
+    */
   def formatSparkLDAWordOutput(wordTopMat: Matrix, wordMap: Map[Int, String]): scala.Predef.Map[String, Array[Double]] = {
 
     // incoming word top matrix is in column-major order and the columns are unnormalized
@@ -216,6 +247,15 @@ object SpotLDAWrapper {
     wordProbs.zipWithIndex.map({ case (topicProbs, wordInd) => (wordMap(wordInd), topicProbs) }).toMap
   }
 
+  /**
+    * Format LDA output topicDistribution for spot-ml scoring
+    *
+    * @param docTopDist         LDA model topicDistribution
+    * @param documentDictionary DataFrame with a distinct list of documents and its id
+    * @param sparkSession       the SparkSession
+    * @param precisionUtility   FloatPointPrecisionUtility implementation based on user configuration (64 or 32 bit)
+    * @return
+    */
   def formatSparkLDADocTopicOutput(docTopDist: RDD[(Long, Vector)], documentDictionary: DataFrame, sparkSession: SparkSession,
                                    precisionUtility: FloatPointPrecisionUtility):
   DataFrame = {

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
index 2a09f7c..2862fb8 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
@@ -19,6 +19,9 @@ package org.apache.spot.lda
 
 import org.apache.spark.sql.types.{LongType, StringType, StructField}
 
+/**
+  * Schemas and column names used in SpotLDAWrapper and *SuspiciousConnectsModel.
+  */
 object SpotLDAWrapperSchema {
 
   // modelDF columns

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
index b369ac1..b6fe791 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
@@ -73,6 +73,15 @@ object FlowSuspiciousConnectsAnalysis {
       ObytField,
       ScoreField)).fieldNames.map(col)
 
+  /**
+    * Runs suspicious connects analysis for Flow records
+    *
+    * @param config           spot-ml configuration
+    * @param sparkSession     the SparkSession
+    * @param logger           application logger
+    * @param inputFlowRecords input Flow records for analysis
+    * @return
+    */
   def run(config: SuspiciousConnectsConfig, sparkSession: SparkSession, logger: Logger,
           inputFlowRecords: DataFrame): SuspiciousConnectsAnalysisResults = {
 
@@ -103,6 +112,8 @@ object FlowSuspiciousConnectsAnalysis {
   }
 
   /**
+    * Apply a filter to raw data (Flow records) and returns only those that are good for word creation and that can be
+    * passed to LDA
     *
     * @param inputFlowRecords raw flow records
     * @return
@@ -125,6 +136,7 @@ object FlowSuspiciousConnectsAnalysis {
   }
 
   /**
+    * Gets invalid records based on required fields validations
     *
     * @param inputFlowRecords raw flow records.
     * @return
@@ -151,6 +163,7 @@ object FlowSuspiciousConnectsAnalysis {
   }
 
   /**
+    * Gets the records with a score equal or below the threshold
     *
     * @param scoredFlowRecords scored flow records.
     * @param threshold         score tolerance.

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
index 7a1d94e..0265d83 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
@@ -25,10 +25,19 @@ import org.apache.spot.utilities.data.validation.InvalidDataHandler
 
 import scala.util.{Success, Try}
 
-
+/**
+  * Convert Proxy log entries into "words" for topic modelling analyses.
+  */
 object ProxyWordCreation {
 
-  def udfWordCreation(topDomains : Broadcast[Set[String]],
+  /**
+    * UDF for word creation
+    *
+    * @param topDomains  List of most popular top level domain names (provided)
+    * @param agentCounts List of user agent values in the data set and its count
+    * @return
+    */
+  def udfWordCreation(topDomains: Broadcast[Set[String]],
                       agentCounts: Broadcast[Map[String, Long]]) =
     udf((host: String, time: String, reqMethod: String, uri: String, contentType: String, userAgent: String, responseCode: String) =>
       ProxyWordCreation.proxyWord(host,
@@ -41,7 +50,20 @@ object ProxyWordCreation {
         topDomains,
         agentCounts))
 
-
+  /**
+    * Creates a word based on values of Proxy record
+    *
+    * @param proxyHost    Host name
+    * @param time         Proxy connection time
+    * @param reqMethod    request method
+    * @param uri          URI
+    * @param contentType  content type
+    * @param userAgent    user agent
+    * @param responseCode response code
+    * @param topDomains   top domains
+    * @param agentCounts  agent counts
+    * @return
+    */
   def proxyWord(proxyHost: String,
                 time: String,
                 reqMethod: String,
@@ -51,7 +73,7 @@ object ProxyWordCreation {
                 responseCode: String,
                 topDomains: Broadcast[Set[String]],
                 agentCounts: Broadcast[Map[String, Long]]): String = {
-    Try{
+    Try {
       List(topDomain(proxyHost, topDomains.value).toString,
         // Time binned by hours
         TimeUtilities.getTimeAsHour(time).toString,
@@ -73,6 +95,13 @@ object ProxyWordCreation {
   }
 
 
+  /**
+    * Classifies proxy host domain based on popular domains
+    *
+    * @param proxyHost  host name
+    * @param topDomains list of top domains
+    * @return
+    */
   def topDomain(proxyHost: String, topDomains: Set[String]): Int = {
 
     val domain = DomainProcessor.extractDomain(proxyHost)
@@ -86,6 +115,12 @@ object ProxyWordCreation {
     }
   }
 
+  /**
+    * Defines if a domain is safe or not based on a known domain
+    *
+    * @param domain domain name
+    * @return
+    */
   def domainBelongsToSafeList(domain: String) = domain == "intel" // TBD parameterize this!
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
index 089a121..b9592ea 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
@@ -19,7 +19,9 @@ package org.apache.spot.utilities
 
 import scala.io.Source
 
-
+/**
+  * List of top domains used for DNS and Proxy analysis.
+  */
 object TopDomains {
   val alexaTop1MPath = "top-1m.csv"
 

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
index ede46c2..481e389 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
@@ -77,6 +77,13 @@ object InputOutputDataHandler {
       s"Please check for SuspiciousConnects or Spark logs to see if there were errors.")
   }
 
+  /**
+    *
+    * @param sparkSession      the Spark Session
+    * @param hdfsScoredConnect HDFS output folder. The location where results were saved; flow, dns or proxy.
+    * @param analysis          Data type to analyze
+    * @param logger            application logger
+    */
   def mergeResultsFileSystem(sparkSession: SparkSession,
                              hdfsScoredConnect: String,
                              analysis: String,

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
new file mode 100644
index 0000000..f61b5ed
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
@@ -0,0 +1,64 @@
+package org.apache.spot
+
+import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
+import org.scalatest.{FlatSpec, Matchers}
+
+class SuspiciousConnectsArgumentParserTest extends FlatSpec with Matchers {
+
+  "Argument parser" should "parse parameters correctly" in {
+
+    val args = Array("--analysis", "dns",
+      "--input", "user/spot-data",
+      "--dupfactor", "1000",
+      "--feedback", "/home/user/ml/dns/test/dns_scores.csv",
+      "--ldatopiccount", "10",
+      "--scored", "/user/user/dns/test/scored_results/scores",
+      "--threshold", "1.1",
+      "--maxresults", "20",
+      "--ldamaxiterations", "20",
+      "--ldaalpha", "0.0009",
+      "--ldabeta", "0.00001",
+      "--ldaoptimizer", "online",
+      "--precision", "64",
+      "--userdomain", "mycompany")
+
+    val parser = SuspiciousConnectsArgumentParser.parser
+    val config = parser.parse(args, SuspiciousConnectsConfig()) match {
+      case Some(config) => config
+      case None => SuspiciousConnectsConfig()
+    }
+
+    config.analysis shouldBe "dns"
+    config.duplicationFactor shouldBe 1000
+    config.topicCount shouldBe 10
+    config.ldaAlpha shouldBe 0.0009
+    config.ldaBeta shouldBe 0.00001
+    config.ldaOptimizer shouldBe "online"
+
+  }
+
+  it should "parse parameters and take defaults when optional parameters are not provided" in {
+    val args = Array("--analysis", "dns",
+      "--input", "user/spot-data",
+      "--feedback", "/home/user/ml/dns/test/dns_scores.csv",
+      "--ldatopiccount", "10",
+      "--scored", "/user/user/dns/test/scored_results/scores",
+      "--threshold", "1.1",
+      "--maxresults", "20",
+      "--userdomain", "mycompany")
+
+    val parser = SuspiciousConnectsArgumentParser.parser
+    val config = parser.parse(args, SuspiciousConnectsConfig()) match {
+      case Some(config) => config
+      case None => SuspiciousConnectsConfig()
+    }
+
+    config.analysis shouldBe "dns"
+    config.duplicationFactor shouldBe 1
+    config.topicCount shouldBe 10
+    config.ldaAlpha shouldBe 1.02
+    config.ldaBeta shouldBe 1.001
+    config.ldaOptimizer shouldBe "em"
+    config.outputDelimiter shouldBe "\t"
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index c34e7c8..5ceccaa 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -21,6 +21,7 @@ package org.apache.spot.dns
 import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types.StructType
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
 import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
 import org.apache.spot.dns.DNSSchema._
 import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
@@ -47,7 +48,7 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
     topicCount = 20,
     hdfsScoredConnect = "",
     threshold = 1.0d,
-    maxResults = 1000,
+    maxResults = -1,
     outputDelimiter = "\t",
     ldaPRGSeed = None,
     ldaMaxiterations = 20,
@@ -97,8 +98,10 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
     val anomalousRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 1, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
     val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
-    val model = DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfig, data)
-    val scoredData = model.score(sparkSession, data, emTestConfig.userDomain, emTestConfig.precisionUtility)
+
+    val SuspiciousConnectsAnalysisResults(scoredData, _) =
+      DNSSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+
     val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
 
@@ -119,8 +122,10 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
     val anomalousRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 1, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
     val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
-    val model = DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfig, data)
-    val scoredData = model.score(sparkSession, data, onlineTestConfig.userDomain, onlineTestConfig.precisionUtility)
+
+    val SuspiciousConnectsAnalysisResults(scoredData, _) = DNSSuspiciousConnectsAnalysis.run(onlineTestConfig,
+      sparkSession, logger, data)
+
     val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
 

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
new file mode 100644
index 0000000..f5a2d79
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
@@ -0,0 +1,40 @@
+package org.apache.spot.dns
+
+import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.apache.spot.utilities.{CountryCodes, TopDomains}
+import org.scalatest.Matchers
+
+class DNSSuspiciousConnectsModelTest extends TestingSparkContextFlatSpec with Matchers {
+
+  "createTempFields" should "create DNS stats based on an URL" in {
+    val countryCodesBC = sparkSession.sparkContext.broadcast(CountryCodes.CountryCodes)
+    val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
+    val userDomain = "intel.com"
+    val url = "iot.intel.com.mx"
+
+
+    val result = DNSSuspiciousConnectsModel.createTempFields(countryCodesBC, topDomainsBC, userDomain, url)
+
+    result.subdomainEntropy shouldBe 1.5849625007211559
+    result.subdomainLength shouldBe 3
+    result.topDomainClass shouldBe 1
+    result.numPeriods shouldBe 4
+
+  }
+
+  it should "create DNS stats based on an URL completely different to userDomain" in {
+    val countryCodesBC = sparkSession.sparkContext.broadcast(CountryCodes.CountryCodes)
+    val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
+    val userDomain = "anothercompany.com"
+    val url = "maps.company.com"
+
+    val result = DNSSuspiciousConnectsModel.createTempFields(countryCodesBC, topDomainsBC, userDomain, url)
+
+    result.subdomainEntropy shouldBe 2.0
+    result.subdomainLength shouldBe 4
+    result.topDomainClass shouldBe 1
+    result.numPeriods shouldBe 3
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index 296e073..9310390 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -20,6 +20,7 @@ package org.apache.spot.netflow
 import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
 import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
 import org.apache.spot.netflow.FlowSchema._
 import org.apache.spot.netflow.FlowSuspiciousConnectsAnalysis.InSchema
@@ -40,7 +41,7 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
     topicCount = 20,
     hdfsScoredConnect = "",
     threshold = 1.0d,
-    maxResults = 1000,
+    maxResults = -1,
     outputDelimiter = "\t",
     ldaPRGSeed = None,
     ldaMaxiterations = 20,
@@ -91,10 +92,8 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
       typicalRecord, typicalRecord, typicalRecord, typicalRecord))
 
-    val model =
-      FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfig, data)
-
-    val scoredData = model.score(sparkSession, data, emTestConfig.precisionUtility)
+    val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(emTestConfig,
+      sparkSession, logger, data)
 
     val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
@@ -125,10 +124,8 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
       typicalRecord, typicalRecord, typicalRecord, typicalRecord))
 
-    val model =
-      FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfig, data)
-
-    val scoredData = model.score(sparkSession, data, onlineTestConfig.precisionUtility)
+    val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(onlineTestConfig,
+      sparkSession, logger, data)
 
     val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index 1b1fc0e..45dc2df 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -20,6 +20,7 @@ package org.apache.spot.proxy
 import org.apache.log4j.{Level, LogManager}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types.StructType
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
 import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
 import org.apache.spot.proxy.ProxySchema._
 import org.apache.spot.testutils.TestingSparkContextFlatSpec
@@ -56,7 +57,7 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
     topicCount = 20,
     hdfsScoredConnect = "",
     threshold = 1.0d,
-    maxResults = 1000,
+    maxResults = -1,
     outputDelimiter = "\t",
     ldaPRGSeed = None,
     ldaMaxiterations = 20,
@@ -106,9 +107,8 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
       typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
 
-    val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfigProxy, data)
-
-    val scoredData = model.score(sparkSession, data, emTestConfigProxy.precisionUtility)
+    val SuspiciousConnectsAnalysisResults(scoredData, _) = ProxySuspiciousConnectsAnalysis.run(emTestConfigProxy,
+      sparkSession, logger, data)
 
     val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))
@@ -150,9 +150,12 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
     val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
       typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
 
-    val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfigProxy, data)
+    //    val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfigProxy, data)
+    //
+    //    val scoredData = model.score(sparkSession, data, onlineTestConfigProxy.precisionUtility)
 
-    val scoredData = model.score(sparkSession, data, onlineTestConfigProxy.precisionUtility)
+    val SuspiciousConnectsAnalysisResults(scoredData, _) =
+      ProxySuspiciousConnectsAnalysis.run(onlineTestConfigProxy, sparkSession, logger, data)
 
     val anomalyScore = scoredData.filter(scoredData(Host) ===  "intel.com").first().getAs[Double](Score)
     val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))