You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/21 21:55:55 UTC
[2/3] incubator-spot git commit: SPOT-166: Missing scaladocs were
added. Unit testing added to improve coverage, current lines covered 84%.
SPOT-166: Missing scaladocs were added.
Unit testing added to improve coverage, current lines covered 84%.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/53d00d5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/53d00d5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/53d00d5c
Branch: refs/heads/master
Commit: 53d00d5ccf71f55ac8d69e0c38be098eb23f2deb
Parents: 2ef6d06
Author: Ricardo Barona <ri...@intel.com>
Authored: Wed Jun 21 13:10:09 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Wed Jun 21 13:10:09 2017 -0500
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/WideUDFs.scala | 16 +++++
.../org/apache/spot/lda/SpotLDAWrapper.scala | 40 ++++++++++++
.../apache/spot/lda/SpotLDAWrapperSchema.scala | 3 +
.../FlowSuspiciousConnectsAnalysis.scala | 13 ++++
.../apache/spot/proxy/ProxyWordCreation.scala | 43 +++++++++++--
.../org/apache/spot/utilities/TopDomains.scala | 4 +-
.../utilities/data/InputOutputDataHandler.scala | 7 +++
.../SuspiciousConnectsArgumentParserTest.scala | 64 ++++++++++++++++++++
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 15 +++--
.../dns/DNSSuspiciousConnectsModelTest.scala | 40 ++++++++++++
.../FlowSuspiciousConnectsAnalysisTest.scala | 15 ++---
.../ProxySuspiciousConnectsAnalysisTest.scala | 15 +++--
12 files changed, 250 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala b/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
index c5e4dec..c60db0d 100644
--- a/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
+++ b/spot-ml/src/main/scala/org/apache/spark/sql/WideUDFs.scala
@@ -23,8 +23,17 @@ import scala.language.implicitConversions
import scala.reflect.runtime.universe.{TypeTag, typeTag}
import scala.util.Try
+/**
+ * Defines custom udf implementations to accept 10 to 12 parameters.
+ */
object WideUDFs {
+ /**
+ * Defines a user-defined function of 10 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
+ *
+ * @group udf_funcs
+ */
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag,
A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]):
UserDefinedFunction = {
@@ -66,6 +75,13 @@ object WideUDFs {
:: Nil).getOrElse(Nil)
UserDefinedFunction(f, ScalaReflection.schemaFor(typeTag[RT]).dataType, Option(inputTypes))
}
+
+ /**
+ * Defines a user-defined function of 12 arguments as user-defined function (UDF).
+ * The data types are automatically inferred based on the function's signature.
+ *
+ * @group udf_funcs
+ */
def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag,
A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](f: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]):
UserDefinedFunction = {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
index 7697acc..2d62817 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
@@ -39,6 +39,21 @@ import scala.collection.immutable.Map
object SpotLDAWrapper {
+ /**
+ * Runs Spark LDA and returns a new model.
+ *
+ * @param sparkSession the SparkSession
+ * @param docWordCount RDD with document list and the word count for each document (corpus)
+ * @param topicCount number of topics to find
+ * @param logger application logger
+ * @param ldaSeed LDA seed
+ * @param ldaAlpha LDA alpha parameter
+ * @param ldaBeta LDA beta parameter
+ * @param ldaOptimizerOption LDA optimizer, em or online
+ * @param maxIterations number of maximum iteration LDA will run
+ * @param precisionUtility FloatPointPrecisionUtility implementation based on user configuration (64 or 32 bit)
+ * @return
+ */
def runLDA(sparkSession: SparkSession,
docWordCount: RDD[SpotLDAInput],
topicCount: Int,
@@ -165,6 +180,15 @@ object SpotLDAWrapper {
SpotLDAOutput(docToTopicMixDF, wordResults)
}
+ /**
+ * Formats input data for LDA algorithm
+ *
+ * @param docWordCount RDD with document list and the word count for each document (corpus)
+ * @param documentDictionary DataFrame with a distinct list of documents and its id
+ * @param wordDictionary immutable Map with distinct list of word and its id
+ * @param sparkSession the SparkSession
+ * @return
+ */
def formatSparkLDAInput(docWordCount: RDD[SpotLDAInput],
documentDictionary: DataFrame,
wordDictionary: Map[String, Int],
@@ -203,6 +227,13 @@ object SpotLDAWrapper {
ldaInput
}
+ /**
+ * Format LDA output topicMatrix for spot-ml scoring
+ *
+ * @param wordTopMat LDA model topicMatrix
+ * @param wordMap immutable Map with distinct list of word and its id
+ * @return
+ */
def formatSparkLDAWordOutput(wordTopMat: Matrix, wordMap: Map[Int, String]): scala.Predef.Map[String, Array[Double]] = {
// incoming word top matrix is in column-major order and the columns are unnormalized
@@ -216,6 +247,15 @@ object SpotLDAWrapper {
wordProbs.zipWithIndex.map({ case (topicProbs, wordInd) => (wordMap(wordInd), topicProbs) }).toMap
}
+ /**
+ * Format LDA output topicDistribution for spot-ml scoring
+ *
+ * @param docTopDist LDA model topicDistribution
+ * @param documentDictionary DataFrame with a distinct list of documents and its id
+ * @param sparkSession the SparkSession
+ * @param precisionUtility FloatPointPrecisionUtility implementation based on user configuration (64 or 32 bit)
+ * @return
+ */
def formatSparkLDADocTopicOutput(docTopDist: RDD[(Long, Vector)], documentDictionary: DataFrame, sparkSession: SparkSession,
precisionUtility: FloatPointPrecisionUtility):
DataFrame = {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
index 2a09f7c..2862fb8 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapperSchema.scala
@@ -19,6 +19,9 @@ package org.apache.spot.lda
import org.apache.spark.sql.types.{LongType, StringType, StructField}
+/**
+ * Schemas and column names used in SpotLDAWrapper and *SuspiciousConnectsModel.
+ */
object SpotLDAWrapperSchema {
// modelDF columns
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
index b369ac1..b6fe791 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
@@ -73,6 +73,15 @@ object FlowSuspiciousConnectsAnalysis {
ObytField,
ScoreField)).fieldNames.map(col)
+ /**
+ * Runs suspicious connects analysis for Flow records
+ *
+ * @param config spot-ml configuration
+ * @param sparkSession the SparkSession
+ * @param logger application logger
+ * @param inputFlowRecords input Flow records for analysis
+ * @return
+ */
def run(config: SuspiciousConnectsConfig, sparkSession: SparkSession, logger: Logger,
inputFlowRecords: DataFrame): SuspiciousConnectsAnalysisResults = {
@@ -103,6 +112,8 @@ object FlowSuspiciousConnectsAnalysis {
}
/**
+ * Apply a filter to raw data (Flow records) and returns only those that are good for word creation and that can be
+ * passed to LDA
*
* @param inputFlowRecords raw flow records
* @return
@@ -125,6 +136,7 @@ object FlowSuspiciousConnectsAnalysis {
}
/**
+ * Gets invalid records based on required fields validations
*
* @param inputFlowRecords raw flow records.
* @return
@@ -151,6 +163,7 @@ object FlowSuspiciousConnectsAnalysis {
}
/**
+ * Gets the records with a score equal or below the threshold
*
* @param scoredFlowRecords scored flow records.
* @param threshold score tolerance.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
index 7a1d94e..0265d83 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyWordCreation.scala
@@ -25,10 +25,19 @@ import org.apache.spot.utilities.data.validation.InvalidDataHandler
import scala.util.{Success, Try}
-
+/**
+ * Convert Proxy log entries into "words" for topic modelling analyses.
+ */
object ProxyWordCreation {
- def udfWordCreation(topDomains : Broadcast[Set[String]],
+ /**
+ * UDF for word creation
+ *
+ * @param topDomains List of most popular top level domain names (provided)
+ * @param agentCounts List of user agent values in the data set and its count
+ * @return
+ */
+ def udfWordCreation(topDomains: Broadcast[Set[String]],
agentCounts: Broadcast[Map[String, Long]]) =
udf((host: String, time: String, reqMethod: String, uri: String, contentType: String, userAgent: String, responseCode: String) =>
ProxyWordCreation.proxyWord(host,
@@ -41,7 +50,20 @@ object ProxyWordCreation {
topDomains,
agentCounts))
-
+ /**
+ * Creates a word based on values of Proxy record
+ *
+ * @param proxyHost Host name
+ * @param time Proxy connection time
+ * @param reqMethod request method
+ * @param uri URI
+ * @param contentType content type
+ * @param userAgent user agent
+ * @param responseCode response code
+ * @param topDomains top domains
+ * @param agentCounts agent counts
+ * @return
+ */
def proxyWord(proxyHost: String,
time: String,
reqMethod: String,
@@ -51,7 +73,7 @@ object ProxyWordCreation {
responseCode: String,
topDomains: Broadcast[Set[String]],
agentCounts: Broadcast[Map[String, Long]]): String = {
- Try{
+ Try {
List(topDomain(proxyHost, topDomains.value).toString,
// Time binned by hours
TimeUtilities.getTimeAsHour(time).toString,
@@ -73,6 +95,13 @@ object ProxyWordCreation {
}
+ /**
+ * Classifies proxy host domain based on popular domains
+ *
+ * @param proxyHost host name
+ * @param topDomains list of top domains
+ * @return
+ */
def topDomain(proxyHost: String, topDomains: Set[String]): Int = {
val domain = DomainProcessor.extractDomain(proxyHost)
@@ -86,6 +115,12 @@ object ProxyWordCreation {
}
}
+ /**
+ * Defines if a domain is safe or not based on a known domain
+ *
+ * @param domain domain name
+ * @return
+ */
def domainBelongsToSafeList(domain: String) = domain == "intel" // TBD parameterize this!
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
index 089a121..b9592ea 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
@@ -19,7 +19,9 @@ package org.apache.spot.utilities
import scala.io.Source
-
+/**
+ * List of top domains used for DNS and Proxy analysis.
+ */
object TopDomains {
val alexaTop1MPath = "top-1m.csv"
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
index ede46c2..481e389 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
@@ -77,6 +77,13 @@ object InputOutputDataHandler {
s"Please check for SuspiciousConnects or Spark logs to see if there were errors.")
}
+ /**
+ *
+ * @param sparkSession the Spark Session
+ * @param hdfsScoredConnect HDFS output folder. The location where results were saved; flow, dns or proxy.
+ * @param analysis Data type to analyze
+ * @param logger application logger
+ */
def mergeResultsFileSystem(sparkSession: SparkSession,
hdfsScoredConnect: String,
analysis: String,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
new file mode 100644
index 0000000..f61b5ed
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
@@ -0,0 +1,64 @@
+package org.apache.spot
+
+import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
+import org.scalatest.{FlatSpec, Matchers}
+
+class SuspiciousConnectsArgumentParserTest extends FlatSpec with Matchers {
+
+ "Argument parser" should "parse parameters correctly" in {
+
+ val args = Array("--analysis", "dns",
+ "--input", "user/spot-data",
+ "--dupfactor", "1000",
+ "--feedback", "/home/user/ml/dns/test/dns_scores.csv",
+ "--ldatopiccount", "10",
+ "--scored", "/user/user/dns/test/scored_results/scores",
+ "--threshold", "1.1",
+ "--maxresults", "20",
+ "--ldamaxiterations", "20",
+ "--ldaalpha", "0.0009",
+ "--ldabeta", "0.00001",
+ "--ldaoptimizer", "online",
+ "--precision", "64",
+ "--userdomain", "mycompany")
+
+ val parser = SuspiciousConnectsArgumentParser.parser
+ val config = parser.parse(args, SuspiciousConnectsConfig()) match {
+ case Some(config) => config
+ case None => SuspiciousConnectsConfig()
+ }
+
+ config.analysis shouldBe "dns"
+ config.duplicationFactor shouldBe 1000
+ config.topicCount shouldBe 10
+ config.ldaAlpha shouldBe 0.0009
+ config.ldaBeta shouldBe 0.00001
+ config.ldaOptimizer shouldBe "online"
+
+ }
+
+ it should "parse parameters and take defaults when optional parameters are not provided" in {
+ val args = Array("--analysis", "dns",
+ "--input", "user/spot-data",
+ "--feedback", "/home/user/ml/dns/test/dns_scores.csv",
+ "--ldatopiccount", "10",
+ "--scored", "/user/user/dns/test/scored_results/scores",
+ "--threshold", "1.1",
+ "--maxresults", "20",
+ "--userdomain", "mycompany")
+
+ val parser = SuspiciousConnectsArgumentParser.parser
+ val config = parser.parse(args, SuspiciousConnectsConfig()) match {
+ case Some(config) => config
+ case None => SuspiciousConnectsConfig()
+ }
+
+ config.analysis shouldBe "dns"
+ config.duplicationFactor shouldBe 1
+ config.topicCount shouldBe 10
+ config.ldaAlpha shouldBe 1.02
+ config.ldaBeta shouldBe 1.001
+ config.ldaOptimizer shouldBe "em"
+ config.outputDelimiter shouldBe "\t"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index c34e7c8..5ceccaa 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -21,6 +21,7 @@ package org.apache.spot.dns
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.dns.DNSSchema._
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
@@ -47,7 +48,7 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
topicCount = 20,
hdfsScoredConnect = "",
threshold = 1.0d,
- maxResults = 1000,
+ maxResults = -1,
outputDelimiter = "\t",
ldaPRGSeed = None,
ldaMaxiterations = 20,
@@ -97,8 +98,10 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
val anomalousRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 1, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model = DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfig, data)
- val scoredData = model.score(sparkSession, data, emTestConfig.userDomain, emTestConfig.precisionUtility)
+
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ DNSSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+
val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
@@ -119,8 +122,10 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
val anomalousRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 1, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model = DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfig, data)
- val scoredData = model.score(sparkSession, data, onlineTestConfig.userDomain, onlineTestConfig.precisionUtility)
+
+ val SuspiciousConnectsAnalysisResults(scoredData, _) = DNSSuspiciousConnectsAnalysis.run(onlineTestConfig,
+ sparkSession, logger, data)
+
val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
new file mode 100644
index 0000000..f5a2d79
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
@@ -0,0 +1,40 @@
+package org.apache.spot.dns
+
+import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.apache.spot.utilities.{CountryCodes, TopDomains}
+import org.scalatest.Matchers
+
+class DNSSuspiciousConnectsModelTest extends TestingSparkContextFlatSpec with Matchers {
+
+ "createTempFields" should "create DNS stats based on an URL" in {
+ val countryCodesBC = sparkSession.sparkContext.broadcast(CountryCodes.CountryCodes)
+ val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = "intel.com"
+ val url = "iot.intel.com.mx"
+
+
+ val result = DNSSuspiciousConnectsModel.createTempFields(countryCodesBC, topDomainsBC, userDomain, url)
+
+ result.subdomainEntropy shouldBe 1.5849625007211559
+ result.subdomainLength shouldBe 3
+ result.topDomainClass shouldBe 1
+ result.numPeriods shouldBe 4
+
+ }
+
+ it should "create DNS stats based on an URL completely different to userDomain" in {
+ val countryCodesBC = sparkSession.sparkContext.broadcast(CountryCodes.CountryCodes)
+ val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = "anothercompany.com"
+ val url = "maps.company.com"
+
+ val result = DNSSuspiciousConnectsModel.createTempFields(countryCodesBC, topDomainsBC, userDomain, url)
+
+ result.subdomainEntropy shouldBe 2.0
+ result.subdomainLength shouldBe 4
+ result.topDomainClass shouldBe 1
+ result.numPeriods shouldBe 3
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index 296e073..9310390 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -20,6 +20,7 @@ package org.apache.spot.netflow
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.FlowSuspiciousConnectsAnalysis.InSchema
@@ -40,7 +41,7 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
topicCount = 20,
hdfsScoredConnect = "",
threshold = 1.0d,
- maxResults = 1000,
+ maxResults = -1,
outputDelimiter = "\t",
ldaPRGSeed = None,
ldaMaxiterations = 20,
@@ -91,10 +92,8 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model =
- FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfig, data)
-
- val scoredData = model.score(sparkSession, data, emTestConfig.precisionUtility)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(emTestConfig,
+ sparkSession, logger, data)
val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
@@ -125,10 +124,8 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model =
- FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfig, data)
-
- val scoredData = model.score(sparkSession, data, onlineTestConfig.precisionUtility)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(onlineTestConfig,
+ sparkSession, logger, data)
val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/53d00d5c/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index 1b1fc0e..45dc2df 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -20,6 +20,7 @@ package org.apache.spot.proxy
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.StructType
+import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.proxy.ProxySchema._
import org.apache.spot.testutils.TestingSparkContextFlatSpec
@@ -56,7 +57,7 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
topicCount = 20,
hdfsScoredConnect = "",
threshold = 1.0d,
- maxResults = 1000,
+ maxResults = -1,
outputDelimiter = "\t",
ldaPRGSeed = None,
ldaMaxiterations = 20,
@@ -106,9 +107,8 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, emTestConfigProxy, data)
-
- val scoredData = model.score(sparkSession, data, emTestConfigProxy.precisionUtility)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) = ProxySuspiciousConnectsAnalysis.run(emTestConfigProxy,
+ sparkSession, logger, data)
val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))
@@ -150,9 +150,12 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfigProxy, data)
+ // val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, onlineTestConfigProxy, data)
+ //
+ // val scoredData = model.score(sparkSession, data, onlineTestConfigProxy.precisionUtility)
- val scoredData = model.score(sparkSession, data, onlineTestConfigProxy.precisionUtility)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ ProxySuspiciousConnectsAnalysis.run(onlineTestConfigProxy, sparkSession, logger, data)
val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))