You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/15 21:56:37 UTC
[07/13] incubator-spot git commit: Changed precision utility to be
more generic and not proper to probabilities. Changed command line option
from scaling to precision.
Changed precision utility to be more generic and not proper to probabilities.
Changed command line option from scaling to precision.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/98ae3bf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/98ae3bf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/98ae3bf2
Branch: refs/heads/master
Commit: 98ae3bf2578dd4e70b19fd37a290492beda24a12
Parents: 5378d97
Author: Ricardo Barona <ri...@intel.com>
Authored: Mon May 29 17:21:00 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Tue Jun 13 12:28:47 2017 -0500
----------------------------------------------------------------------
spot-ml/ml_ops.sh | 2 +-
spot-ml/ml_test.sh | 2 +-
.../spot/SuspiciousConnectsArgumentParser.scala | 14 +--
.../spot/SuspiciousConnectsScoreFunction.scala | 9 +-
.../spot/dns/model/DNSScoreFunction.scala | 22 ++--
.../dns/model/DNSSuspiciousConnectsModel.scala | 8 +-
.../org/apache/spot/lda/SpotLDAWrapper.scala | 10 +-
.../spot/netflow/model/FlowScoreFunction.scala | 26 ++---
.../model/FlowSuspiciousConnectsModel.scala | 14 ++-
.../proxy/ProxySuspiciousConnectsModel.scala | 8 +-
.../transformation/PrecisionUtility.scala | 115 +++++++++++++++++++
.../transformation/ProbabilityConverter.scala | 110 ------------------
.../SuspiciousConnectsScoreFunctionTest.scala | 14 +--
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 4 +-
.../apache/spot/lda/SpotLDAWrapperTest.scala | 18 +--
.../FlowSuspiciousConnectsAnalysisTest.scala | 4 +-
.../ProxySuspiciousConnectsAnalysisTest.scala | 4 +-
.../PrecisionUtilityDoubleTest.scala | 48 ++++++++
.../PrecisionUtilityFloatTest.scala | 49 ++++++++
.../ProbabilityConverterDoubleTest.scala | 48 --------
.../ProbabilityConverterFloatTest.scala | 49 --------
21 files changed, 293 insertions(+), 285 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/ml_ops.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_ops.sh b/spot-ml/ml_ops.sh
index 005186e..4b1bc45 100755
--- a/spot-ml/ml_ops.sh
+++ b/spot-ml/ml_ops.sh
@@ -105,6 +105,6 @@ time spark-submit --class "org.apache.spot.SuspiciousConnects" \
--threshold ${TOL} \
--maxresults ${MAXRESULTS} \
--ldamaxiterations 20 \
- --scalingOption ${SCALING_OPTION} \
+ --precision ${PRECISION} \
$USER_DOMAIN_CMD
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/ml_test.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_test.sh b/spot-ml/ml_test.sh
index d352d3b..8fed837 100755
--- a/spot-ml/ml_test.sh
+++ b/spot-ml/ml_test.sh
@@ -76,5 +76,5 @@ time spark-submit --class "org.apache.spot.SuspiciousConnects" \
--threshold ${TOL} \
--maxresults ${MAXRESULTS} \
--ldamaxiterations 20 \
- --scalingOption ${SCALING_OPTION} \
+ --precision ${PRECISION} \
$USER_DOMAIN_CMD
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
index a1e4d94..8b4cdfc 100644
--- a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
@@ -17,7 +17,7 @@
package org.apache.spot
-import org.apache.spot.utilities.transformation.{ProbabilityConverter, ProbabilityConverterDouble, ProbabilityConverterFloat}
+import org.apache.spot.utilities.transformation.{PrecisionUtility, PrecisionUtilityDouble, PrecisionUtilityFloat}
/**
@@ -87,11 +87,11 @@ object SuspiciousConnectsArgumentParser {
action((x, c) => c.copy(ldaBeta = x)).
text("topic concentration for lda, default 1.001")
- opt[Int]("scalingOption").optional().valueName("int").
- action((x, c) => c.copy(probabilityConversionOption = x match {
- case 32 => ProbabilityConverterFloat
- case 64 => ProbabilityConverterDouble
- case _ => ProbabilityConverterDouble
+ opt[Int]("precision").optional().valueName("int").
+ action((x, c) => c.copy(precisionUtility = x match {
+ case 32 => PrecisionUtilityFloat
+ case 64 => PrecisionUtilityDouble
+ case _ => PrecisionUtilityDouble
}))
}
@@ -109,5 +109,5 @@ object SuspiciousConnectsArgumentParser {
ldaMaxiterations: Int = 20,
ldaAlpha: Double = 1.02,
ldaBeta: Double = 1.001,
- probabilityConversionOption: ProbabilityConverter = ProbabilityConverterDouble)
+ precisionUtility: PrecisionUtility = PrecisionUtilityDouble)
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
index 978ac93..aa9cde9 100644
--- a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
@@ -19,7 +19,7 @@ package org.apache.spot
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.ProbabilityConverter
+import org.apache.spot.utilities.transformation.PrecisionUtility
/**
* Base class for scoring suspicious connects models.
@@ -32,8 +32,8 @@ import org.apache.spot.utilities.transformation.ProbabilityConverter
class SuspiciousConnectsScoreFunction(topicCount: Int,
wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]]) extends Serializable {
- def score[P <: ProbabilityConverter](probabilityConversionOption: P)
- (documentTopicMix: Seq[probabilityConversionOption.ScalingType], word: String): Double = {
+ def score[P <: PrecisionUtility](precisionUtility: P)
+ (documentTopicMix: Seq[precisionUtility.TargetType], word: String): Double = {
val zeroProb = Array.fill(topicCount) {
0d
@@ -44,8 +44,9 @@ class SuspiciousConnectsScoreFunction(topicCount: Int,
} else {
// If either the ip or the word key value cannot be found it means that it was not seen in training.
val wordGivenTopicProbabilities = wordToPerTopicProbBC.value.getOrElse(word, zeroProb)
+ val documentTopicMixDouble: Seq[Double] = precisionUtility.toDoubles(documentTopicMix)
- probabilityConversionOption.convertBackSetOfProbabilities(documentTopicMix).zip(wordGivenTopicProbabilities)
+ documentTopicMixDouble.zip(wordGivenTopicProbabilities)
.map({ case (pWordGivenTopic, pTopicGivenDoc) => pWordGivenTopic * pTopicGivenDoc })
.sum
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
index c767f30..79070b2 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
@@ -20,7 +20,7 @@ package org.apache.spot.dns.model
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.SuspiciousConnectsScoreFunction
import org.apache.spot.dns.DNSWordCreation
-import org.apache.spot.utilities.transformation.ProbabilityConverter
+import org.apache.spot.utilities.transformation.PrecisionUtility
/**
@@ -58,15 +58,15 @@ class DNSScoreFunction(frameLengthCuts: Array[Double],
topDomainsBC,
userDomain)
- def score[P <: ProbabilityConverter](probabilityConversionOption: P)(timeStamp: String,
- unixTimeStamp: Long,
- frameLength: Int,
- clientIP: String,
- queryName: String,
- queryClass: String,
- queryType: Int,
- queryResponseCode: Int,
- documentTopicMix: Seq[probabilityConversionOption.ScalingType]): Double = {
+ def score[P <: PrecisionUtility](precisionUtility: P)(timeStamp: String,
+ unixTimeStamp: Long,
+ frameLength: Int,
+ clientIP: String,
+ queryName: String,
+ queryClass: String,
+ queryType: Int,
+ queryResponseCode: Int,
+ documentTopicMix: Seq[precisionUtility.TargetType]): Double = {
val word = dnsWordCreator.dnsWord(timeStamp,
unixTimeStamp,
@@ -77,6 +77,6 @@ class DNSScoreFunction(frameLengthCuts: Array[Double],
queryType,
queryResponseCode)
- suspiciousConnectsScoreFunction.score(probabilityConversionOption)(documentTopicMix, word)
+ suspiciousConnectsScoreFunction.score(precisionUtility)(documentTopicMix, word)
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
index a3d3b68..eaf363c 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
@@ -90,7 +90,7 @@ class DNSSuspiciousConnectsModel(inTopicCount: Int,
* probability estimated for the network event at that row
*/
def score(sc: SparkContext, sqlContext: SQLContext, inDF: DataFrame, userDomain: String
- , probabilityConversionOption: ProbabilityConverter): DataFrame = {
+ , precisionUtility: PrecisionUtility): DataFrame = {
val topDomainsBC = sc.broadcast(TopDomains.TopDomains)
val wordToPerTopicProbBC = sc.broadcast(wordToPerTopicProb)
@@ -115,8 +115,8 @@ class DNSSuspiciousConnectsModel(inTopicCount: Int,
queryClass: String,
queryType: Int,
queryResponseCode: Int,
- documentTopicMix: Seq[probabilityConversionOption.ScalingType]) =>
- scoreFunction.score(probabilityConversionOption)(timeStamp,
+ documentTopicMix: Seq[precisionUtility.TargetType]) =>
+ scoreFunction.score(precisionUtility)(timeStamp,
unixTimeStamp,
frameLength,
clientIP,
@@ -288,7 +288,7 @@ object DNSSuspiciousConnectsModel {
config.ldaAlpha,
config.ldaBeta,
config.ldaMaxiterations,
- config.probabilityConversionOption)
+ config.precisionUtility)
new DNSSuspiciousConnectsModel(config.topicCount,
ipToTopicMix,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
index 95eab8e..1f81123 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
@@ -25,7 +25,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.functions._
import org.apache.spark.sql.{DataFrame, Row, SQLContext}
import org.apache.spot.lda.SpotLDAWrapperSchema._
-import org.apache.spot.utilities.transformation.ProbabilityConverter
+import org.apache.spot.utilities.transformation.PrecisionUtility
import scala.collection.immutable.Map
@@ -49,7 +49,7 @@ object SpotLDAWrapper {
ldaAlpha: Double,
ldaBeta: Double,
maxIterations: Int,
- probabilityConversionOption: ProbabilityConverter): SpotLDAOutput = {
+ precisionUtility: PrecisionUtility): SpotLDAOutput = {
import sqlContext.implicits._
@@ -119,7 +119,7 @@ object SpotLDAWrapper {
//Create doc results from vector: convert docID back to string, convert vector of probabilities to array
val docToTopicMixDF =
- formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext, probabilityConversionOption)
+ formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext, precisionUtility)
documentDictionary.unpersist()
@@ -183,7 +183,7 @@ object SpotLDAWrapper {
}
def formatSparkLDADocTopicOutput(docTopDist: RDD[(Long, Vector)], documentDictionary: DataFrame, sqlContext:
- SQLContext, probabilityConversionOption: ProbabilityConverter):
+ SQLContext, precisionUtility: PrecisionUtility):
DataFrame = {
import sqlContext.implicits._
@@ -198,7 +198,7 @@ object SpotLDAWrapper {
.withColumn(TopicProbabilityMixArray, topicDistributionToArray(documentToTopicDistributionDF(TopicProbabilityMix)))
.selectExpr(s"$DocumentName AS $DocumentName", s"$TopicProbabilityMixArray AS $TopicProbabilityMix")
- probabilityConversionOption.convertDataFrameColumn(documentToTopicDistributionArray, TopicProbabilityMix)
+ precisionUtility.castColumn(documentToTopicDistributionArray, TopicProbabilityMix)
}
case class SpotLDAInput(doc: String, word: String, count: Int) extends Serializable
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
index c247994..45c1f33 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
@@ -20,7 +20,7 @@ package org.apache.spot.netflow.model
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.SuspiciousConnectsScoreFunction
import org.apache.spot.netflow.{FlowWordCreator, FlowWords}
-import org.apache.spot.utilities.transformation.ProbabilityConverter
+import org.apache.spot.utilities.transformation.PrecisionUtility
/**
@@ -53,22 +53,22 @@ class FlowScoreFunction(topicCount: Int,
* @param dstTopicMix topic mix assigned of destination IP
* @return Minium of probability of this word from the source IP and probability of this word from the dest IP.
*/
- def score[P <: ProbabilityConverter](probabilityConverter: P)(hour: Int,
- srcIP: String,
- dstIP: String,
- srcPort: Int,
- dstPort: Int,
- protocol: String,
- ibyt: Long,
- ipkt: Long,
- srcTopicMix: Seq[probabilityConverter.ScalingType],
- dstTopicMix: Seq[probabilityConverter.ScalingType]): Double = {
+ def score[P <: PrecisionUtility](precisionUtility: P)(hour: Int,
+ srcIP: String,
+ dstIP: String,
+ srcPort: Int,
+ dstPort: Int,
+ protocol: String,
+ ibyt: Long,
+ ipkt: Long,
+ srcTopicMix: Seq[precisionUtility.TargetType],
+ dstTopicMix: Seq[precisionUtility.TargetType]): Double = {
val FlowWords(srcWord, dstWord) = FlowWordCreator.flowWords(hour, srcPort, dstPort, protocol, ibyt, ipkt)
- val scoreOfConnectionFromSrcIP = suspiciousConnectsScoreFunction.score(probabilityConverter)(srcTopicMix, srcWord)
- val scoreOfConnectionsFromDstIP = suspiciousConnectsScoreFunction.score(probabilityConverter)(dstTopicMix, dstWord)
+ val scoreOfConnectionFromSrcIP = suspiciousConnectsScoreFunction.score(precisionUtility)(srcTopicMix, srcWord)
+ val scoreOfConnectionsFromDstIP = suspiciousConnectsScoreFunction.score(precisionUtility)(dstTopicMix, dstWord)
Math.min(scoreOfConnectionFromSrcIP, scoreOfConnectionsFromDstIP)
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
index 776b57c..4822721 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
@@ -30,7 +30,7 @@ import org.apache.spot.lda.SpotLDAWrapperSchema._
import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.FlowWordCreator
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.{ProbabilityConverter, Quantiles}
+import org.apache.spot.utilities.transformation.{PrecisionUtility, Quantiles}
import scala.util.{Failure, Success, Try}
@@ -60,7 +60,7 @@ class FlowSuspiciousConnectsModel(topicCount: Int,
wordToPerTopicProb: Map[String, Array[Double]]) {
def score(sc: SparkContext, sqlContext: SQLContext, flowRecords: DataFrame,
- probabilityConversionOption: ProbabilityConverter): DataFrame = {
+ precisionUtility: PrecisionUtility): DataFrame = {
val wordToPerTopicProbBC = sc.broadcast(wordToPerTopicProb)
@@ -94,9 +94,11 @@ class FlowSuspiciousConnectsModel(topicCount: Int,
protocol: String,
ibyt: Long,
ipkt: Long,
- srcIpTopicMix: Seq[probabilityConversionOption.ScalingType],
- dstIpTopicMix: Seq[probabilityConversionOption.ScalingType]) =>
- scoreFunction.score(probabilityConversionOption)(hour,
+ srcIpTopicMix: Seq[precisionUtility.TargetType],
+ dstIpTopicMix: Seq[precisionUtility.TargetType]) =>
+ scoreFunction.score(precisionUtility)(hour,
+ minute,
+ second,
srcIP,
dstIP,
srcPort,
@@ -187,7 +189,7 @@ object FlowSuspiciousConnectsModel {
config.ldaAlpha,
config.ldaBeta,
config.ldaMaxiterations,
- config.probabilityConversionOption)
+ config.precisionUtility)
new FlowSuspiciousConnectsModel(config.topicCount,
ipToTopicMix,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
index 8d8ee26..93fd883 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
@@ -51,7 +51,7 @@ class ProxySuspiciousConnectsModel(topicCount: Int,
* (as defined in ProxySchema object).
* @return Dataframe with Score column added.
*/
- def score(sc: SparkContext, dataFrame: DataFrame, probabilityConversionOption: ProbabilityConverter): DataFrame = {
+ def score(sc: SparkContext, dataFrame: DataFrame, precisionUtility: PrecisionUtility): DataFrame = {
val topDomains: Broadcast[Set[String]] = sc.broadcast(TopDomains.TopDomains)
@@ -76,8 +76,8 @@ class ProxySuspiciousConnectsModel(topicCount: Int,
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
- def udfScoreFunction = udf((documentTopicMix: Seq[probabilityConversionOption.ScalingType], word: String) =>
- scoreFunction.score(probabilityConversionOption)(documentTopicMix, word))
+ def udfScoreFunction = udf((documentTopicMix: Seq[precisionUtility.TargetType], word: String) =>
+ scoreFunction.score(precisionUtility)(documentTopicMix, word))
wordedDataFrame
.join(org.apache.spark.sql.functions.broadcast(ipToTopicMIx), dataFrame(ClientIP) === ipToTopicMIx(DocumentName), "left_outer")
@@ -154,7 +154,7 @@ object ProxySuspiciousConnectsModel {
config.ldaAlpha,
config.ldaBeta,
config.ldaMaxiterations,
- config.probabilityConversionOption)
+ config.precisionUtility)
new ProxySuspiciousConnectsModel(config.topicCount, ipToTopicMix, wordResults)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
new file mode 100644
index 0000000..4b85787
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities.transformation
+
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.functions._
+
+/**
+ * PrecisionUtility will transform a number from Double to Float if precision option is set to 32 bit,
+ * if default or 64 bit is selected, it will just return the same number type Double.
+ *
+ * This abstract class permits the execution of a single path during the entire analysis. Instead of checking what
+ * type should the data be converted, each implementation will know what type should be converted into and converted
+ * back.
+ *
+ */
+
+sealed trait PrecisionUtility extends Serializable {
+
+ type TargetType
+
+ /**
+ * Converts a number into the precision type; it can be Float (32) or Double (64).
+ * For the Double implementation it will just return the same value without any transformation.
+ *
+ * @param double a number to convert from Double to Target type.
+ * @return
+ */
+ def toTargetType(double: Double): TargetType
+
+ /**
+ * Converts back an Iterable of numbers from TargetType to Double. <% determines that it can process any object that
+ * extends from Iterable i.e. List, Vector, Seq, Queue, etc.
+ *
+ * @param targetTypeIterable an iterable that needs to be converted each element from TargetType to Double
+ * @return
+ */
+ def toDoubles[A <% Traversable[TargetType], B <% Traversable[Double]](targetTypeIterable: A): B
+
+ /**
+ * Converts a DataFrame column from Seq[Double] to a Seq[TargetType]. If the TargetType is Double, it will
+ * just return the same DataFrame without any transformation.
+ *
+ * @param dataFrame a DataFrame containing a column to be converted from Double to the TargetType
+ * @param columnName the name of the column to convert, the column should be Seq[Double]
+ * @return
+ */
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame
+
+}
+
+/**
+ * PrecisionUtility implementation for Float.
+ * Will convert numbers from Double to Float and back to Double to make the workload half the size. This conversion
+ * will allow to broadcast dictionaries if it fits in the autoBroadcastJoinThreshold - defined by users.
+ */
+object PrecisionUtilityFloat extends PrecisionUtility {
+
+ type TargetType = Float
+
+ def toTargetType(double: Double): Float = double.toFloat
+
+ def toDoubles[A <% Traversable[Float], B <% Traversable[Double]](targetTypeIterable: A): B =
+ targetTypeIterable.map(_.toDouble).asInstanceOf[B]
+
+
+ private val convertUDF = udf((doubles: Seq[Double]) => {
+ doubles.map(double => toTargetType(double))
+ })
+
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = {
+ val TempColumn = "temp_column"
+
+ dataFrame.withColumn(TempColumn, convertUDF(dataFrame(columnName)))
+ .drop(columnName)
+ .withColumnRenamed(TempColumn, columnName)
+ }
+
+}
+
+/**
+ * PrecisionUtility implementation for Double.
+ * Users that don't want to reduce the workload, can continue working with Doubles. This implementation will receive
+ * and send the same value, it won't do any transformation. The reason for this to exists is to avoid multiple code
+ * paths depending on the user wanting to change precision or not.
+ */
+object PrecisionUtilityDouble extends PrecisionUtility {
+
+ type TargetType = Double
+
+ def toTargetType(double: Double): Double = double
+
+ // Since Double is the default data type, this code is going to return the same array without any transformation.
+ def toDoubles[A <% Traversable[Double], B <% Traversable[Double]](targetTypeIterable: A): B =
+ targetTypeIterable.asInstanceOf[B]
+
+ // Since Double is the default data type, this code won't actually do any calculation but instead just return the
+ // same DataFrame.
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = dataFrame
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/ProbabilityConverter.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/ProbabilityConverter.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/ProbabilityConverter.scala
deleted file mode 100644
index 67d8d6b..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/ProbabilityConverter.scala
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.functions._
-
-/**
- * ProbabilityConverter will transform document probability from Double to Float if scaling option is set to 32 bit,
- * if default or 64 bit is selected, it will just return the same Double probability.
- *
- * This abstract class permits the execution of a single path during the entire analysis. Instead of checking what
- * type should the data be converted, each implementation will know what type should be converted into and converted
- * back.
- *
- */
-
-abstract class ProbabilityConverter extends Serializable {
-
- type ScalingType
-
- /**
- * Converts a probability into the scaling type; it can be Float or Double.
- * For the Double implementation it will just return the same value without any transformation.
- *
- * @param probability a probability to convert from Double to scaling type.
- * @return
- */
- def convertProbability(probability: Double): ScalingType
-
- /**
- * Converts back a set of probabilities from the scaling type into Double.
- *
- * @param scaledProbabilities
- * @return
- */
- def convertBackSetOfProbabilities(scaledProbabilities: Seq[ScalingType]): Array[Double]
-
- /**
- * Converts a column for a given DataFrame from Double to the scaling type. If the scaling type is Double, it will
- * just return the same DataFrame without any transformation.
- *
- * @param dataFrame a DataFrame containing a column to be converted from Double to the scaling type
- * @param columnName the name of the column to be converted.
- * @return
- */
- def convertDataFrameColumn(dataFrame: DataFrame, columnName: String): DataFrame
-
-}
-
-/**
- * ProbabilityConverter implementation for Float.
- * Will convert probabilities from Double to Float and back to Double to make the workload half the size that it'd
- * be with Double. This conversion will permit to broadcast document probabilities if it falls in the
- * autoBroadcastJoinThreshold - defined by users - and accelerate the join process between document probabilities
- * and the entire data set.
- */
-object ProbabilityConverterFloat extends ProbabilityConverter {
-
- type ScalingType = Float
- val convertUDF = udf((probabilities: Seq[Double]) => {
- for (probability <- probabilities) yield convertProbability(probability)
- })
-
- def convertProbability(probability: Double): Float = probability.toFloat
-
- def convertBackSetOfProbabilities(scaledProbabilities: Seq[Float]): Array[Double] =
- for (scaledProbability <- scaledProbabilities.toArray) yield scaledProbability.toDouble
-
- override def convertDataFrameColumn(dataFrame: DataFrame, columnName: String): DataFrame = {
- dataFrame.withColumn("temp_column", convertUDF(dataFrame(columnName)))
- .drop(columnName)
- .withColumnRenamed("temp_column", columnName)
- }
-}
-
-/**
- * ProbabilityConverter implementation for Double.
- * Users that don't want to reduce the workload, can continue working with Doubles. This implementation will receive
- * and send the same value, it won't do any transformation. The reason for this to exists is to avoid multiple code
- * paths depending on the user wanting to scale or not.
- */
-object ProbabilityConverterDouble extends ProbabilityConverter {
-
- type ScalingType = Double
-
- def convertProbability(probability: Double): Double = probability
-
- // Since Double is the default data type, this code is going to return the same array without any transformation.
- def convertBackSetOfProbabilities(scaledProbability: Seq[Double]): Array[Double] = scaledProbability.toArray
-
- // Since Double is the default data type, this code won't actually do any calculation but instead just return the
- // same DataFrame.
- override def convertDataFrameColumn(dataFrame: DataFrame, columnName: String): DataFrame = dataFrame
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
index 1412716..26faa9b 100644
--- a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
@@ -19,7 +19,7 @@ package org.apache.spot
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.{ProbabilityConverterDouble, ProbabilityConverterFloat}
+import org.apache.spot.utilities.transformation.{PrecisionUtilityDouble, PrecisionUtilityFloat}
import org.scalatest.Matchers
/**
@@ -33,12 +33,12 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val probabilityConversionOption = ProbabilityConverterDouble
+ val precisionUtility = PrecisionUtilityDouble
val documentProbabilities = Seq(0.05d, 0.05d, 0.05d, 0.05d)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
- val score = scoreFunction.score(probabilityConversionOption)(documentProbabilities, "word_1")
+ val score = scoreFunction.score(precisionUtility)(documentProbabilities, "word_1")
score shouldBe 0.010000000000000002d
@@ -50,12 +50,12 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val probabilityConversionOption = ProbabilityConverterFloat
+ val precisionUtility = PrecisionUtilityFloat
val documentProbabilities = Seq(0.05f, 0.05f, 0.05f, 0.05f)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
- val score = scoreFunction.score(probabilityConversionOption)(documentProbabilities, "word_1")
+ val score = scoreFunction.score(precisionUtility)(documentProbabilities, "word_1")
score shouldBe 0.010000000149011612d
@@ -66,12 +66,12 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val probabilityConversionOption = ProbabilityConverterFloat
+ val precisionUtility = PrecisionUtilityFloat
val documentProbabilities = Seq(0.05f, 0.05f, 0.05f, 0.05f)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
- val score = scoreFunction.score(probabilityConversionOption)(documentProbabilities, "word_error")
+ val score = scoreFunction.score(precisionUtility)(documentProbabilities, "word_error")
score shouldBe -1d
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index 681fe8c..4e37656 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -25,7 +25,7 @@ import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.dns.DNSSchema._
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.ProbabilityConverterFloat
+import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
import org.scalatest.Matchers
@@ -67,7 +67,7 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- probabilityConversionOption = ProbabilityConverterFloat)
+ precisionUtility = PrecisionUtilityFloat)
"dns suspicious connects analysis" should "estimate correct probabilities in toy data with framelength anomaly" in {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala b/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
index 2660ef7..a3e3bee 100644
--- a/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spot.lda.SpotLDAWrapper._
import org.apache.spot.lda.SpotLDAWrapperSchema._
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.{ProbabilityConverterDouble, ProbabilityConverterFloat}
+import org.apache.spot.utilities.transformation.{PrecisionUtilityDouble, PrecisionUtilityFloat}
import org.scalatest.Matchers
import scala.collection.immutable.Map
@@ -45,7 +45,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, ProbabilityConverterDouble)
+ ldaMaxIterations, PrecisionUtilityDouble)
val topicMixDF = out.docToTopicMix
@@ -68,7 +68,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, ProbabilityConverterFloat)
+ ldaMaxIterations, PrecisionUtilityFloat)
val topicMixDF = out.docToTopicMix
@@ -90,7 +90,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, ProbabilityConverterDouble)
+ ldaMaxIterations, PrecisionUtilityDouble)
val topicMixDF = out.docToTopicMix
val dogTopicMix: Array[Double] =
@@ -116,7 +116,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, ProbabilityConverterFloat)
+ ldaMaxIterations, PrecisionUtilityFloat)
val topicMixDF = out.docToTopicMix
val dogTopicMix: Array[Float] =
@@ -161,7 +161,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
}
"formatSparkLDADocTopicOutput" should "return RDD[(String,Array(Double))] after converting doc results from vector " +
- "using ProbabilityConverterDouble: convert docID back to string, convert vector of probabilities to array" in {
+ "using PrecisionUtilityDouble: convert docID back to string, convert vector of probabilities to array" in {
val documentWordData = sparkContext.parallelize(Seq(SpotLDAInput("192.168.1.1", "333333_7.0_0.0_1.0", 8),
SpotLDAInput("10.10.98.123", "1111111_6.0_3.0_5.0", 4),
@@ -179,7 +179,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
(2.toLong, Vectors.dense(0.4, 0.1, 0.3, 0.2))))
val sparkDocRes: DataFrame = formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext,
- ProbabilityConverterDouble)
+ PrecisionUtilityDouble)
val documents = sparkDocRes.select(DocumentName).map(documentName => documentName.toString.replaceAll("\\[", "")
.replaceAll("\\]", "")).collect()
@@ -195,7 +195,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
}
it should "return RDD[(String,Array(Float))] after converting doc results from vector " +
- "using ProbabilityConverterFloat: convert docID back to string, convert vector of probabilities to array" in {
+ "using PrecisionUtilityFloat: convert docID back to string, convert vector of probabilities to array" in {
val documentWordData = sparkContext.parallelize(Seq(SpotLDAInput("192.168.1.1", "333333_7.0_0.0_1.0", 8),
SpotLDAInput("10.10.98.123", "1111111_6.0_3.0_5.0", 4),
@@ -213,7 +213,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
(2.toLong, Vectors.dense(0.4, 0.1, 0.3, 0.2))))
val sparkDocRes: DataFrame = formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext,
- ProbabilityConverterFloat)
+ PrecisionUtilityFloat)
val documents = sparkDocRes.select(DocumentName).map(documentName => documentName.toString.replaceAll("\\[", "")
.replaceAll("\\]", "")).collect()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index 4897e3a..4abda8f 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -25,7 +25,7 @@ import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.FlowSuspiciousConnectsAnalysis.InSchema
import org.apache.spot.netflow.model.FlowSuspiciousConnectsModel
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.ProbabilityConverterFloat
+import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
import org.scalatest.Matchers
@@ -60,7 +60,7 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- probabilityConversionOption = ProbabilityConverterFloat)
+ precisionUtility = PrecisionUtilityFloat)
"netflow suspicious connects" should "correctly identify time-of-day anomalies" in {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index d3f90eb..2788818 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.proxy.ProxySchema._
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.ProbabilityConverterFloat
+import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
import org.scalatest.Matchers
case class ProxyInput(p_date: String,
@@ -76,7 +76,7 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- probabilityConversionOption = ProbabilityConverterFloat)
+ precisionUtility = PrecisionUtilityFloat)
"proxy supicious connects analysis" should "estimate correct probabilities in toy data with top domain anomaly" in {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
new file mode 100644
index 0000000..b15bc5d
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
@@ -0,0 +1,48 @@
+package org.apache.spot.utilities.transformation
+
+import org.apache.spark.sql.types._
+import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.scalatest.Matchers
+
+/**
+ * Created by rabarona on 5/17/17.
+ */
+class PrecisionUtilityDoubleTest extends TestingSparkContextFlatSpec with Matchers {
+
+ "toTargetType" should "just return the same value with the same type" in {
+ val testValue: Double = 5d
+
+ val result = PrecisionUtilityDouble.toTargetType(testValue)
+
+ result shouldBe testValue
+ result shouldBe a[java.lang.Double]
+
+ }
+
+ "toDoubles" should "return an array of the same type" in {
+
+ val testSeq: Seq[Double] = Seq(1d, 2d, 3d)
+
+ val result: Seq[Double] = PrecisionUtilityDouble.toDoubles(testSeq)
+
+ result shouldBe a[Seq[Double]]
+ result.length shouldBe 3
+ }
+
+ "castColumn" should "return the exact same data frame" in {
+
+ val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
+ .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
+
+ val result = PrecisionUtilityDouble.castColumn(testDataFrame, TopicProbabilityMix)
+
+ val schema = StructType(
+ Array(StructField(DocumentName, StringType, true),
+ StructField(TopicProbabilityMix, ArrayType(DoubleType, false), true)))
+
+ result shouldBe testDataFrame
+ result.schema shouldBe schema
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
new file mode 100644
index 0000000..f965b3d
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
@@ -0,0 +1,49 @@
+package org.apache.spot.utilities.transformation
+
+import org.apache.spark.sql.types._
+import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.scalatest.Matchers
+
+/**
+ * Created by rabarona on 5/17/17.
+ */
+class PrecisionUtilityFloatTest extends TestingSparkContextFlatSpec with Matchers {
+
+ "toTargetType" should "just return value converted to Float" in {
+ val testValue: Double = 5d
+
+ val result = PrecisionUtilityFloat.toTargetType(testValue)
+
+ result shouldBe 5f
+ result shouldBe a[java.lang.Float]
+
+ }
+
+ "toDoubles" should "convert a Seq of Float to Seq of Double" in {
+
+ val testSeq: Seq[Float] = Seq(1f, 2f, 3f)
+
+ val result: Seq[Double] = PrecisionUtilityFloat.toDoubles(testSeq)
+
+ result(0) shouldBe a[java.lang.Double]
+ result(1) shouldBe a[java.lang.Double]
+ result(2) shouldBe a[java.lang.Double]
+ }
+
+ "castColumn" should "return a data frame with a schema modified, Seq[Float] instead of Seq[Double]" in {
+
+ val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
+ .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
+
+ val result = PrecisionUtilityFloat.castColumn(testDataFrame, TopicProbabilityMix)
+ result.count
+
+ val schema = StructType(
+ Array(StructField(DocumentName, StringType, true),
+ StructField(TopicProbabilityMix, ArrayType(FloatType, false), true)))
+
+ result.schema shouldBe schema
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterDoubleTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterDoubleTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterDoubleTest.scala
deleted file mode 100644
index 50e46b4..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterDoubleTest.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.types._
-import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.scalatest.Matchers
-
-/**
- * Created by rabarona on 5/17/17.
- */
-class ProbabilityConverterDoubleTest extends TestingSparkContextFlatSpec with Matchers {
-
- "convertProbability" should "just return the same value with the same type" in {
- val testValue: Double = 5d
-
- val result = ProbabilityConverterDouble.convertProbability(testValue)
-
- result shouldBe testValue
- result shouldBe a[java.lang.Double]
-
- }
-
- "convertBackSetOfProbabilities" should "return a array of the same type" in {
-
- val testSeq: Seq[Double] = Seq(1d, 2d, 3d)
-
- val result = ProbabilityConverterDouble.convertBackSetOfProbabilities(testSeq)
-
- result shouldBe a[Array[Double]]
- result.length shouldBe 3
- }
-
- "convertDataFrameColumn" should "return the exact same data frame" in {
-
- val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
- .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
-
- val result = ProbabilityConverterDouble.convertDataFrameColumn(testDataFrame, TopicProbabilityMix)
-
- val schema = StructType(
- Array(StructField(DocumentName, StringType, true),
- StructField(TopicProbabilityMix, ArrayType(DoubleType, false), true)))
-
- result shouldBe testDataFrame
- result.schema shouldBe schema
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/98ae3bf2/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterFloatTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterFloatTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterFloatTest.scala
deleted file mode 100644
index 6aa949c..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/ProbabilityConverterFloatTest.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.types._
-import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.scalatest.Matchers
-
-/**
- * Created by rabarona on 5/17/17.
- */
-class ProbabilityConverterFloatTest extends TestingSparkContextFlatSpec with Matchers {
-
- "convertProbability" should "just return value converted to Float" in {
- val testValue: Double = 5d
-
- val result = ProbabilityConverterFloat.convertProbability(testValue)
-
- result shouldBe 5f
- result shouldBe a[java.lang.Float]
-
- }
-
- "convertBackSetOfProbabilities" should "convert a Seq of Float to Seq of Double" in {
-
- val testSeq: Seq[Float] = Seq(1f, 2f, 3f)
-
- val result = ProbabilityConverterFloat.convertBackSetOfProbabilities(testSeq)
-
- result(0) shouldBe a[java.lang.Double]
- result(1) shouldBe a[java.lang.Double]
- result(2) shouldBe a[java.lang.Double]
- }
-
- "convertDataFrameColumn" should "return the exact same data frame" in {
-
- val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
- .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
-
- val result = ProbabilityConverterFloat.convertDataFrameColumn(testDataFrame, TopicProbabilityMix)
- result.count
-
- val schema = StructType(
- Array(StructField(DocumentName, StringType, true),
- StructField(TopicProbabilityMix, ArrayType(FloatType, false), true)))
-
- result.schema shouldBe schema
-
- }
-}