You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/15 21:56:40 UTC
[10/13] incubator-spot git commit: Reverting utilities.transformation
package. Changed name of PrecisionUtility to FloatingPointPrecisionUtility.
Minor documentation fixes.
Reverting utilities.transformation package.
Changed name of PrecisionUtility to FloatingPointPrecisionUtility.
Minor documentation fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/4c7ad4eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/4c7ad4eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/4c7ad4eb
Branch: refs/heads/master
Commit: 4c7ad4eb6e4e712708b242a0499e4c24afb51296
Parents: 2904940
Author: Ricardo Barona <ri...@intel.com>
Authored: Wed May 31 10:51:52 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Tue Jun 13 12:33:30 2017 -0500
----------------------------------------------------------------------
spot-ml/SPARKCONF.md | 4 +-
.../spot/SuspiciousConnectsArgumentParser.scala | 10 +-
.../spot/SuspiciousConnectsScoreFunction.scala | 6 +-
.../org/apache/spot/dns/DNSWordCreation.scala | 4 +-
.../spot/dns/model/DNSScoreFunction.scala | 20 +--
.../dns/model/DNSSuspiciousConnectsModel.scala | 6 +-
.../org/apache/spot/lda/SpotLDAWrapper.scala | 6 +-
.../spot/netflow/model/FlowScoreFunction.scala | 23 ++-
.../model/FlowSuspiciousConnectsModel.scala | 4 +-
.../proxy/ProxySuspiciousConnectsModel.scala | 2 +-
.../apache/spot/utilities/CountryCodes.scala | 36 ++++
.../apache/spot/utilities/DomainProcessor.scala | 164 +++++++++++++++++
.../org/apache/spot/utilities/Entropy.scala | 40 +++++
.../utilities/FloatPointPrecisionUtility.scala | 108 +++++++++++
.../org/apache/spot/utilities/Quantiles.scala | 124 +++++++++++++
.../apache/spot/utilities/TimeUtilities.scala | 51 ++++++
.../org/apache/spot/utilities/TopDomains.scala | 31 ++++
.../utilities/transformation/CountryCodes.scala | 36 ----
.../transformation/DomainProcessor.scala | 164 -----------------
.../spot/utilities/transformation/Entropy.scala | 40 -----
.../transformation/PrecisionUtility.scala | 115 ------------
.../utilities/transformation/Quantiles.scala | 124 -------------
.../transformation/TimeUtilities.scala | 51 ------
.../utilities/transformation/TopDomains.scala | 31 ----
.../SuspiciousConnectsScoreFunctionTest.scala | 8 +-
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 4 +-
.../apache/spot/dns/DNSWordCreationTest.scala | 2 +-
.../apache/spot/lda/SpotLDAWrapperTest.scala | 14 +-
.../FlowSuspiciousConnectsAnalysisTest.scala | 4 +-
.../ProxySuspiciousConnectsAnalysisTest.scala | 4 +-
.../spot/utilities/DomainProcessorTest.scala | 146 +++++++++++++++
.../FloatPointPrecisionUtility32Test.scala | 49 +++++
.../spot/utilities/FloatingPointUtility64.scala | 48 +++++
.../apache/spot/utilities/QuantilesTest.scala | 178 +++++++++++++++++++
.../transformation/DomainProcessorTest.scala | 146 ---------------
.../PrecisionUtilityDoubleTest.scala | 48 -----
.../PrecisionUtilityFloatTest.scala | 49 -----
.../transformation/QuantilesTest.scala | 178 -------------------
38 files changed, 1035 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/SPARKCONF.md
----------------------------------------------------------------------
diff --git a/spot-ml/SPARKCONF.md b/spot-ml/SPARKCONF.md
index 7b57098..e7fd8f9 100644
--- a/spot-ml/SPARKCONF.md
+++ b/spot-ml/SPARKCONF.md
@@ -1,6 +1,6 @@
## Spark Configuration
-spot-ml main component uses Spark and Spark SQL to analyze network events and produce a list of least probable events
+spot-ml main component uses Spark and Spark SQL to analyze network events and those considered the most unlikely
or most suspicious.
To run spot-ml with its best performance and scalability, it will probably be necessary to configure Yarn, Spark and Spot. Here are our recommended settings.
@@ -138,7 +138,7 @@ The correct setting of SPK_AUTO_BRDCST_JOIN_THR and PRECISION can help to alway
As a first step, users need to decide whether they want to change from 64 bit floating point probabilities to 32 bit floating point probabilities; if users decide to change from 64 to 32 bit, the document probability distribution lookup table will be half the size and more easily broadcasted.
-If users want to cut the payload on half, they should set precision option to 32.
+If users want to cut payload memory consumption roughly in half, they should set the precision option to 32.
>PRECISION='32'
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
index 8b4cdfc..f5472e6 100644
--- a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsArgumentParser.scala
@@ -17,7 +17,7 @@
package org.apache.spot
-import org.apache.spot.utilities.transformation.{PrecisionUtility, PrecisionUtilityDouble, PrecisionUtilityFloat}
+import org.apache.spot.utilities.{FloatPointPrecisionUtility, FloatPointPrecisionUtility32, FloatPointPrecisionUtility64}
/**
@@ -89,9 +89,9 @@ object SuspiciousConnectsArgumentParser {
opt[Int]("precision").optional().valueName("int").
action((x, c) => c.copy(precisionUtility = x match {
- case 32 => PrecisionUtilityFloat
- case 64 => PrecisionUtilityDouble
- case _ => PrecisionUtilityDouble
+ case 32 => FloatPointPrecisionUtility32
+ case 64 => FloatPointPrecisionUtility64
+ case _ => FloatPointPrecisionUtility64
}))
}
@@ -109,5 +109,5 @@ object SuspiciousConnectsArgumentParser {
ldaMaxiterations: Int = 20,
ldaAlpha: Double = 1.02,
ldaBeta: Double = 1.001,
- precisionUtility: PrecisionUtility = PrecisionUtilityDouble)
+ precisionUtility: FloatPointPrecisionUtility = FloatPointPrecisionUtility64)
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
index aa9cde9..97cde68 100644
--- a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnectsScoreFunction.scala
@@ -18,8 +18,8 @@
package org.apache.spot
import org.apache.spark.broadcast.Broadcast
+import org.apache.spot.utilities.FloatPointPrecisionUtility
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.PrecisionUtility
/**
* Base class for scoring suspicious connects models.
@@ -32,8 +32,8 @@ import org.apache.spot.utilities.transformation.PrecisionUtility
class SuspiciousConnectsScoreFunction(topicCount: Int,
wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]]) extends Serializable {
- def score[P <: PrecisionUtility](precisionUtility: P)
- (documentTopicMix: Seq[precisionUtility.TargetType], word: String): Double = {
+ def score[P <: FloatPointPrecisionUtility](precisionUtility: P)
+ (documentTopicMix: Seq[precisionUtility.TargetType], word: String): Double = {
val zeroProb = Array.fill(topicCount) {
0d
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/dns/DNSWordCreation.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/DNSWordCreation.scala b/spot-ml/src/main/scala/org/apache/spot/dns/DNSWordCreation.scala
index 1f2b15b..135755a 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/DNSWordCreation.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/DNSWordCreation.scala
@@ -19,9 +19,9 @@ package org.apache.spot.dns
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.functions._
+import org.apache.spot.utilities.DomainProcessor.{DomainInfo, extractDomainInfo}
+import org.apache.spot.utilities.Quantiles
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.DomainProcessor.{DomainInfo, extractDomainInfo}
-import org.apache.spot.utilities.transformation.Quantiles
import scala.util.{Success, Try}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
index 79070b2..f98f7cb 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSScoreFunction.scala
@@ -20,7 +20,7 @@ package org.apache.spot.dns.model
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.SuspiciousConnectsScoreFunction
import org.apache.spot.dns.DNSWordCreation
-import org.apache.spot.utilities.transformation.PrecisionUtility
+import org.apache.spot.utilities.FloatPointPrecisionUtility
/**
@@ -58,15 +58,15 @@ class DNSScoreFunction(frameLengthCuts: Array[Double],
topDomainsBC,
userDomain)
- def score[P <: PrecisionUtility](precisionUtility: P)(timeStamp: String,
- unixTimeStamp: Long,
- frameLength: Int,
- clientIP: String,
- queryName: String,
- queryClass: String,
- queryType: Int,
- queryResponseCode: Int,
- documentTopicMix: Seq[precisionUtility.TargetType]): Double = {
+ def score[P <: FloatPointPrecisionUtility](precisionUtility: P)(timeStamp: String,
+ unixTimeStamp: Long,
+ frameLength: Int,
+ clientIP: String,
+ queryName: String,
+ queryClass: String,
+ queryType: Int,
+ queryResponseCode: Int,
+ documentTopicMix: Seq[precisionUtility.TargetType]): Double = {
val word = dnsWordCreator.dnsWord(timeStamp,
unixTimeStamp,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
index eaf363c..d5d35e8 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
@@ -30,9 +30,9 @@ import org.apache.spot.dns.DNSWordCreation
import org.apache.spot.lda.SpotLDAWrapper
import org.apache.spot.lda.SpotLDAWrapper._
import org.apache.spot.lda.SpotLDAWrapperSchema._
+import org.apache.spot.utilities.DomainProcessor.DomainInfo
+import org.apache.spot.utilities._
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.DomainProcessor.DomainInfo
-import org.apache.spot.utilities.transformation._
import scala.util.{Failure, Success, Try}
@@ -90,7 +90,7 @@ class DNSSuspiciousConnectsModel(inTopicCount: Int,
* probability estimated for the network event at that row
*/
def score(sc: SparkContext, sqlContext: SQLContext, inDF: DataFrame, userDomain: String
- , precisionUtility: PrecisionUtility): DataFrame = {
+ , precisionUtility: FloatPointPrecisionUtility): DataFrame = {
val topDomainsBC = sc.broadcast(TopDomains.TopDomains)
val wordToPerTopicProbBC = sc.broadcast(wordToPerTopicProb)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
index 1f81123..9e882e0 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
@@ -25,7 +25,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.functions._
import org.apache.spark.sql.{DataFrame, Row, SQLContext}
import org.apache.spot.lda.SpotLDAWrapperSchema._
-import org.apache.spot.utilities.transformation.PrecisionUtility
+import org.apache.spot.utilities.FloatPointPrecisionUtility
import scala.collection.immutable.Map
@@ -49,7 +49,7 @@ object SpotLDAWrapper {
ldaAlpha: Double,
ldaBeta: Double,
maxIterations: Int,
- precisionUtility: PrecisionUtility): SpotLDAOutput = {
+ precisionUtility: FloatPointPrecisionUtility): SpotLDAOutput = {
import sqlContext.implicits._
@@ -183,7 +183,7 @@ object SpotLDAWrapper {
}
def formatSparkLDADocTopicOutput(docTopDist: RDD[(Long, Vector)], documentDictionary: DataFrame, sqlContext:
- SQLContext, precisionUtility: PrecisionUtility):
+ SQLContext, precisionUtility: FloatPointPrecisionUtility):
DataFrame = {
import sqlContext.implicits._
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
index 45c1f33..c841c66 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowScoreFunction.scala
@@ -20,7 +20,7 @@ package org.apache.spot.netflow.model
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.SuspiciousConnectsScoreFunction
import org.apache.spot.netflow.{FlowWordCreator, FlowWords}
-import org.apache.spot.utilities.transformation.PrecisionUtility
+import org.apache.spot.utilities.FloatPointPrecisionUtility
/**
@@ -53,17 +53,16 @@ class FlowScoreFunction(topicCount: Int,
* @param dstTopicMix topic mix assigned of destination IP
* @return Minium of probability of this word from the source IP and probability of this word from the dest IP.
*/
- def score[P <: PrecisionUtility](precisionUtility: P)(hour: Int,
- srcIP: String,
- dstIP: String,
- srcPort: Int,
- dstPort: Int,
- protocol: String,
- ibyt: Long,
- ipkt: Long,
- srcTopicMix: Seq[precisionUtility.TargetType],
- dstTopicMix: Seq[precisionUtility.TargetType]): Double = {
-
+ def score[P <: FloatPointPrecisionUtility](precisionUtility: P)(hour: Int,
+ srcIP: String,
+ dstIP: String,
+ srcPort: Int,
+ dstPort: Int,
+ protocol: String,
+ ibyt: Long,
+ ipkt: Long,
+ srcTopicMix: Seq[precisionUtility.TargetType],
+ dstTopicMix: Seq[precisionUtility.TargetType]): Double = {
val FlowWords(srcWord, dstWord) = FlowWordCreator.flowWords(hour, srcPort, dstPort, protocol, ibyt, ipkt)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
index 4822721..fae58a3 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
@@ -30,7 +30,7 @@ import org.apache.spot.lda.SpotLDAWrapperSchema._
import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.FlowWordCreator
import org.apache.spot.utilities.data.validation.InvalidDataHandler
-import org.apache.spot.utilities.transformation.{PrecisionUtility, Quantiles}
+import org.apache.spot.utilities.{FloatPointPrecisionUtility, Quantiles}
import scala.util.{Failure, Success, Try}
@@ -60,7 +60,7 @@ class FlowSuspiciousConnectsModel(topicCount: Int,
wordToPerTopicProb: Map[String, Array[Double]]) {
def score(sc: SparkContext, sqlContext: SQLContext, flowRecords: DataFrame,
- precisionUtility: PrecisionUtility): DataFrame = {
+ precisionUtility: FloatPointPrecisionUtility): DataFrame = {
val wordToPerTopicProbBC = sc.broadcast(wordToPerTopicProb)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
index 93fd883..80daf4f 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
@@ -51,7 +51,7 @@ class ProxySuspiciousConnectsModel(topicCount: Int,
* (as defined in ProxySchema object).
* @return Dataframe with Score column added.
*/
- def score(sc: SparkContext, dataFrame: DataFrame, precisionUtility: PrecisionUtility): DataFrame = {
+ def score(sc: SparkContext, dataFrame: DataFrame, precisionUtility: FloatPointPrecisionUtility): DataFrame = {
val topDomains: Broadcast[Set[String]] = sc.broadcast(TopDomains.TopDomains)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/CountryCodes.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/CountryCodes.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/CountryCodes.scala
new file mode 100644
index 0000000..e54ca6c
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/CountryCodes.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+object CountryCodes {
+
+ val CountryCodes = Set("ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "as", "at", "au",
+ "aw", "ax", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bt",
+ "bv", "bw", "by", "bz", "ca", "cc", "cd", "cf", "cg", "ch", "ci", "ck", "cl", "cm", "cn", "co", "cr", "cu", "cv",
+ "cw", "cx", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "eh", "er", "es", "et", "eu", "fi",
+ "fj", "fk", "fm", "fo", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gi", "gl", "gm", "gn", "gp", "gq", "gr",
+ "gs", "gt", "gu", "gw", "gy", "hk", "hm", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "io", "iq", "ir",
+ "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr", "krd", "kw", "ky", "kz", "la",
+ "lb", "lc", "li", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mg", "mh", "mk", "ml", "mm",
+ "mn", "mo", "mp", "mq", "mr", "ms", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "nf", "ng", "ni",
+ "nl", "no", "np", "nr", "nu", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pm", "pn", "pr", "ps", "pt",
+ "pw", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sb", "sc", "sd", "se", "sg", "sh", "si", "sj", "", "sk",
+ "sl", "sm", "sn", "so", "sr", "ss", "st", "su", "sv", "sx", "sy", "sz", "tc", "td", "tf", "tg", "th", "tj", "tk",
+ "tl", "tm", "tn", "to", "tp", "tr", "tt", "tv", "tw", "tz", "ua", "ug", "uk", "us", "uy", "uz", "va", "vc", "ve",
+ "vg", "vi", "vn", "vu", "wf", "ws", "ye", "yt", "za", "zm", "zw")
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/DomainProcessor.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/DomainProcessor.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/DomainProcessor.scala
new file mode 100644
index 0000000..d0fabf8
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/DomainProcessor.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import org.apache.spark.broadcast.Broadcast
+
+/**
+ * Routines and data for processing URLs for domains, subdomains, country code, top-level domains, etc.
+ */
+object DomainProcessor extends Serializable {
+
+ val CountryCodes = Set("ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "as", "at", "au",
+ "aw", "ax", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bt",
+ "bv", "bw", "by", "bz", "ca", "cc", "cd", "cf", "cg", "ch", "ci", "ck", "cl", "cm", "cn", "co", "cr", "cu", "cv",
+ "cw", "cx", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "eh", "er", "es", "et", "eu", "fi",
+ "fj", "fk", "fm", "fo", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gi", "gl", "gm", "gn", "gp", "gq", "gr",
+ "gs", "gt", "gu", "gw", "gy", "hk", "hm", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "io", "iq", "ir",
+ "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr", "krd", "kw", "ky", "kz", "la",
+ "lb", "lc", "li", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mg", "mh", "mk", "ml", "mm",
+ "mn", "mo", "mp", "mq", "mr", "ms", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "nf", "ng", "ni",
+ "nl", "no", "np", "nr", "nu", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pm", "pn", "pr", "ps", "pt",
+ "pw", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sb", "sc", "sd", "se", "sg", "sh", "si", "sj", "", "sk",
+ "sl", "sm", "sn", "so", "sr", "ss", "st", "su", "sv", "sx", "sy", "sz", "tc", "td", "tf", "tg", "th", "tj", "tk",
+ "tl", "tm", "tn", "to", "tp", "tr", "tt", "tv", "tw", "tz", "ua", "ug", "uk", "us", "uy", "uz", "va", "vc", "ve",
+ "vg", "vi", "vn", "vu", "wf", "ws", "ye", "yt", "za", "zm", "zw")
+
+ val TopLevelDomainNames = Set("com", "org", "net", "int", "edu", "gov", "mil")
+ val None = "None"
+
+ /**
+ * Extract domain info from a url.
+ * @param url Incoming url.
+ * @param topDomainsBC Broadcast variable containing the top domains set.
+ * @param userDomain Domain of the spot user (example:'intel').
+ * @return New [[DomainInfo]] object containing extracted domain information.
+ */
+ def extractDomainInfo(url: String, topDomainsBC: Broadcast[Set[String]], userDomain: String): DomainInfo = {
+
+ val spliturl = url.split('.')
+ val numParts = spliturl.length
+
+ val (domain, subdomain) = extractDomainSubdomain(url)
+
+
+ val subdomainLength = if (subdomain != None) {
+ subdomain.length
+ } else {
+ 0
+ }
+
+ val topDomainClass = if (userDomain != "" && domain == userDomain) {
+ 2
+ } else if (topDomainsBC.value contains domain) {
+ 1
+ } else {
+ 0
+ }
+
+ val subdomainEntropy = if (subdomain != "None") Entropy.stringEntropy(subdomain) else 0d
+
+ DomainInfo(domain, topDomainClass, subdomain, subdomainLength, subdomainEntropy, numParts)
+ }
+
+ /**
+ *
+ * @param url Url from which to extract domain.
+ * @return Domain name or "None" if there is none.
+ */
+ def extractDomain(url: String) : String = {
+ val (domain, _) = extractDomainSubdomain(url)
+ domain
+ }
+
+ /**
+ * Extrat the domain and subdomain from a URL.
+ * @param url URL to be parsed.
+ * @return Pair of (domain, subdomain). If there is no domain, both fields contain "None".
+ * If there is no subdomain then the subdomain field is "None"
+ */
+ def extractDomainSubdomain(url: String) : (String, String) = {
+ val spliturl = url.split('.')
+ val numParts = spliturl.length
+
+
+ var domain = None
+ var subdomain = None
+ // First check if query is an IP address e.g.: 123.103.104.10.in-addr.arpa or a name.
+ // Such URLs receive a domain of NO_DOMAIN
+
+ if (numParts >= 2
+ && !(numParts > 2 && spliturl(numParts - 1) == "arpa" && spliturl(numParts - 2) == "in-addr")
+ && (CountryCodes.contains(spliturl.last) || TopLevelDomainNames.contains(spliturl.last))) {
+ val strippedSplitURL = removeTopLevelDomainName(removeCountryCode(spliturl))
+ if (strippedSplitURL.length > 0) {
+ domain = strippedSplitURL.last
+ if (strippedSplitURL.length > 1) {
+ subdomain = strippedSplitURL.slice(0, strippedSplitURL.length - 1).mkString(".")
+ }
+ }
+ }
+
+
+ (domain, subdomain)
+ }
+
+ /**
+ * Strip the country code from a split URL.
+ * @param urlComponents Array of the entries of a URL after splitting on periods.
+ * @return URL components with the country code stripped.
+ */
+ def removeCountryCode(urlComponents: Array[String]): Array[String] = {
+ if (CountryCodes.contains(urlComponents.last)) {
+ urlComponents.dropRight(1)
+ } else {
+ urlComponents
+ }
+ }
+
+ /**
+ * Strip the top-level domain name from a split URL.
+ * @param urlComponents Array of the entries ofa URL after splitting on periods.
+ * @return URL components with the top-level domain name stripped.
+ */
+ def removeTopLevelDomainName(urlComponents: Array[String]): Array[String] = {
+ if (TopLevelDomainNames.contains(urlComponents.last)) {
+ urlComponents.dropRight(1)
+ } else {
+ urlComponents
+ }
+ }
+
+ /**
+ * Commonly extracted domain features.
+ *
+ * @param domain Domain (if any) of a url.
+ * @param topDomain Numerical class of domain: 2 for Intel, 1 for Alexa top domains, 0 for others.
+ * @param subdomain Subdomain (if any) in the url.
+ * @param subdomainLength Length of the subdomain. 0 if there is none.
+ * @param subdomainEntropy Entropy of the subdomain viewed as a distribution on its character set.
+ * 0 if there is no subdomain.
+ * @param numPeriods Number of periods + 1 in the url. (Number of sub-strings where url is split by periods.)
+ */
+ case class DomainInfo(domain: String,
+ topDomain: Int,
+ subdomain: String,
+ subdomainLength: Int,
+ subdomainEntropy: Double,
+ numPeriods: Int)
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/Entropy.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/Entropy.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/Entropy.scala
new file mode 100644
index 0000000..ff46bd3
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/Entropy.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import scala.math._
+
+/**
+ * Created by nlsegerl on 8/4/16.
+ */
+object Entropy {
+
+ /**
+ * Calculates the "entropy" in string, interpreted as a distribution on its constituent characters.
+ *
+ * @param v Input string
+ * @return
+ */
+ def stringEntropy(v: String): Double = {
+ if (v.length() > 0) {
+ v.groupBy(a => a).values.map(i => i.length.toDouble / v.length).map(p => -p * log10(p) / log10(2)).sum
+ } else {
+ 0
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/FloatPointPrecisionUtility.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/FloatPointPrecisionUtility.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/FloatPointPrecisionUtility.scala
new file mode 100644
index 0000000..a4da7a0
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/FloatPointPrecisionUtility.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.functions._
+
+/**
+ * PrecisionUtility will transform a number from Double to Float if precision option is set to 32 bit,
+ * if default or 64 bit is selected, it will just return the same number type Double.
+ *
+ */
+sealed trait FloatPointPrecisionUtility extends Serializable {
+
+ type TargetType
+
+ /**
+ * Converts a number into the precision type; it can be Float (32) or Double (64).
+ *
+ * @param double a number to convert from Double to Target type.
+ * @return
+ */
+ def toTargetType(double: Double): TargetType
+
+ /**
+ * Converts back an Iterable of numbers from TargetType to Double. <% determines that it can process any object that
+ * extends from Iterable i.e. List, Vector, Seq, Queue, etc.
+ *
+ * @param targetTypeIterable an iterable that needs to be converted each element from TargetType to Double
+ * @return
+ */
+ def toDoubles[A <% Traversable[TargetType], B <% Traversable[Double]](targetTypeIterable: A): B
+
+ /**
+ * Converts a DataFrame column from Seq[Double] to a Seq[TargetType].
+ *
+ * @param dataFrame a DataFrame containing a column to be converted from Double to the TargetType
+ * @param columnName the name of the column to convert, the column should be Seq[Double]
+ * @return
+ */
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame
+
+}
+
+/**
+ * PrecisionUtility implementation for Float.
+ * Will convert numbers from Double to Float and back to Double.
+ *
+ */
+object FloatPointPrecisionUtility32 extends FloatPointPrecisionUtility {
+
+ type TargetType = Float
+
+ def toTargetType(double: Double): Float = double.toFloat
+
+ def toDoubles[A <% Traversable[Float], B <% Traversable[Double]](targetTypeIterable: A): B =
+ targetTypeIterable.map(_.toDouble).asInstanceOf[B]
+
+
+ private val convertUDF = udf((doubles: Seq[Double]) => {
+ doubles.map(double => toTargetType(double))
+ })
+
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = {
+ val TempColumn = "temp_column"
+
+ dataFrame.withColumn(TempColumn, convertUDF(dataFrame(columnName)))
+ .drop(columnName)
+ .withColumnRenamed(TempColumn, columnName)
+ }
+
+}
+
+/**
+ * PrecisionUtility implementation for Double.
+ * This implementation will receive and send the same value, it won't do any transformation.
+ *
+ */
+object FloatPointPrecisionUtility64 extends FloatPointPrecisionUtility {
+
+ type TargetType = Double
+
+ // For this implementation it will just return the same value without any transformation.
+ def toTargetType(double: Double): Double = double
+
+ // Since Double is the default data type, this code is going to return the same array without any transformation.
+ def toDoubles[A <% Traversable[Double], B <% Traversable[Double]](targetTypeIterable: A): B =
+ targetTypeIterable.asInstanceOf[B]
+
+ // Since Double is the default data type, this code won't actually do any calculation but instead just return the
+ // same DataFrame.
+ def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = dataFrame
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/Quantiles.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/Quantiles.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/Quantiles.scala
new file mode 100644
index 0000000..5494ab5
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/Quantiles.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import org.apache.spark.rdd.RDD
+
+import scala.math._
+
+/**
+ * Contains routines for the distributed calculation of quantiles and the empirical cumulative distribution function.
+ */
+
+object Quantiles extends Serializable {
+
+ val DECILES = Array(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
+ val QUINTILES = Array(0.2, 0.4, 0.6, 0.8, 1.0)
+
+ /**
+ * Compute the deciles of a distribution.
+ *
+ * @param data RDD[Double] Incoming data.
+ * @return Array[Double]. The deciles of the distribution.
+ */
+
+ def computeDeciles(data: RDD[Double]): Array[Double] = computeQuantiles(data, DECILES)
+
+ /**
+ * Compute the quintiles of a distribution.
+ *
+ * @param data RDD[Double] Incoming data.
+ * @return Array[Double]. The quintiles of the distribution.
+ */
+ def computeQuintiles(data: RDD[Double]): Array[Double] = computeQuantiles(data, QUINTILES)
+
+ /**
+ * Compute the quantiles for a given dataset and array of thresholds for the cumulative distribution.
+ *
+ * @param data Incoming dataset.
+ * @param quantiles Sorted array of doubles in the range from 0.0 to 1.0.
+ * @return The quantiles of the data, that is, if t is the ith entry of the quantiles array, and x is the
+ * ith entry of returned quantiles array, then x is the minimum value in the input dataset so that
+ * a >= t fraction of the mass of the input is <= x.
+ * IE. x is the least x so that Pr ( X <= x ) >= t
+ * In the case of an empty input dataset, the quantile cutoffs returned are all positive infinity.
+ */
+
+ def computeQuantiles(data: RDD[Double], quantiles: Array[Double]): Array[Double] = {
+
+ /*
+ * Calculation is based on the fact that the quantile of a threshold t is the minimum x in the dataset so
+ * that the empirical cumulative distribution at x is >= t.
+ */
+
+ def addDataPointToKnownCutoffs(cutoffs: Array[Double], newValueCDFPair: (Double, Double)): Array[Double] = {
+ val newPoint = newValueCDFPair._1
+ val cdfAtNewPoint = newValueCDFPair._2
+ quantiles.zip(cutoffs).map({ case (threshold, oldCutoff) =>
+ if (cdfAtNewPoint >= threshold && newPoint <= oldCutoff) newPoint else oldCutoff
+ })
+ }
+
+ def mergeCutoffs(cuts1: Array[Double], cuts2: Array[Double]) = cuts1.zip(cuts2).map({ case (x, y) => min(x, y) })
+
+ // Initial cutoffs are the trivial "positive infinity" cutoffs. These are the correct values for an empty data set.
+ val initialCutoffs = Array.fill[Double](quantiles.length)(Double.PositiveInfinity)
+
+ computeEcdf(data).aggregate(initialCutoffs)(addDataPointToKnownCutoffs, mergeCutoffs)
+ }
+
+ /**
+ * Compute the empirical cumulative distribution function,
+ * cf. [[https://en.wikipedia.org/wiki/Empirical_distribution_function]]
+ *
+ * @param data An RDD of doubles.
+ * @return RDD[(Double,Double)] where each pair is of the form (value, ecdf at value)
+ * That is, each pair is a value and the fraction of the input data less-than-or-equal to the value.
+ */
+
+ def computeEcdf(data: RDD[Double]): RDD[(Double, Double)] = {
+
+ // pair each distinct value with the number of times it appears in the data, then sort by the data values
+ val valueCountPairs = data.map(v => (v, 1L)).reduceByKey(_ + _).sortByKey().persist()
+
+ // for each partition, sum the counts of all values in the partition
+ val countsPerPartition: Array[Double] = valueCountPairs.mapPartitionsWithIndex {
+ case (_, partition) => Iterator(partition.map({ case (_, count) => count }).sum.toDouble)
+ }.collect()
+
+ val totalCount = countsPerPartition.sum
+
+ // pair each value v with the sum of counts of all values <= v in the data
+ val valueCountLEQPairs : RDD[(Double, Double)] = valueCountPairs.mapPartitionsWithIndex {
+ case (index, partition) =>
+ val countInPrecedingPartitions = countsPerPartition.take(index).sum
+ val p = partition.scanLeft((0.0, countInPrecedingPartitions))({ case ((_, countOfLEQValues), (value, countOfValue)) =>
+ (value, countOfLEQValues + countOfValue)})
+
+ // first element is an extraneous zero and must be dropped
+ p.drop(1)
+ }
+ valueCountPairs.unpersist()
+ // normalize counts by total number entries in the data to obtain the ecdf
+ valueCountLEQPairs.map({case (value, countToLeftOfValue) => (value, countToLeftOfValue / totalCount)})
+ }
+
+ def bin(value: Double, cuts: Array[Double]) : Int = {
+ cuts.indexWhere(cut => value <= cut)
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/TimeUtilities.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TimeUtilities.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TimeUtilities.scala
new file mode 100644
index 0000000..830e93e
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TimeUtilities.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+object TimeUtilities {
+
+
+ /**
+ * It converts HH:MM:SS string to seconds
+ *
+ * @param timeStr This is time in the form of a string
+ * @return It returns time converted to seconds
+ */
+
+ def getTimeAsDouble(timeStr: String) : Double = {
+ val s = timeStr.split(":")
+ val hours = s(0).toInt
+ val minutes = s(1).toInt
+ val seconds = s(2).toInt
+
+ (3600*hours + 60*minutes + seconds).toDouble
+ }
+
+ /**
+ * It takes only the hour element of time
+ *
+ * @param timeStr This is time in the form of a string
+ * @return It returns only the hour of time
+ */
+ def getTimeAsHour(timeStr: String): Int = {
+ val s = timeStr.split(":")
+ val hours = s(0).toInt
+ hours
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
new file mode 100644
index 0000000..089a121
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import scala.io.Source
+
+
+object TopDomains {
+ val alexaTop1MPath = "top-1m.csv"
+
+ val TopDomains: Set[String] = Source.fromFile(alexaTop1MPath).getLines.map(line => {
+ val parts = line.split(",")
+ val l = parts.length
+ parts(1).split('.')(0)
+ }).toSet
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/CountryCodes.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/CountryCodes.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/CountryCodes.scala
deleted file mode 100644
index 9f166ba..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/CountryCodes.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-object CountryCodes {
-
- val CountryCodes = Set("ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "as", "at", "au",
- "aw", "ax", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bt",
- "bv", "bw", "by", "bz", "ca", "cc", "cd", "cf", "cg", "ch", "ci", "ck", "cl", "cm", "cn", "co", "cr", "cu", "cv",
- "cw", "cx", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "eh", "er", "es", "et", "eu", "fi",
- "fj", "fk", "fm", "fo", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gi", "gl", "gm", "gn", "gp", "gq", "gr",
- "gs", "gt", "gu", "gw", "gy", "hk", "hm", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "io", "iq", "ir",
- "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr", "krd", "kw", "ky", "kz", "la",
- "lb", "lc", "li", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mg", "mh", "mk", "ml", "mm",
- "mn", "mo", "mp", "mq", "mr", "ms", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "nf", "ng", "ni",
- "nl", "no", "np", "nr", "nu", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pm", "pn", "pr", "ps", "pt",
- "pw", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sb", "sc", "sd", "se", "sg", "sh", "si", "sj", "", "sk",
- "sl", "sm", "sn", "so", "sr", "ss", "st", "su", "sv", "sx", "sy", "sz", "tc", "td", "tf", "tg", "th", "tj", "tk",
- "tl", "tm", "tn", "to", "tp", "tr", "tt", "tv", "tw", "tz", "ua", "ug", "uk", "us", "uy", "uz", "va", "vc", "ve",
- "vg", "vi", "vn", "vu", "wf", "ws", "ye", "yt", "za", "zm", "zw")
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/DomainProcessor.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/DomainProcessor.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/DomainProcessor.scala
deleted file mode 100644
index efa2e8f..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/DomainProcessor.scala
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.broadcast.Broadcast
-
-/**
- * Routines and data for processing URLs for domains, subdomains, country code, top-level domains, etc.
- */
-object DomainProcessor extends Serializable {
-
- val CountryCodes = Set("ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "as", "at", "au",
- "aw", "ax", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bt",
- "bv", "bw", "by", "bz", "ca", "cc", "cd", "cf", "cg", "ch", "ci", "ck", "cl", "cm", "cn", "co", "cr", "cu", "cv",
- "cw", "cx", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "eh", "er", "es", "et", "eu", "fi",
- "fj", "fk", "fm", "fo", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gi", "gl", "gm", "gn", "gp", "gq", "gr",
- "gs", "gt", "gu", "gw", "gy", "hk", "hm", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "io", "iq", "ir",
- "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr", "krd", "kw", "ky", "kz", "la",
- "lb", "lc", "li", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mg", "mh", "mk", "ml", "mm",
- "mn", "mo", "mp", "mq", "mr", "ms", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "nf", "ng", "ni",
- "nl", "no", "np", "nr", "nu", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pm", "pn", "pr", "ps", "pt",
- "pw", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sb", "sc", "sd", "se", "sg", "sh", "si", "sj", "", "sk",
- "sl", "sm", "sn", "so", "sr", "ss", "st", "su", "sv", "sx", "sy", "sz", "tc", "td", "tf", "tg", "th", "tj", "tk",
- "tl", "tm", "tn", "to", "tp", "tr", "tt", "tv", "tw", "tz", "ua", "ug", "uk", "us", "uy", "uz", "va", "vc", "ve",
- "vg", "vi", "vn", "vu", "wf", "ws", "ye", "yt", "za", "zm", "zw")
-
- val TopLevelDomainNames = Set("com", "org", "net", "int", "edu", "gov", "mil")
- val None = "None"
-
- /**
- * Extract domain info from a url.
- * @param url Incoming url.
- * @param topDomainsBC Broadcast variable containing the top domains set.
- * @param userDomain Domain of the spot user (example:'intel').
- * @return New [[DomainInfo]] object containing extracted domain information.
- */
- def extractDomainInfo(url: String, topDomainsBC: Broadcast[Set[String]], userDomain: String): DomainInfo = {
-
- val spliturl = url.split('.')
- val numParts = spliturl.length
-
- val (domain, subdomain) = extractDomainSubdomain(url)
-
-
- val subdomainLength = if (subdomain != None) {
- subdomain.length
- } else {
- 0
- }
-
- val topDomainClass = if (userDomain != "" && domain == userDomain) {
- 2
- } else if (topDomainsBC.value contains domain) {
- 1
- } else {
- 0
- }
-
- val subdomainEntropy = if (subdomain != "None") Entropy.stringEntropy(subdomain) else 0d
-
- DomainInfo(domain, topDomainClass, subdomain, subdomainLength, subdomainEntropy, numParts)
- }
-
- /**
- *
- * @param url Url from which to extract domain.
- * @return Domain name or "None" if there is none.
- */
- def extractDomain(url: String) : String = {
- val (domain, _) = extractDomainSubdomain(url)
- domain
- }
-
- /**
- * Extrat the domain and subdomain from a URL.
- * @param url URL to be parsed.
- * @return Pair of (domain, subdomain). If there is no domain, both fields contain "None".
- * If there is no subdomain then the subdomain field is "None"
- */
- def extractDomainSubdomain(url: String) : (String, String) = {
- val spliturl = url.split('.')
- val numParts = spliturl.length
-
-
- var domain = None
- var subdomain = None
- // First check if query is an IP address e.g.: 123.103.104.10.in-addr.arpa or a name.
- // Such URLs receive a domain of NO_DOMAIN
-
- if (numParts >= 2
- && !(numParts > 2 && spliturl(numParts - 1) == "arpa" && spliturl(numParts - 2) == "in-addr")
- && (CountryCodes.contains(spliturl.last) || TopLevelDomainNames.contains(spliturl.last))) {
- val strippedSplitURL = removeTopLevelDomainName(removeCountryCode(spliturl))
- if (strippedSplitURL.length > 0) {
- domain = strippedSplitURL.last
- if (strippedSplitURL.length > 1) {
- subdomain = strippedSplitURL.slice(0, strippedSplitURL.length - 1).mkString(".")
- }
- }
- }
-
-
- (domain, subdomain)
- }
-
- /**
- * Strip the country code from a split URL.
- * @param urlComponents Array of the entries of a URL after splitting on periods.
- * @return URL components with the country code stripped.
- */
- def removeCountryCode(urlComponents: Array[String]): Array[String] = {
- if (CountryCodes.contains(urlComponents.last)) {
- urlComponents.dropRight(1)
- } else {
- urlComponents
- }
- }
-
- /**
- * Strip the top-level domain name from a split URL.
- * @param urlComponents Array of the entries ofa URL after splitting on periods.
- * @return URL components with the top-level domain name stripped.
- */
- def removeTopLevelDomainName(urlComponents: Array[String]): Array[String] = {
- if (TopLevelDomainNames.contains(urlComponents.last)) {
- urlComponents.dropRight(1)
- } else {
- urlComponents
- }
- }
-
- /**
- * Commonly extracted domain features.
- *
- * @param domain Domain (if any) of a url.
- * @param topDomain Numerical class of domain: 2 for Intel, 1 for Alexa top domains, 0 for others.
- * @param subdomain Subdomain (if any) in the url.
- * @param subdomainLength Length of the subdomain. 0 if there is none.
- * @param subdomainEntropy Entropy of the subdomain viewed as a distribution on its character set.
- * 0 if there is no subdomain.
- * @param numPeriods Number of periods + 1 in the url. (Number of sub-strings where url is split by periods.)
- */
- case class DomainInfo(domain: String,
- topDomain: Int,
- subdomain: String,
- subdomainLength: Int,
- subdomainEntropy: Double,
- numPeriods: Int)
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Entropy.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Entropy.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Entropy.scala
deleted file mode 100644
index cc7becf..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Entropy.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import scala.math._
-
-/**
- * Created by nlsegerl on 8/4/16.
- */
-object Entropy {
-
- /**
- * Calculates the "entropy" in string, interpreted as a distribution on its constituent characters.
- *
- * @param v Input string
- * @return
- */
- def stringEntropy(v: String): Double = {
- if (v.length() > 0) {
- v.groupBy(a => a).values.map(i => i.length.toDouble / v.length).map(p => -p * log10(p) / log10(2)).sum
- } else {
- 0
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
deleted file mode 100644
index 4b85787..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/PrecisionUtility.scala
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.functions._
-
-/**
- * PrecisionUtility will transform a number from Double to Float if precision option is set to 32 bit,
- * if default or 64 bit is selected, it will just return the same number type Double.
- *
- * This abstract class permits the execution of a single path during the entire analysis. Instead of checking what
- * type should the data be converted, each implementation will know what type should be converted into and converted
- * back.
- *
- */
-
-sealed trait PrecisionUtility extends Serializable {
-
- type TargetType
-
- /**
- * Converts a number into the precision type; it can be Float (32) or Double (64).
- * For the Double implementation it will just return the same value without any transformation.
- *
- * @param double a number to convert from Double to Target type.
- * @return
- */
- def toTargetType(double: Double): TargetType
-
- /**
- * Converts back an Iterable of numbers from TargetType to Double. <% determines that it can process any object that
- * extends from Iterable i.e. List, Vector, Seq, Queue, etc.
- *
- * @param targetTypeIterable an iterable that needs to be converted each element from TargetType to Double
- * @return
- */
- def toDoubles[A <% Traversable[TargetType], B <% Traversable[Double]](targetTypeIterable: A): B
-
- /**
- * Converts a DataFrame column from Seq[Double] to a Seq[TargetType]. If the TargetType is Double, it will
- * just return the same DataFrame without any transformation.
- *
- * @param dataFrame a DataFrame containing a column to be converted from Double to the TargetType
- * @param columnName the name of the column to convert, the column should be Seq[Double]
- * @return
- */
- def castColumn(dataFrame: DataFrame, columnName: String): DataFrame
-
-}
-
-/**
- * PrecisionUtility implementation for Float.
- * Will convert numbers from Double to Float and back to Double to make the workload half the size. This conversion
- * will allow to broadcast dictionaries if it fits in the autoBroadcastJoinThreshold - defined by users.
- */
-object PrecisionUtilityFloat extends PrecisionUtility {
-
- type TargetType = Float
-
- def toTargetType(double: Double): Float = double.toFloat
-
- def toDoubles[A <% Traversable[Float], B <% Traversable[Double]](targetTypeIterable: A): B =
- targetTypeIterable.map(_.toDouble).asInstanceOf[B]
-
-
- private val convertUDF = udf((doubles: Seq[Double]) => {
- doubles.map(double => toTargetType(double))
- })
-
- def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = {
- val TempColumn = "temp_column"
-
- dataFrame.withColumn(TempColumn, convertUDF(dataFrame(columnName)))
- .drop(columnName)
- .withColumnRenamed(TempColumn, columnName)
- }
-
-}
-
-/**
- * PrecisionUtility implementation for Double.
- * Users that don't want to reduce the workload, can continue working with Doubles. This implementation will receive
- * and send the same value, it won't do any transformation. The reason for this to exists is to avoid multiple code
- * paths depending on the user wanting to change precision or not.
- */
-object PrecisionUtilityDouble extends PrecisionUtility {
-
- type TargetType = Double
-
- def toTargetType(double: Double): Double = double
-
- // Since Double is the default data type, this code is going to return the same array without any transformation.
- def toDoubles[A <% Traversable[Double], B <% Traversable[Double]](targetTypeIterable: A): B =
- targetTypeIterable.asInstanceOf[B]
-
- // Since Double is the default data type, this code won't actually do any calculation but instead just return the
- // same DataFrame.
- def castColumn(dataFrame: DataFrame, columnName: String): DataFrame = dataFrame
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Quantiles.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Quantiles.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Quantiles.scala
deleted file mode 100644
index 20ccd40..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/Quantiles.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.rdd.RDD
-
-import scala.math._
-
-/**
- * Contains routines for the distributed calculation of quantiles and the empirical cumulative distribution function.
- */
-
-object Quantiles extends Serializable {
-
- val DECILES = Array(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
- val QUINTILES = Array(0.2, 0.4, 0.6, 0.8, 1.0)
-
- /**
- * Compute the deciles of a distribution.
- *
- * @param data RDD[Double] Incoming data.
- * @return Array[Double]. The deciles of the distribution.
- */
-
- def computeDeciles(data: RDD[Double]): Array[Double] = computeQuantiles(data, DECILES)
-
- /**
- * Compute the quintiles of a distribution.
- *
- * @param data RDD[Double] Incoming data.
- * @return Array[Double]. The quintiles of the distribution.
- */
- def computeQuintiles(data: RDD[Double]): Array[Double] = computeQuantiles(data, QUINTILES)
-
- /**
- * Compute the quantiles for a given dataset and array of thresholds for the cumulative distribution.
- *
- * @param data Incoming dataset.
- * @param quantiles Sorted array of doubles in the range from 0.0 to 1.0.
- * @return The quantiles of the data, that is, if t is the ith entry of the quantiles array, and x is the
- * ith entry of returned quantiles array, then x is the minimum value in the input dataset so that
- * a >= t fraction of the mass of the input is <= x.
- * IE. x is the least x so that Pr ( X <= x ) >= t
- * In the case of an empty input dataset, the quantile cutoffs returned are all positive infinity.
- */
-
- def computeQuantiles(data: RDD[Double], quantiles: Array[Double]): Array[Double] = {
-
- /*
- * Calculation is based on the fact that the quantile of a threshold t is the minimum x in the dataset so
- * that the empirical cumulative distribution at x is >= t.
- */
-
- def addDataPointToKnownCutoffs(cutoffs: Array[Double], newValueCDFPair: (Double, Double)): Array[Double] = {
- val newPoint = newValueCDFPair._1
- val cdfAtNewPoint = newValueCDFPair._2
- quantiles.zip(cutoffs).map({ case (threshold, oldCutoff) =>
- if (cdfAtNewPoint >= threshold && newPoint <= oldCutoff) newPoint else oldCutoff
- })
- }
-
- def mergeCutoffs(cuts1: Array[Double], cuts2: Array[Double]) = cuts1.zip(cuts2).map({ case (x, y) => min(x, y) })
-
- // Initial cutoffs are the trivial "positive infinity" cutoffs. These are the correct values for an empty data set.
- val initialCutoffs = Array.fill[Double](quantiles.length)(Double.PositiveInfinity)
-
- computeEcdf(data).aggregate(initialCutoffs)(addDataPointToKnownCutoffs, mergeCutoffs)
- }
-
- /**
- * Compute the empirical cumulative distribution function,
- * cf. [[https://en.wikipedia.org/wiki/Empirical_distribution_function]]
- *
- * @param data An RDD of doubles.
- * @return RDD[(Double,Double)] where each pair is of the form (value, ecdf at value)
- * That is, each pair is a value and the fraction of the input data less-than-or-equal to the value.
- */
-
- def computeEcdf(data: RDD[Double]): RDD[(Double, Double)] = {
-
- // pair each distinct value with the number of times it appears in the data, then sort by the data values
- val valueCountPairs = data.map(v => (v, 1L)).reduceByKey(_ + _).sortByKey().persist()
-
- // for each partition, sum the counts of all values in the partition
- val countsPerPartition: Array[Double] = valueCountPairs.mapPartitionsWithIndex {
- case (_, partition) => Iterator(partition.map({ case (_, count) => count }).sum.toDouble)
- }.collect()
-
- val totalCount = countsPerPartition.sum
-
- // pair each value v with the sum of counts of all values <= v in the data
- val valueCountLEQPairs : RDD[(Double, Double)] = valueCountPairs.mapPartitionsWithIndex {
- case (index, partition) =>
- val countInPrecedingPartitions = countsPerPartition.take(index).sum
- val p = partition.scanLeft((0.0, countInPrecedingPartitions))({ case ((_, countOfLEQValues), (value, countOfValue)) =>
- (value, countOfLEQValues + countOfValue)})
-
- // first element is an extraneous zero and must be dropped
- p.drop(1)
- }
- valueCountPairs.unpersist()
- // normalize counts by total number entries in the data to obtain the ecdf
- valueCountLEQPairs.map({case (value, countToLeftOfValue) => (value, countToLeftOfValue / totalCount)})
- }
-
- def bin(value: Double, cuts: Array[Double]) : Int = {
- cuts.indexWhere(cut => value <= cut)
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TimeUtilities.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TimeUtilities.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TimeUtilities.scala
deleted file mode 100644
index 1318dbb..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TimeUtilities.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-object TimeUtilities {
-
-
- /**
- * It converts HH:MM:SS string to seconds
- *
- * @param timeStr This is time in the form of a string
- * @return It returns time converted to seconds
- */
-
- def getTimeAsDouble(timeStr: String) : Double = {
- val s = timeStr.split(":")
- val hours = s(0).toInt
- val minutes = s(1).toInt
- val seconds = s(2).toInt
-
- (3600*hours + 60*minutes + seconds).toDouble
- }
-
- /**
- * It takes only the hour element of time
- *
- * @param timeStr This is time in the form of a string
- * @return It returns only the hour of time
- */
- def getTimeAsHour(timeStr: String): Int = {
- val s = timeStr.split(":")
- val hours = s(0).toInt
- hours
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TopDomains.scala
deleted file mode 100644
index c0ac20f..0000000
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/transformation/TopDomains.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import scala.io.Source
-
-
-object TopDomains {
- val alexaTop1MPath = "top-1m.csv"
-
- val TopDomains: Set[String] = Source.fromFile(alexaTop1MPath).getLines.map(line => {
- val parts = line.split(",")
- val l = parts.length
- parts(1).split('.')(0)
- }).toSet
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
index 26faa9b..08f782e 100644
--- a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsScoreFunctionTest.scala
@@ -19,7 +19,7 @@ package org.apache.spot
import org.apache.spark.broadcast.Broadcast
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.{PrecisionUtilityDouble, PrecisionUtilityFloat}
+import org.apache.spot.utilities.{FloatPointPrecisionUtility32, FloatPointPrecisionUtility64}
import org.scalatest.Matchers
/**
@@ -33,7 +33,7 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val precisionUtility = PrecisionUtilityDouble
+ val precisionUtility = FloatPointPrecisionUtility64
val documentProbabilities = Seq(0.05d, 0.05d, 0.05d, 0.05d)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
@@ -50,7 +50,7 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val precisionUtility = PrecisionUtilityFloat
+ val precisionUtility = FloatPointPrecisionUtility32
val documentProbabilities = Seq(0.05f, 0.05f, 0.05f, 0.05f)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
@@ -66,7 +66,7 @@ class SuspiciousConnectsScoreFunctionTest extends TestingSparkContextFlatSpec wi
val wordToPerTopicProbBC: Broadcast[Map[String, Array[Double]]] = sparkContext.broadcast(wordToPerTopicProb)
val topicCount = 4
- val precisionUtility = PrecisionUtilityFloat
+ val precisionUtility = FloatPointPrecisionUtility32
val documentProbabilities = Seq(0.05f, 0.05f, 0.05f, 0.05f)
val scoreFunction = new SuspiciousConnectsScoreFunction(topicCount, wordToPerTopicProbBC)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index 4e37656..ef0fce9 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -25,7 +25,7 @@ import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.dns.DNSSchema._
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
+import org.apache.spot.utilities.FloatPointPrecisionUtility32
import org.scalatest.Matchers
@@ -67,7 +67,7 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- precisionUtility = PrecisionUtilityFloat)
+ precisionUtility = FloatPointPrecisionUtility32)
"dns suspicious connects analysis" should "estimate correct probabilities in toy data with framelength anomaly" in {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/dns/DNSWordCreationTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSWordCreationTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSWordCreationTest.scala
index 52710d0..b9e0a2d 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSWordCreationTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSWordCreationTest.scala
@@ -18,7 +18,7 @@
package org.apache.spot.dns
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.Entropy
+import org.apache.spot.utilities.Entropy
import org.scalatest.Matchers
class DNSWordCreationTest extends TestingSparkContextFlatSpec with Matchers {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala b/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
index a3e3bee..e701bb9 100644
--- a/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/lda/SpotLDAWrapperTest.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spot.lda.SpotLDAWrapper._
import org.apache.spot.lda.SpotLDAWrapperSchema._
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.{PrecisionUtilityDouble, PrecisionUtilityFloat}
+import org.apache.spot.utilities.{FloatPointPrecisionUtility32, FloatPointPrecisionUtility64}
import org.scalatest.Matchers
import scala.collection.immutable.Map
@@ -45,7 +45,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, PrecisionUtilityDouble)
+ ldaMaxIterations, FloatPointPrecisionUtility64)
val topicMixDF = out.docToTopicMix
@@ -68,7 +68,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, PrecisionUtilityFloat)
+ ldaMaxIterations, FloatPointPrecisionUtility32)
val topicMixDF = out.docToTopicMix
@@ -90,7 +90,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, PrecisionUtilityDouble)
+ ldaMaxIterations, FloatPointPrecisionUtility64)
val topicMixDF = out.docToTopicMix
val dogTopicMix: Array[Double] =
@@ -116,7 +116,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
val data = sparkContext.parallelize(Seq(catFancy, dogWorld))
val out = SpotLDAWrapper.runLDA(sparkContext, sqlContext, data, 2, logger, Some(0xdeadbeef), ldaAlpha, ldaBeta,
- ldaMaxIterations, PrecisionUtilityFloat)
+ ldaMaxIterations, FloatPointPrecisionUtility32)
val topicMixDF = out.docToTopicMix
val dogTopicMix: Array[Float] =
@@ -179,7 +179,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
(2.toLong, Vectors.dense(0.4, 0.1, 0.3, 0.2))))
val sparkDocRes: DataFrame = formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext,
- PrecisionUtilityDouble)
+ FloatPointPrecisionUtility64)
val documents = sparkDocRes.select(DocumentName).map(documentName => documentName.toString.replaceAll("\\[", "")
.replaceAll("\\]", "")).collect()
@@ -213,7 +213,7 @@ class SpotLDAWrapperTest extends TestingSparkContextFlatSpec with Matchers {
(2.toLong, Vectors.dense(0.4, 0.1, 0.3, 0.2))))
val sparkDocRes: DataFrame = formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext,
- PrecisionUtilityFloat)
+ FloatPointPrecisionUtility32)
val documents = sparkDocRes.select(DocumentName).map(documentName => documentName.toString.replaceAll("\\[", "")
.replaceAll("\\]", "")).collect()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index 4abda8f..abaed97 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -25,7 +25,7 @@ import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.FlowSuspiciousConnectsAnalysis.InSchema
import org.apache.spot.netflow.model.FlowSuspiciousConnectsModel
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
+import org.apache.spot.utilities.FloatPointPrecisionUtility32
import org.scalatest.Matchers
@@ -60,7 +60,7 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- precisionUtility = PrecisionUtilityFloat)
+ precisionUtility = FloatPointPrecisionUtility32)
"netflow suspicious connects" should "correctly identify time-of-day anomalies" in {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index 2788818..189c96d 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.proxy.ProxySchema._
import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation.PrecisionUtilityFloat
+import org.apache.spot.utilities.FloatPointPrecisionUtility32
import org.scalatest.Matchers
case class ProxyInput(p_date: String,
@@ -76,7 +76,7 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
ldaMaxiterations = 20,
ldaAlpha = 1.02,
ldaBeta = 1.001,
- precisionUtility = PrecisionUtilityFloat)
+ precisionUtility = FloatPointPrecisionUtility32)
"proxy supicious connects analysis" should "estimate correct probabilities in toy data with top domain anomaly" in {