You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ra...@apache.org on 2017/06/23 16:40:05 UTC
[35/50] [abbrv] incubator-spot git commit: Fixed errors after rebase
with incubator-spot/master
Fixed errors after rebase with incubator-spot/master
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0fbea08f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0fbea08f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0fbea08f
Branch: refs/heads/SPOT-35_graphql_api
Commit: 0fbea08f63b30addf9e8e465e0a0f03aa4a3b30f
Parents: 9a71071
Author: Ricardo Barona <ri...@intel.com>
Authored: Wed Jun 14 11:18:33 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Fri Jun 16 17:55:10 2017 -0500
----------------------------------------------------------------------
.../netflow/model/FlowSuspiciousConnectsModel.scala | 2 +-
.../spot/dns/DNSSuspiciousConnectsAnalysisTest.scala | 5 ++++-
.../netflow/FlowSuspiciousConnectsAnalysisTest.scala | 11 +++++------
.../proxy/ProxySuspiciousConnectsAnalysisTest.scala | 13 +++----------
4 files changed, 13 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fbea08f/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
index da13b8e..e053437 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
@@ -189,7 +189,7 @@ object FlowSuspiciousConnectsModel {
config.ldaMaxiterations,
config.precisionUtility)
- new FlowSuspiciousConnectsModel(topicCount,
+ new FlowSuspiciousConnectsModel(config.topicCount,
ipToTopicMix,
wordToPerTopicProb)
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fbea08f/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index 92aba5c..2435561 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -155,7 +155,8 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
1,
0)
val data = sqlContext.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val scoredData = DNSSuspiciousConnectsAnalysis.scoreDNSRecords(data, emTestConfig, sparkContext, sqlContext, logger)
+ val model = DNSSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, emTestConfig, data)
+ val scoredData = model.score(sparkContext, sqlContext, data, emTestConfig.userDomain)
val anomalyScore = scoredData.
filter(scoredData(QueryName) === "1111111111111111111111111111111111111111111111111111111111111.tinker.turner.com").
first().
@@ -195,6 +196,8 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
1,
0)
val data = sqlContext.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
+ val model = DNSSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, onlineTestConfig, data)
+ val scoredData = model.score(sparkContext, sqlContext, data, onlineTestConfig.userDomain)
val model = DNSSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, testConfig, data)
val scoredData = model.score(sparkContext, sqlContext, data, testConfig.userDomain, testConfig.precisionUtility)
val anomalyScore = scoredData.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fbea08f/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index eea218c..397d938 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -92,7 +92,7 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
val model =
- FlowSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, testConfig, data)
+ FlowSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, emTestConfig, data)
val scoredData = model.score(sparkContext, sqlContext, data, testConfig.precisionUtility)
@@ -125,11 +125,10 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
val data = sqlContext.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val scoredData: DataFrame = FlowSuspiciousConnectsAnalysis.detectFlowAnomalies(data,
- onlineTestConfig,
- sparkContext,
- sqlContext,
- logger)
+ val model =
+ FlowSuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, onlineTestConfig, data)
+
+ val scoredData = model.score(sparkContext, sqlContext, data)
val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fbea08f/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index 5ed5673..f7b2877 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -106,12 +106,9 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val data = sqlContext.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val model = ProxySuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, testConfigProxy, data)
- val scoredData = ProxySuspiciousConnectsAnalysis.detectProxyAnomalies(data, emTestConfigProxy,
- sparkContext,
- sqlContext,
- logger)
+ val model = ProxySuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, emTestConfigProxy, data)
+ val scoredData = model.score(sparkContext, data)
val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))
@@ -153,11 +150,7 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val data = sqlContext.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val scoredData = ProxySuspiciousConnectsAnalysis.detectProxyAnomalies(data, onlineTestConfigProxy,
- sparkContext,
- sqlContext,
- logger)
-
+ val model = ProxySuspiciousConnectsModel.trainModel(sparkContext, sqlContext, logger, onlineTestConfigProxy, data)
val scoredData = model.score(sparkContext, data, testConfigProxy.precisionUtility)