You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/07/13 22:18:06 UTC
[4/7] incubator-spot git commit: Deleted unused code in
src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala.
Deleted unused code in src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/156f571b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/156f571b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/156f571b
Branch: refs/heads/SPOT-35_graphql_api
Commit: 156f571b1edcca550dc74c4406d03b38b308a24c
Parents: e06c7ca
Author: Brandon Edwards <br...@intel.com>
Authored: Fri Jul 7 15:53:48 2017 -0700
Committer: Ricardo Barona <ri...@intel.com>
Committed: Wed Jul 12 12:28:37 2017 -0500
----------------------------------------------------------------------
.../dns/model/DNSSuspiciousConnectsModel.scala | 28 --------------------
1 file changed, 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/156f571b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
index 3089bc8..dfcb543 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
@@ -159,12 +159,8 @@ object DNSSuspiciousConnectsModel {
val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
val userDomain = config.userDomain
- val domainStatsRecords = createDomainStatsDF(sparkSession, countryCodesBC, topDomainsBC, userDomain, totalRecords)
-
// simplify DNS log entries into "words"
-
val dnsWordCreator = new DNSWordCreation(topDomainsBC, userDomain)
-
val dataWithWord = totalRecords.withColumn(Word, dnsWordCreator.wordCreationUDF(modelColumns: _*))
import sparkSession.implicits._
@@ -195,30 +191,6 @@ object DNSSuspiciousConnectsModel {
}
- /**
- * Add domain statistics fields to a data frame.
- *
- * @param sparkSession Spark Session
- * @param countryCodesBC Broadcast of the country codes set.
- * @param topDomainsBC Broadcast of the most-popular domains set.
- * @param userDomain Domain associated to network data (ex: 'intel')
- * @param inDF Incoming dataframe. Schema is expected to provide the field [[QueryName]]
- * @return A new dataframe with the new columns added. The new columns have the schema [[DomainStatsSchema]]
- */
-
- def createDomainStatsDF(sparkSession: SparkSession,
- countryCodesBC: Broadcast[Set[String]],
- topDomainsBC: Broadcast[Set[String]],
- userDomain: String,
- inDF: DataFrame): DataFrame = {
-
- val queryNameIndex = inDF.schema.fieldNames.indexOf(QueryName)
-
- val domainStatsRDD: RDD[Row] = inDF.rdd.map(row =>
- Row.fromTuple(createTempFields(countryCodesBC, topDomainsBC, userDomain, row.getString(queryNameIndex))))
-
- sparkSession.createDataFrame(domainStatsRDD, DomainStatsSchema)
- }
/**
*