You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 22:41:05 UTC
[02/50] [abbrv] incubator-spot git commit: Spot-197: Removing
top-1m.csv file. Removed top-1m.csv file as per SPOT-197,
the file should not be in repository. Modified build.sbt to download
top-1m.csv file in compilation time. Changed the location of top-
Spot-197: Removing top-1m.csv file.
Removed top-1m.csv file as per SPOT-197, the file should not be in repository.
Modified build.sbt to download top-1m.csv file in compilation time.
Changed the location of top-1m.csv file to top-1m/ folder.
Modified TopDomains.scala object to reflect the new location of top-1m.csv.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0fa4e5cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0fa4e5cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0fa4e5cd
Branch: refs/heads/SPOT-181_ODM
Commit: 0fa4e5cd95c29a926958f76be885d19e6ccea846
Parents: 156f571
Author: Ricardo Barona <ri...@intel.com>
Authored: Wed Jul 12 10:38:31 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Wed Jul 12 13:03:28 2017 -0500
----------------------------------------------------------------------
spot-ml/build.sbt | 18 +-
.../org/apache/spot/utilities/TopDomains.scala | 3 +-
spot-ml/top-1m.csv | 1000000 ---------------
3 files changed, 19 insertions(+), 1000002 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fa4e5cd/spot-ml/build.sbt
----------------------------------------------------------------------
diff --git a/spot-ml/build.sbt b/spot-ml/build.sbt
index 0ac6e60..5ffc520 100644
--- a/spot-ml/build.sbt
+++ b/spot-ml/build.sbt
@@ -53,4 +53,20 @@ mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) => {
}
// super important with multiple tests running spark Contexts
-parallelExecution in Test := false
\ No newline at end of file
+parallelExecution in Test := false
+
+lazy val getTop1MFileFromAlexa = taskKey[Seq[File]]("Download Amazon/alexa-static top-1m.csv file for DNS and Proxy")
+
+getTop1MFileFromAlexa := {
+ if (!java.nio.file.Files.exists(new File("top-1m/top-1m.csv").toPath())) {
+ val location = url("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip")
+ println("Getting top-1m.csv from Amazon/alexa-static")
+ IO.unzipURL(location, new File("top-1m")).toSeq
+ }
+ else {
+ println("File top-1m.csv already provided")
+ Seq.empty[File]
+ }
+}
+
+resourceGenerators in Compile <+= getTop1MFileFromAlexa
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0fa4e5cd/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
index b9592ea..0183027 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
@@ -23,7 +23,8 @@ import scala.io.Source
* List of top domains used for DNS and Proxy analysis.
*/
object TopDomains {
- val alexaTop1MPath = "top-1m.csv"
+ // Auto-generated by SBT at compile time
+ val alexaTop1MPath = "top-1m/top-1m.csv"
val TopDomains: Set[String] = Source.fromFile(alexaTop1MPath).getLines.map(line => {
val parts = line.split(",")