You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/15 21:56:39 UTC
[09/13] incubator-spot git commit: Reverting utilities.transformation
package. Changed name of PrecisionUtility to FloatingPointPrecisionUtility.
Minor documentation fixes.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/DomainProcessorTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/DomainProcessorTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/DomainProcessorTest.scala
new file mode 100644
index 0000000..692b1cd
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/DomainProcessorTest.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.apache.spot.utilities
+import org.apache.spot.utilities.DomainProcessor._
+import org.scalatest.Matchers
+
+
+class DomainProcessorTest extends TestingSparkContextFlatSpec with Matchers {
+ val countryCodesSet = utilities.CountryCodes.CountryCodes
+
+
+ "extractDomain" should "return domain when provided a url with top-level domain and country code" in {
+ val url = "fatosdesconhecidos.com.br"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("fatosdesconhecidos")
+ }
+
+ it should "return domain when provided a short url with no top-level domain but only a country code" in {
+ val url = "panasonic.jp"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("panasonic")
+ }
+
+ it should "return domain when provided a long url with no top-level domain but only a country code" in {
+ val url = "get.your.best.electronic.at.panasonic.jp"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("panasonic")
+ }
+
+ it should "return domain when provided a short url with a top-level domain no country code" in {
+ val url = "forrealz.net"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("forrealz")
+ }
+
+ it should "return domain when provided a long url with a top-level domain no country code" in {
+ val url = "wow.its.really.long.super.long.yeah.so.long.long.long.long.forrealz.net"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("forrealz")
+ }
+
+ it should "should return \"None\" when provided an address" in {
+ val url = "123.103.104.10.in-addr.arpa"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("None")
+ }
+
+ it should "return \"None\" when provided a short url with a bad top-level domain / country code" in {
+ val url = "panasonic.c"
+ val result = DomainProcessor.extractDomain(url)
+ result shouldEqual ("None")
+ }
+
+ "extractDomainInfo" should "handle an in-addr.arpa url" in {
+
+ val url = "123.103.104.10.in-addr.arpa"
+
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+
+ val userDomain = "intel"
+
+ // case class DerivedFields(topDomain: String, subdomainLength: Double, subdomainEntropy: Double, numPeriods: Double)
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "None", topDomain = 0, subdomain = "None", subdomainLength = 0, subdomainEntropy = 0, numPeriods = 6)
+ }
+
+ it should "handle an Alexa top 1M domain with a subdomain, top-level domain name and country code" in {
+
+ val url = "services.amazon.com.mx"
+
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+
+ val userDomain = "intel"
+
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "amazon", topDomain = 1, subdomain = "services",
+ subdomainLength = 8, subdomainEntropy = 2.5, numPeriods = 4)
+ }
+
+ it should "handle an Alexa top 1M domain with a top-level domain name and country code but no subdomain" in {
+
+ val url = "amazon.com.mx"
+ val countryCodes = sparkContext.broadcast(countryCodesSet)
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = "intel"
+
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "amazon", subdomain = "None", topDomain = 1, subdomainLength = 0, subdomainEntropy = 0, numPeriods = 3)
+ }
+
+ it should "handle an Alexa top 1M domain with a subdomain and top-level domain name but no country code" in {
+
+ val url = "services.amazon.com"
+ val countryCodes = sparkContext.broadcast(countryCodesSet)
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = "intel"
+
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "amazon", subdomain = "services", topDomain = 1, subdomainLength = 8, subdomainEntropy = 2.5, numPeriods = 3)
+ }
+
+
+ it should "handle an Alexa top 1M domain with no subdomain or country code" in {
+
+ val url = "amazon.com"
+ val countryCodes = sparkContext.broadcast(countryCodesSet)
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = "intel"
+
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "amazon", subdomain = "None", topDomain = 1, subdomainLength = 0, subdomainEntropy = 0, numPeriods = 2)
+ }
+ it should "not identify the domain as the users domain when both are empty strings" in {
+ val url = "ab..com"
+ val countryCodes = sparkContext.broadcast(countryCodesSet)
+ val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
+ val userDomain = ""
+
+ val result = extractDomainInfo(url, topDomains, userDomain)
+
+ result shouldBe DomainInfo(domain = "", subdomain = "ab", topDomain = 0, subdomainLength = 2, subdomainEntropy = 1, numPeriods = 3)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
new file mode 100644
index 0000000..de66876
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
@@ -0,0 +1,49 @@
+package org.apache.spot.utilities
+
+import org.apache.spark.sql.types._
+import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.scalatest.Matchers
+
+/**
+ * Created by rabarona on 5/17/17.
+ */
+class FloatPointPrecisionUtility32Test extends TestingSparkContextFlatSpec with Matchers {
+
+ "toTargetType" should "just return value converted to Float" in {
+ val testValue: Double = 5d
+
+ val result = FloatPointPrecisionUtility32.toTargetType(testValue)
+
+ result shouldBe 5f
+ result shouldBe a[java.lang.Float]
+
+ }
+
+ "toDoubles" should "convert a Seq of Float to Seq of Double" in {
+
+ val testSeq: Seq[Float] = Seq(1f, 2f, 3f)
+
+ val result: Seq[Double] = FloatPointPrecisionUtility32.toDoubles(testSeq)
+
+ result(0) shouldBe a[java.lang.Double]
+ result(1) shouldBe a[java.lang.Double]
+ result(2) shouldBe a[java.lang.Double]
+ }
+
+ "castColumn" should "return a data frame with a schema modified, Seq[Float] instead of Seq[Double]" in {
+
+ val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
+ .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
+
+ val result = FloatPointPrecisionUtility32.castColumn(testDataFrame, TopicProbabilityMix)
+ result.count
+
+ val schema = StructType(
+ Array(StructField(DocumentName, StringType, true),
+ StructField(TopicProbabilityMix, ArrayType(FloatType, false), true)))
+
+ result.schema shouldBe schema
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
new file mode 100644
index 0000000..2006fde
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
@@ -0,0 +1,48 @@
+package org.apache.spot.utilities
+
+import org.apache.spark.sql.types._
+import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.scalatest.Matchers
+
+/**
+ * Created by rabarona on 5/17/17.
+ */
+class FloatingPointUtility64 extends TestingSparkContextFlatSpec with Matchers {
+
+ "toTargetType" should "just return the same value with the same type" in {
+ val testValue: Double = 5d
+
+ val result = FloatPointPrecisionUtility64.toTargetType(testValue)
+
+ result shouldBe testValue
+ result shouldBe a[java.lang.Double]
+
+ }
+
+ "toDoubles" should "return an array of the same type" in {
+
+ val testSeq: Seq[Double] = Seq(1d, 2d, 3d)
+
+ val result: Seq[Double] = FloatPointPrecisionUtility64.toDoubles(testSeq)
+
+ result shouldBe a[Seq[Double]]
+ result.length shouldBe 3
+ }
+
+ "castColumn" should "return the exact same data frame" in {
+
+ val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
+ .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
+
+ val result = FloatPointPrecisionUtility64.castColumn(testDataFrame, TopicProbabilityMix)
+
+ val schema = StructType(
+ Array(StructField(DocumentName, StringType, true),
+ StructField(TopicProbabilityMix, ArrayType(DoubleType, false), true)))
+
+ result shouldBe testDataFrame
+ result.schema shouldBe schema
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/QuantilesTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/QuantilesTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/QuantilesTest.scala
new file mode 100644
index 0000000..bf71abe
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/QuantilesTest.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spot.utilities
+
+import org.apache.spark.rdd.RDD
+import org.apache.spot.testutils.TestingSparkContextFlatSpec
+import org.apache.spot.utilities.Quantiles
+import org.scalatest.Matchers
+
+
+class QuantilesTest extends TestingSparkContextFlatSpec with Matchers {
+
+ val allOnes = List(1.0, 1.0, 1.0, 1.0, 1.0)
+ val onesAndTwos = List(1.0, 2.0, 1.0, 2.0)
+ val countToTen = List(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0)
+
+
+ "ecdf" should "work on an empty list" in {
+ val rddIn: RDD[Double] = sparkContext.parallelize(List(), numSlices = 3)
+ val rddOut = Quantiles.computeEcdf(rddIn)
+
+ val out = rddOut.collect()
+
+ out.length shouldBe 0
+ }
+
+ "ecdf" should "work on a constant list" in {
+ val rddIn = sparkContext.parallelize(allOnes, numSlices = 3)
+ val rddOut = Quantiles.computeEcdf(rddIn)
+
+ val out = rddOut.collect()
+
+ out.length shouldBe 1
+ out(0) shouldBe(1.0, 1.0)
+ }
+
+ "ecdf" should "work on a split 50/50 list" in {
+ val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
+ val rddOut = Quantiles.computeEcdf(rddIn)
+
+ val out = rddOut.collect()
+
+ out.length shouldBe 2
+ out(0) shouldBe(1.0, 0.5)
+ out(1) shouldBe(2.0, 1.0)
+ }
+
+ "ecdf" should "work on count-to-ten list" in {
+ val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
+ val rddOut = Quantiles.computeEcdf(rddIn)
+
+ val out = rddOut.collect()
+
+ out.length shouldBe 10
+ out(0) shouldBe(1.0, 0.1)
+ out(1) shouldBe(2.0, 0.2)
+ out(2) shouldBe(3.0, 0.3)
+ out(3) shouldBe(4.0, 0.4)
+ out(4) shouldBe(5.0, 0.5)
+ out(5) shouldBe(6.0, 0.6)
+ out(6) shouldBe(7.0, 0.7)
+ out(7) shouldBe(8.0, 0.8)
+ out(8) shouldBe(9.0, 0.9)
+ out(9) shouldBe(10.0, 1.0)
+ }
+
+ "quantiles" should "work on an empty list" in {
+ val rddIn: RDD[Double] = sparkContext.parallelize(List(), numSlices = 3)
+ val quantiles = Array(0.0, 0.5)
+ val out = Quantiles.computeQuantiles(rddIn, quantiles)
+
+ out.length shouldBe 2
+ out(0) shouldBe Double.PositiveInfinity
+ out(1) shouldBe Double.PositiveInfinity
+ }
+
+ "quantiles" should "work on all ones list" in {
+ val rddIn = sparkContext.parallelize(allOnes, numSlices = 3)
+ val quantiles = Array(0.0, 0.5)
+ val out = Quantiles.computeQuantiles(rddIn, quantiles)
+
+ out.length shouldBe 2
+ out(0) shouldBe 1.0
+ out(1) shouldBe 1.0
+ }
+
+ "quantiles" should "work on a 50/50 1s and 2s list" in {
+ val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
+ val quantiles = Array(0.0, 0.5)
+ val out = Quantiles.computeQuantiles(rddIn, quantiles)
+
+ out.length shouldBe 2
+ out(0) shouldBe 1.0
+ out(1) shouldBe 1.0
+ }
+
+ "quantiles" should "work on a 50/50 1s and 2s list with unbalanced thresholds" in {
+ val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
+ val quantiles = Array(0.0, 0.6, 0.99)
+ val out = Quantiles.computeQuantiles(rddIn, quantiles)
+
+ out.length shouldBe 3
+ out(0) shouldBe 1.0
+ out(1) shouldBe 2.0
+ out(2) shouldBe 2.0
+ }
+
+ "quantiles" should "work on a count-to-ten list" in {
+ val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
+ val quantiles = Array(0.0, 0.5)
+ val out = Quantiles.computeQuantiles(rddIn, quantiles)
+
+ out.length shouldBe 2
+ out(0) shouldBe 1.0
+ out(1) shouldBe 5.0
+
+ }
+
+ "deciles" should "work on a count-to-ten list" in {
+ val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
+ val out = Quantiles.computeDeciles(rddIn)
+
+ out.length shouldBe 10
+ out(0) shouldBe 1.0
+ out(1) shouldBe 2.0
+ out(2) shouldBe 3.0
+ out(3) shouldBe 4.0
+ out(4) shouldBe 5.0
+ out(5) shouldBe 6.0
+ out(6) shouldBe 7.0
+ out(7) shouldBe 8.0
+ out(8) shouldBe 9.0
+ out(9) shouldBe 10.0
+ }
+
+ "quintiles" should "work on a count-to-ten list" in {
+ val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
+ val out = Quantiles.computeQuintiles(rddIn)
+
+ out.length shouldBe 5
+ out(0) shouldBe 2.0
+ out(1) shouldBe 4.0
+ out(2) shouldBe 6.0
+ out(3) shouldBe 8.0
+ out(4) shouldBe 10.0
+ }
+
+ "bin" should "return 3 when the value is not bigger than the fourth quintile" in {
+ val quintiles = Array(1.0, 2.0, 3.0, 4.0, 5.0)
+
+ val result = Quantiles.bin(3.5, quintiles)
+
+ result shouldBe 3
+ }
+
+ it should "return 0 when the value is less than the first quintile" in {
+ val quintiles = Array(1.0, 2.0, 3.0, 4.0, 5.0)
+
+ val result = Quantiles.bin(0.0, quintiles)
+
+ result shouldBe 0
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/DomainProcessorTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/DomainProcessorTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/DomainProcessorTest.scala
deleted file mode 100644
index 0bd50ac..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/DomainProcessorTest.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities.transformation
-
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.transformation
-import org.apache.spot.utilities.transformation.DomainProcessor._
-import org.scalatest.Matchers
-
-
-class DomainProcessorTest extends TestingSparkContextFlatSpec with Matchers {
- val countryCodesSet = transformation.CountryCodes.CountryCodes
-
-
- "extractDomain" should "return domain when provided a url with top-level domain and country code" in {
- val url = "fatosdesconhecidos.com.br"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("fatosdesconhecidos")
- }
-
- it should "return domain when provided a short url with no top-level domain but only a country code" in {
- val url = "panasonic.jp"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("panasonic")
- }
-
- it should "return domain when provided a long url with no top-level domain but only a country code" in {
- val url = "get.your.best.electronic.at.panasonic.jp"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("panasonic")
- }
-
- it should "return domain when provided a short url with a top-level domain no country code" in {
- val url = "forrealz.net"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("forrealz")
- }
-
- it should "return domain when provided a long url with a top-level domain no country code" in {
- val url = "wow.its.really.long.super.long.yeah.so.long.long.long.long.forrealz.net"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("forrealz")
- }
-
- it should "should return \"None\" when provided an address" in {
- val url = "123.103.104.10.in-addr.arpa"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("None")
- }
-
- it should "return \"None\" when provided a short url with a bad top-level domain / country code" in {
- val url = "panasonic.c"
- val result = DomainProcessor.extractDomain(url)
- result shouldEqual ("None")
- }
-
- "extractDomainInfo" should "handle an in-addr.arpa url" in {
-
- val url = "123.103.104.10.in-addr.arpa"
-
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
-
- val userDomain = "intel"
-
- // case class DerivedFields(topDomain: String, subdomainLength: Double, subdomainEntropy: Double, numPeriods: Double)
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "None", topDomain = 0, subdomain = "None", subdomainLength = 0, subdomainEntropy = 0, numPeriods = 6)
- }
-
- it should "handle an Alexa top 1M domain with a subdomain, top-level domain name and country code" in {
-
- val url = "services.amazon.com.mx"
-
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
-
- val userDomain = "intel"
-
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "amazon", topDomain = 1, subdomain = "services",
- subdomainLength = 8, subdomainEntropy = 2.5, numPeriods = 4)
- }
-
- it should "handle an Alexa top 1M domain with a top-level domain name and country code but no subdomain" in {
-
- val url = "amazon.com.mx"
- val countryCodes = sparkContext.broadcast(countryCodesSet)
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
- val userDomain = "intel"
-
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "amazon", subdomain = "None", topDomain = 1, subdomainLength = 0, subdomainEntropy = 0, numPeriods = 3)
- }
-
- it should "handle an Alexa top 1M domain with a subdomain and top-level domain name but no country code" in {
-
- val url = "services.amazon.com"
- val countryCodes = sparkContext.broadcast(countryCodesSet)
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
- val userDomain = "intel"
-
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "amazon", subdomain = "services", topDomain = 1, subdomainLength = 8, subdomainEntropy = 2.5, numPeriods = 3)
- }
-
-
- it should "handle an Alexa top 1M domain with no subdomain or country code" in {
-
- val url = "amazon.com"
- val countryCodes = sparkContext.broadcast(countryCodesSet)
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
- val userDomain = "intel"
-
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "amazon", subdomain = "None", topDomain = 1, subdomainLength = 0, subdomainEntropy = 0, numPeriods = 2)
- }
- it should "not identify the domain as the users domain when both are empty strings" in {
- val url = "ab..com"
- val countryCodes = sparkContext.broadcast(countryCodesSet)
- val topDomains = sparkContext.broadcast(TopDomains.TopDomains)
- val userDomain = ""
-
- val result = extractDomainInfo(url, topDomains, userDomain)
-
- result shouldBe DomainInfo(domain = "", subdomain = "ab", topDomain = 0, subdomainLength = 2, subdomainEntropy = 1, numPeriods = 3)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
deleted file mode 100644
index b15bc5d..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityDoubleTest.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.types._
-import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.scalatest.Matchers
-
-/**
- * Created by rabarona on 5/17/17.
- */
-class PrecisionUtilityDoubleTest extends TestingSparkContextFlatSpec with Matchers {
-
- "toTargetType" should "just return the same value with the same type" in {
- val testValue: Double = 5d
-
- val result = PrecisionUtilityDouble.toTargetType(testValue)
-
- result shouldBe testValue
- result shouldBe a[java.lang.Double]
-
- }
-
- "toDoubles" should "return an array of the same type" in {
-
- val testSeq: Seq[Double] = Seq(1d, 2d, 3d)
-
- val result: Seq[Double] = PrecisionUtilityDouble.toDoubles(testSeq)
-
- result shouldBe a[Seq[Double]]
- result.length shouldBe 3
- }
-
- "castColumn" should "return the exact same data frame" in {
-
- val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
- .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
-
- val result = PrecisionUtilityDouble.castColumn(testDataFrame, TopicProbabilityMix)
-
- val schema = StructType(
- Array(StructField(DocumentName, StringType, true),
- StructField(TopicProbabilityMix, ArrayType(DoubleType, false), true)))
-
- result shouldBe testDataFrame
- result.schema shouldBe schema
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
deleted file mode 100644
index f965b3d..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/PrecisionUtilityFloatTest.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.apache.spot.utilities.transformation
-
-import org.apache.spark.sql.types._
-import org.apache.spot.lda.SpotLDAWrapperSchema.{DocumentName, TopicProbabilityMix}
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.scalatest.Matchers
-
-/**
- * Created by rabarona on 5/17/17.
- */
-class PrecisionUtilityFloatTest extends TestingSparkContextFlatSpec with Matchers {
-
- "toTargetType" should "just return value converted to Float" in {
- val testValue: Double = 5d
-
- val result = PrecisionUtilityFloat.toTargetType(testValue)
-
- result shouldBe 5f
- result shouldBe a[java.lang.Float]
-
- }
-
- "toDoubles" should "convert a Seq of Float to Seq of Double" in {
-
- val testSeq: Seq[Float] = Seq(1f, 2f, 3f)
-
- val result: Seq[Double] = PrecisionUtilityFloat.toDoubles(testSeq)
-
- result(0) shouldBe a[java.lang.Double]
- result(1) shouldBe a[java.lang.Double]
- result(2) shouldBe a[java.lang.Double]
- }
-
- "castColumn" should "return a data frame with a schema modified, Seq[Float] instead of Seq[Double]" in {
-
- val testDataFrame = sqlContext.createDataFrame(Seq(("doc1", Array(1d, 2d)), ("doc2", Array(2d, 3d))))
- .withColumnRenamed("_1", DocumentName).withColumnRenamed("_2", TopicProbabilityMix)
-
- val result = PrecisionUtilityFloat.castColumn(testDataFrame, TopicProbabilityMix)
- result.count
-
- val schema = StructType(
- Array(StructField(DocumentName, StringType, true),
- StructField(TopicProbabilityMix, ArrayType(FloatType, false), true)))
-
- result.schema shouldBe schema
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4c7ad4eb/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/QuantilesTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/QuantilesTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/QuantilesTest.scala
deleted file mode 100644
index bf71abe..0000000
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/transformation/QuantilesTest.scala
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spot.utilities
-
-import org.apache.spark.rdd.RDD
-import org.apache.spot.testutils.TestingSparkContextFlatSpec
-import org.apache.spot.utilities.Quantiles
-import org.scalatest.Matchers
-
-
-class QuantilesTest extends TestingSparkContextFlatSpec with Matchers {
-
- val allOnes = List(1.0, 1.0, 1.0, 1.0, 1.0)
- val onesAndTwos = List(1.0, 2.0, 1.0, 2.0)
- val countToTen = List(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0)
-
-
- "ecdf" should "work on an empty list" in {
- val rddIn: RDD[Double] = sparkContext.parallelize(List(), numSlices = 3)
- val rddOut = Quantiles.computeEcdf(rddIn)
-
- val out = rddOut.collect()
-
- out.length shouldBe 0
- }
-
- "ecdf" should "work on a constant list" in {
- val rddIn = sparkContext.parallelize(allOnes, numSlices = 3)
- val rddOut = Quantiles.computeEcdf(rddIn)
-
- val out = rddOut.collect()
-
- out.length shouldBe 1
- out(0) shouldBe(1.0, 1.0)
- }
-
- "ecdf" should "work on a split 50/50 list" in {
- val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
- val rddOut = Quantiles.computeEcdf(rddIn)
-
- val out = rddOut.collect()
-
- out.length shouldBe 2
- out(0) shouldBe(1.0, 0.5)
- out(1) shouldBe(2.0, 1.0)
- }
-
- "ecdf" should "work on count-to-ten list" in {
- val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
- val rddOut = Quantiles.computeEcdf(rddIn)
-
- val out = rddOut.collect()
-
- out.length shouldBe 10
- out(0) shouldBe(1.0, 0.1)
- out(1) shouldBe(2.0, 0.2)
- out(2) shouldBe(3.0, 0.3)
- out(3) shouldBe(4.0, 0.4)
- out(4) shouldBe(5.0, 0.5)
- out(5) shouldBe(6.0, 0.6)
- out(6) shouldBe(7.0, 0.7)
- out(7) shouldBe(8.0, 0.8)
- out(8) shouldBe(9.0, 0.9)
- out(9) shouldBe(10.0, 1.0)
- }
-
- "quantiles" should "work on an empty list" in {
- val rddIn: RDD[Double] = sparkContext.parallelize(List(), numSlices = 3)
- val quantiles = Array(0.0, 0.5)
- val out = Quantiles.computeQuantiles(rddIn, quantiles)
-
- out.length shouldBe 2
- out(0) shouldBe Double.PositiveInfinity
- out(1) shouldBe Double.PositiveInfinity
- }
-
- "quantiles" should "work on all ones list" in {
- val rddIn = sparkContext.parallelize(allOnes, numSlices = 3)
- val quantiles = Array(0.0, 0.5)
- val out = Quantiles.computeQuantiles(rddIn, quantiles)
-
- out.length shouldBe 2
- out(0) shouldBe 1.0
- out(1) shouldBe 1.0
- }
-
- "quantiles" should "work on a 50/50 1s and 2s list" in {
- val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
- val quantiles = Array(0.0, 0.5)
- val out = Quantiles.computeQuantiles(rddIn, quantiles)
-
- out.length shouldBe 2
- out(0) shouldBe 1.0
- out(1) shouldBe 1.0
- }
-
- "quantiles" should "work on a 50/50 1s and 2s list with unbalanced thresholds" in {
- val rddIn = sparkContext.parallelize(onesAndTwos, numSlices = 3)
- val quantiles = Array(0.0, 0.6, 0.99)
- val out = Quantiles.computeQuantiles(rddIn, quantiles)
-
- out.length shouldBe 3
- out(0) shouldBe 1.0
- out(1) shouldBe 2.0
- out(2) shouldBe 2.0
- }
-
- "quantiles" should "work on a count-to-ten list" in {
- val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
- val quantiles = Array(0.0, 0.5)
- val out = Quantiles.computeQuantiles(rddIn, quantiles)
-
- out.length shouldBe 2
- out(0) shouldBe 1.0
- out(1) shouldBe 5.0
-
- }
-
- "deciles" should "work on a count-to-ten list" in {
- val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
- val out = Quantiles.computeDeciles(rddIn)
-
- out.length shouldBe 10
- out(0) shouldBe 1.0
- out(1) shouldBe 2.0
- out(2) shouldBe 3.0
- out(3) shouldBe 4.0
- out(4) shouldBe 5.0
- out(5) shouldBe 6.0
- out(6) shouldBe 7.0
- out(7) shouldBe 8.0
- out(8) shouldBe 9.0
- out(9) shouldBe 10.0
- }
-
- "quintiles" should "work on a count-to-ten list" in {
- val rddIn = sparkContext.parallelize(countToTen, numSlices = 3)
- val out = Quantiles.computeQuintiles(rddIn)
-
- out.length shouldBe 5
- out(0) shouldBe 2.0
- out(1) shouldBe 4.0
- out(2) shouldBe 6.0
- out(3) shouldBe 8.0
- out(4) shouldBe 10.0
- }
-
- "bin" should "return 3 when the value is not bigger than the fourth quintile" in {
- val quintiles = Array(1.0, 2.0, 3.0, 4.0, 5.0)
-
- val result = Quantiles.bin(3.5, quintiles)
-
- result shouldBe 3
- }
-
- it should "return 0 when the value is less than the first quintile" in {
- val quintiles = Array(1.0, 2.0, 3.0, 4.0, 5.0)
-
- val result = Quantiles.bin(0.0, quintiles)
-
- result shouldBe 0
- }
-}
\ No newline at end of file