You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by ya...@apache.org on 2017/01/27 03:08:30 UTC
incubator-hivemall git commit: Close #30: [HIVEMALL-37][SPARK]
Support a SST-based change-point detector in DataFrame/Spark
Repository: incubator-hivemall
Updated Branches:
refs/heads/master 88fae5208 -> 892ae1c16
Close #30: [HIVEMALL-37][SPARK] Support a SST-based change-point detector in DataFrame/Spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/892ae1c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/892ae1c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/892ae1c1
Branch: refs/heads/master
Commit: 892ae1c160bc4166f7b80ebb16b0aac1705ae805
Parents: 88fae52
Author: Takeshi YAMAMURO <li...@gmail.com>
Authored: Fri Jan 27 12:08:21 2017 +0900
Committer: Takeshi YAMAMURO <li...@gmail.com>
Committed: Fri Jan 27 12:08:21 2017 +0900
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/hive/HivemallOps.scala | 14 ++++++++++++++
.../org/apache/spark/sql/hive/HivemallOpsSuite.scala | 7 +++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/892ae1c1/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala
----------------------------------------------------------------------
diff --git a/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala b/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala
index 6e588a9..28653a5 100644
--- a/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala
+++ b/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala
@@ -41,6 +41,7 @@ import org.apache.spark.unsafe.types.UTF8String
* @groupname classifier
* @groupname classifier.multiclass
* @groupname xgboost
+ * @groupname anomaly
* @groupname knn.similarity
* @groupname knn.distance
* @groupname knn.lsh
@@ -856,6 +857,19 @@ object HivemallOps {
}
/**
+ * @see [[hivemall.anomaly.SingularSpectrumTransformUDF]]
+ * @group anomaly
+ */
+ @scala.annotation.varargs
+ def sst(exprs: Column*): Column = withExpr {
+ planHiveGenericUDF(
+ "hivemall.anomaly.SingularSpectrumTransformUDF",
+ "sst",
+ exprs
+ )
+ }
+
+ /**
* @see [[hivemall.knn.similarity.CosineSimilarityUDF]]
* @group knn.similarity
*/
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/892ae1c1/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
----------------------------------------------------------------------
diff --git a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
index 493feda..bab5085 100644
--- a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
+++ b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
@@ -30,6 +30,13 @@ import org.apache.spark.test.TestFPWrapper._
final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
+ test("anomaly") {
+ import hiveContext.implicits._
+ val df = spark.range(1000).selectExpr("id AS time", "rand() AS x")
+ // TODO: Test results more exactly
+ assert(df.sort($"time".asc).select(sst($"x", lit("-th 0.005"))).count === 1000)
+ }
+
test("knn.similarity") {
val df1 = DummyInputData.select(cosine_sim(lit2(Seq(1, 2, 3, 4)), lit2(Seq(3, 4, 5, 6))))
assert(df1.collect.apply(0).getFloat(0) ~== 0.500f)