You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2018/11/19 08:58:03 UTC
incubator-griffin git commit: [GRIFFIN-212] Add matchedFraction in
streaming accuracy metrics
Repository: incubator-griffin
Updated Branches:
refs/heads/master 5c34a83f1 -> b4f7d1551
[GRIFFIN-212] Add matchedFraction in streaming accuracy metrics
Author: Lionel Liu <bh...@163.com>
Closes #455 from bhlx3lyx7/util-update-0.3.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/b4f7d155
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/b4f7d155
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/b4f7d155
Branch: refs/heads/master
Commit: b4f7d1551cc4165a81f9cf9340b50c3099ee0f1b
Parents: 5c34a83
Author: Lionel Liu <bh...@163.com>
Authored: Mon Nov 19 16:57:57 2018 +0800
Committer: William Guo <gu...@apache.org>
Committed: Mon Nov 19 16:57:57 2018 +0800
----------------------------------------------------------------------
.../measure/context/streaming/metric/AccuracyMetric.scala | 4 +++-
.../apache/griffin/measure/step/transform/DataFrameOps.scala | 8 +++++---
2 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b4f7d155/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala
----------------------------------------------------------------------
diff --git a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala
index 19dfb9e..5065417 100644
--- a/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala
+++ b/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/AccuracyMetric.scala
@@ -51,6 +51,8 @@ case class AccuracyMetric(miss: Long, total: Long) extends Metric {
def getMatch: Long = total - miss
- def matchPercentage: Double = if (getTotal <= 0) 0 else getMatch.toDouble / getTotal * 100
+ def matchFraction: Double = if (getTotal <= 0) 1 else getMatch.toDouble / getTotal
+
+ def matchPercentage: Double = matchFraction * 100
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b4f7d155/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala
----------------------------------------------------------------------
diff --git a/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala b/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala
index 088f328..ba64d33 100644
--- a/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala
+++ b/measure/src/main/scala/org/apache/griffin/measure/step/transform/DataFrameOps.scala
@@ -21,8 +21,7 @@ package org.apache.griffin.measure.step.transform
import java.util.Date
import org.apache.spark.sql.{Encoders, Row, SQLContext, _}
-import org.apache.spark.sql.types.{BooleanType, LongType, StructField, StructType}
-
+import org.apache.spark.sql.types._
import org.apache.griffin.measure.context.ContextId
import org.apache.griffin.measure.context.streaming.metric._
import org.apache.griffin.measure.context.streaming.metric.CacheResults.CacheResult
@@ -43,6 +42,7 @@ object DataFrameOps {
val _miss = "miss"
val _total = "total"
val _matched = "matched"
+ val _matchedFraction = "matchedFraction"
}
def fromJson(sqlContext: SQLContext,
@@ -70,6 +70,7 @@ object DataFrameOps {
val miss = details.getStringOrKey(_miss)
val total = details.getStringOrKey(_total)
val matched = details.getStringOrKey(_matched)
+ val matchedFraction = details.getStringOrKey(_matchedFraction)
val updateTime = new Date().getTime
@@ -107,12 +108,13 @@ object DataFrameOps {
StructField(miss, LongType),
StructField(total, LongType),
StructField(matched, LongType),
+ StructField(matchedFraction, DoubleType),
StructField(ConstantColumns.record, BooleanType),
StructField(ConstantColumns.empty, BooleanType)
))
val rows = updatedResults.map { r =>
val ar = r.result.asInstanceOf[AccuracyMetric]
- Row(r.timeStamp, ar.miss, ar.total, ar.getMatch, !ar.initial, ar.eventual)
+ Row(r.timeStamp, ar.miss, ar.total, ar.getMatch, ar.matchFraction, !ar.initial, ar.eventual)
}.toArray
val rowRdd = sqlContext.sparkContext.parallelize(rows)
val retDf = sqlContext.createDataFrame(rowRdd, schema)