You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/05/23 03:36:00 UTC

[1/3] incubator-griffin git commit: clean files

Repository: incubator-griffin
Updated Branches:
  refs/heads/master a0117811c -> c4340cea1


http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/config1.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/config1.json b/griffin-measure/griffin-measure-batch/src/test/resources/config1.json
deleted file mode 100644
index d7290ba..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/config1.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "name": "accu-test",
-  "type": "accuracy",
-
-  "source": {
-    "type": "hive",
-    "version": "1.2",
-    "config": {
-      "table.name": "rheos_view_event",
-      "partitions": "dt=20170410, hour=15"
-    }
-  },
-
-  "target": {
-    "type": "hive",
-    "version": "1.2",
-    "config": {
-      "table.name": "be_view_event_queue",
-      "partitions": "dt=20170410, hour=15; dt=20170410, hour=16"
-    }
-  },
-
-  "evaluateRule": {
-    "sampleRatio": 1,
-    "rules": "@Key ${source}['uid'] === ${target}['uid']; @Key ${source}['eventtimestamp'] === ${target}['eventtimestamp']; ${source}['page_id'] === ${target}['page_id']; ${source}['site_id'] === ${target}['site_id']; ${source}['itm'] === ${target}['itm']"
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/env.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/env.json b/griffin-measure/griffin-measure-batch/src/test/resources/env.json
deleted file mode 100644
index 3a9e38c..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/env.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "spark": {
-    "log.level": "ERROR",
-    "checkpoint.dir": "hdfs:///griffin/batch/cp",
-    "config": {}
-  },
-
-  "persist": [
-    {
-      "type": "hdfs",
-      "config": {
-        "path": "hdfs:///griffin/streaming/persist"
-      }
-    },
-    {
-      "type": "http",
-      "config": {
-        "method": "post",
-        "api": "http://phxbark4dq-360935.stratus.phx.ebay.com:8080/"
-      }
-    }
-  ],
-
-  "cleaner": {
-
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/env1.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/env1.json b/griffin-measure/griffin-measure-batch/src/test/resources/env1.json
deleted file mode 100644
index a059715..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/env1.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "spark": {
-    "log.level": "INFO",
-    "checkpoint.dir": "hdfs:///griffin/batch/cp",
-    "config": {}
-  },
-
-  "persist": [
-    {
-      "type": "hdfs",
-      "config": {
-        "path": "hdfs:///user/b_des/bark/griffin-batch/test",
-        "max.lines.per.file": 10000
-      }
-    }
-  ],
-
-  "cleaner": {
-
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/log4j.properties b/griffin-measure/griffin-measure-batch/src/test/resources/log4j.properties
deleted file mode 100644
index bd31e15..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-log4j.rootLogger=INFO, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.Target=System.out
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%c] - %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.avro
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.avro b/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.avro
deleted file mode 100644
index 3d5c939..0000000
Binary files a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.avro and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.dat
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.dat b/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.dat
deleted file mode 100644
index ce49443..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_src.dat
+++ /dev/null
@@ -1,50 +0,0 @@
-10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000001|94022
-10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000002|94022
-10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
-10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
-10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
-10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
-10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
-10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
-10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
-10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
-10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
-10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
-10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
-10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
-10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
-10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
-10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
-10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
-10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
-10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
-10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
-10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
-10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
-10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
-10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
-10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
-10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
-10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
-10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
-10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
-10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
-10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
-10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
-10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
-10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
-10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
-10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
-10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
-10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
-10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
-10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
-10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
-10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
-10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
-10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
-10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
-10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
-10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
-10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
-10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.avro
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.avro b/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.avro
deleted file mode 100644
index 104dd6c..0000000
Binary files a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.avro and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.dat
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.dat b/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.dat
deleted file mode 100644
index 07a6b40..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/users_info_target.dat
+++ /dev/null
@@ -1,50 +0,0 @@
-10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000101|94022
-10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000102|94022
-10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
-10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
-10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
-10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
-10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
-10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
-10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
-10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
-10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
-10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
-10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
-10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
-10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
-10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
-10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
-10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
-10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
-10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
-10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
-10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
-10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
-10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
-10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
-10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
-10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
-10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
-10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
-10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
-10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
-10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
-10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
-10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
-10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
-10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
-10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
-10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
-10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
-10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
-10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
-10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
-10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
-10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
-10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
-10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
-10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
-10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
-10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
-10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgoTest.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgoTest.scala b/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgoTest.scala
deleted file mode 100644
index 5149847..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgoTest.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-package org.apache.griffin.measure.batch.algo
-
-import java.util.Date
-
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
-import org.scalatest.junit.JUnitRunner
-import org.apache.griffin.measure.batch.config.params.user._
-import org.apache.griffin.measure.batch.config.params.env._
-import org.apache.griffin.measure.batch.config.params._
-import org.apache.griffin.measure.batch.config.reader._
-import org.apache.griffin.measure.batch.config.validator._
-import org.apache.griffin.measure.batch.connector.{DataConnector, DataConnectorFactory, CacheDataUtil}
-import org.apache.griffin.measure.batch.rule.{RuleAnalyzer, RuleFactory}
-import org.apache.griffin.measure.batch.rule.expr._
-
-import scala.util.{Failure, Success, Try}
-import org.apache.griffin.measure.batch.log.Loggable
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-
-@RunWith(classOf[JUnitRunner])
-class BatchAccuracyAlgoTest extends FunSuite with Matchers with BeforeAndAfter with Loggable {
-
-  val envFile = "src/test/resources/env.json"
-//  val confFile = "src/test/resources/config.json"
-  val confFile = "{\"name\":\"accu1\",\"type\":\"accuracy\",\"source\":{\"type\":\"avro\",\"version\":\"1.7\",\"config\":{\"file.name\":\"src/test/resources/users_info_src.avro\"}},\"target\":{\"type\":\"avro\",\"version\":\"1.7\",\"config\":{\"file.name\":\"src/test/resources/users_info_target.avro\"}},\"evaluateRule\":{\"sampleRatio\":1,\"rules\":\"$source.user_id > 10020 AND $source.user_id + 5 = $target.user_id + (2 + 3) AND $source.first_name + 12 = $target.first_name + (10 + 2) AND $source.last_name = $target.last_name AND $source.address = $target.address AND $source.email = $target.email AND $source.phone = $target.phone AND $source.post_code = $target.post_code\"}}"
-  val envFsType = "local"
-  val userFsType = "raw"
-
-  val args = Array(envFile, confFile)
-
-  var sc: SparkContext = _
-  var sqlContext: SQLContext = _
-
-  var allParam: AllParam = _
-
-  before {
-    // read param files
-    val envParam = readParamFile[EnvParam](envFile, envFsType) match {
-      case Success(p) => p
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-2)
-      }
-    }
-    val userParam = readParamFile[UserParam](confFile, userFsType) match {
-      case Success(p) => p
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-2)
-      }
-    }
-    allParam = AllParam(envParam, userParam)
-
-    // validate param files
-    validateParams(allParam) match {
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-3)
-      }
-      case _ => {
-        info("params validation pass")
-      }
-    }
-
-    val metricName = userParam.name
-    val conf = new SparkConf().setMaster("local[*]").setAppName(metricName)
-    sc = new SparkContext(conf)
-    sqlContext = new SQLContext(sc)
-  }
-
-  test("algorithm") {
-    Try {
-      val envParam = allParam.envParam
-      val userParam = allParam.userParam
-
-      // start time
-      val startTime = new Date().getTime()
-
-      // get spark application id
-      val applicationId = sc.applicationId
-
-      // rules
-      val ruleFactory = RuleFactory(userParam.evaluateRuleParam)
-      val rule: StatementExpr = ruleFactory.generateRule()
-      val ruleAnalyzer: RuleAnalyzer = RuleAnalyzer(rule)
-
-      ruleAnalyzer.globalCacheExprs.foreach(println)
-      ruleAnalyzer.globalFinalCacheExprs.foreach(println)
-
-      // global cache data
-      val globalCachedData = CacheDataUtil.genCachedMap(None, ruleAnalyzer.globalCacheExprs, Map[String, Any]())
-      val globalFinalCachedData = CacheDataUtil.filterCachedMap(ruleAnalyzer.globalFinalCacheExprs, globalCachedData)
-
-      // data connector
-      val sourceDataConnector: DataConnector =
-        DataConnectorFactory.getDataConnector(sqlContext, userParam.sourceParam,
-          ruleAnalyzer.sourceGroupbyExprs, ruleAnalyzer.sourceCacheExprs,
-          ruleAnalyzer.sourceFinalCacheExprs, globalFinalCachedData) match {
-          case Success(cntr) => {
-            if (cntr.available) cntr
-            else throw new Exception("source data not available!")
-          }
-          case Failure(ex) => throw ex
-        }
-      val targetDataConnector: DataConnector =
-        DataConnectorFactory.getDataConnector(sqlContext, userParam.targetParam,
-          ruleAnalyzer.targetGroupbyExprs, ruleAnalyzer.targetCacheExprs,
-          ruleAnalyzer.targetFinalCacheExprs, globalFinalCachedData) match {
-          case Success(cntr) => {
-            if (cntr.available) cntr
-            else throw new Exception("target data not available!")
-          }
-          case Failure(ex) => throw ex
-        }
-
-      // get metadata
-//      val sourceMetaData: Iterable[(String, String)] = sourceDataConnector.metaData() match {
-//        case Success(md) => md
-//        case Failure(ex) => throw ex
-//      }
-//      val targetMetaData: Iterable[(String, String)] = targetDataConnector.metaData() match {
-//        case Success(md) => md
-//        case Failure(ex) => throw ex
-//      }
-
-      // get data
-      val sourceData: RDD[(Product, Map[String, Any])] = sourceDataConnector.data() match {
-        case Success(dt) => dt
-        case Failure(ex) => throw ex
-      }
-      val targetData: RDD[(Product, Map[String, Any])] = targetDataConnector.data() match {
-        case Success(dt) => dt
-        case Failure(ex) => throw ex
-      }
-
-//      println("-- global cache exprs --")
-//      ruleAnalyzer.globalCacheExprs.foreach(a => println(s"${a._id} ${a.desc}"))
-//      println("-- global cache data --")
-//      globalCachedData.foreach(println)
-//      println("-- global final cache data --")
-//      globalFinalCachedData.foreach(println)
-//
-//      println("-- source persist exprs --")
-//      ruleAnalyzer.sourcePersistExprs.foreach(a => println(s"${a._id} ${a.desc}"))
-//      println("-- source cache exprs --")
-//      ruleAnalyzer.sourceCacheExprs.foreach(a => println(s"${a._id} ${a.desc}"))
-//
-//      println("-- source --")
-//      sourceData.foreach { a =>
-//        val printMap = ruleAnalyzer.sourcePersistExprs.flatMap { expr =>
-//          a._2.get(expr._id) match {
-//            case Some(v) => Some((expr._id + expr.desc, v))
-//            case _ => None
-//          }
-//        }.toMap
-//        println(printMap)
-//
-//        val cacheMap = ruleAnalyzer.sourceCacheExprs.flatMap { expr =>
-//          a._2.get(expr._id) match {
-//            case Some(v) => Some((expr._id + expr.desc, v))
-//            case _ => None
-//          }
-//        }.toMap
-//        println(cacheMap)
-//
-//        println(a)
-//        println(a._2.size)
-//      }
-
-//      println("-- target --")
-//      targetData.foreach { a =>
-//        val printMap = ruleAnalyzer.targetPersistExprs.flatMap { expr =>
-//          a._2.get(expr._id) match {
-//            case Some(v) => Some((expr.desc, v))
-//            case _ => None
-//          }
-//        }.toMap
-//        println(printMap)
-//      }
-
-      // my algo
-      val algo = BatchAccuracyAlgo(allParam)
-
-      // accuracy algorithm
-      val (accuResult, missingRdd, matchingRdd) = algo.accuracy(sourceData, targetData, ruleAnalyzer)
-
-      println(s"match percentage: ${accuResult.matchPercentage}, total count: ${accuResult.total}")
-
-      missingRdd.foreach(println)
-
-      // end time
-      val endTime = new Date().getTime
-      println(s"using time: ${endTime - startTime} ms")
-    } match {
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-4)
-      }
-      case _ => {
-        info("calculation finished")
-      }
-    }
-  }
-
-  private def readParamFile[T <: Param](file: String, fsType: String)(implicit m : Manifest[T]): Try[T] = {
-    val paramReader = ParamReaderFactory.getParamReader(file, fsType)
-    paramReader.readConfig[T]
-  }
-
-  private def validateParams(allParam: AllParam): Try[Boolean] = {
-    val allParamValidator = AllParamValidator()
-    allParamValidator.validate(allParam)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReaderTest.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReaderTest.scala b/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReaderTest.scala
deleted file mode 100644
index f1ccefb..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReaderTest.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
-import org.scalatest.junit.JUnitRunner
-import org.apache.griffin.measure.batch.config.params.user._
-import org.apache.griffin.measure.batch.config.params.env._
-
-import scala.util.{Failure, Success}
-import org.apache.griffin.measure.batch.log.Loggable
-import org.apache.griffin.measure.batch.persist.{HdfsPersist, PersistFactory}
-
-
-@RunWith(classOf[JUnitRunner])
-class ParamFileReaderTest extends FunSuite with Matchers with BeforeAndAfter with Loggable {
-
-  test("test file reader") {
-    val userReader = ParamFileReader("src/test/resources/config1.json")
-    val envReader = ParamFileReader("src/test/resources/env1.json")
-
-    val p1 = userReader.readConfig[UserParam]
-    val p2 = envReader.readConfig[EnvParam]
-
-    p1 match {
-      case Success(v) => println(v)
-      case Failure(ex) => error(ex.getMessage)
-    }
-
-    p2 match {
-      case Success(v) => println(v)
-      case Failure(ex) => error(ex.getMessage)
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/rule/RuleParserTest.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/rule/RuleParserTest.scala b/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/rule/RuleParserTest.scala
deleted file mode 100644
index f964bcd..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/scala/org/apache/griffin/measure/batch/rule/RuleParserTest.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-package org.apache.griffin.measure.batch.rule
-
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
-import org.scalatest.junit.JUnitRunner
-
-import scala.util.{Success, Failure}
-
-import org.apache.griffin.measure.batch.log.Loggable
-
-
-@RunWith(classOf[JUnitRunner])
-class RuleParserTest extends FunSuite with Matchers with BeforeAndAfter with Loggable {
-
-//  test("test rule parser") {
-//    val ruleParser = RuleParser()
-//
-////    val rules = "outTime = 24h; @Invalid ${source}['__time'] + ${outTime} > ${target}['__time']"
-//    val rules = "@Key ${source}.json()['seeds'][*].json()['metadata'].json()['tracker']['crawlRequestCreateTS'] === ${target}.json()['groups'][0][\"attrsList\"]['name'=\"CRAWLMETADATA\"]['values'][0].json()['tracker']['crawlRequestCreateTS']"
-////    val rules = "${source}['__time'] + ${outTime} > ${target}['__time']"
-////    val rules = "${source}['__time'] > ${target}['__time']"
-////    val rules = "432"
-////    val rules = "${target}.json()['groups'][0]['attrsList']['name'='URL']['values'][0]"
-//
-//    val result = ruleParser.parseAll(ruleParser.statementsExpr, rules)
-//
-//    println(result)
-//  }
-
-//  test("treat escape") {
-//    val es = """Hello\tworld\nmy name is \"ABC\""""
-//    val un = StringContext treatEscapes es
-//
-//    println(es)
-//    println(un)
-//  }
-
-  test("test rule parser") {
-    val ruleParser = RuleParser()
-
-//    val rules = "$SOUrce['tgt' < $source['tag' != 2] - -+-++---1] between ( -$target['32a'] + 9, 100, ----1000 ) and (45 > 9 or $target.type + 8 == 9 and $source['a'] >= 0) when not not not not $source._time + 24h < $target._time"
-    val rules = "$source['aaaf fd', (21, 43), '12']"
-
-    val result = ruleParser.parseAll(ruleParser.selection, rules)
-
-    println(result)
-  }
-
-  test("test rule analyzer") {
-    val ruleParser = RuleParser()
-
-//    val rules = "$source.tag == $target['take' >= 5] and $source.price + $source.price1 > $target['kk' < $target.age] and $source.ee = $target.fe + $target.a when $target.ggg = 1"
-    val rules = "$source."
-    val result = ruleParser.parseAll(ruleParser.rule, rules)
-    println(result)
-
-    if (result.successful) {
-      val ruleAnalyzer = RuleAnalyzer(result.get)
-
-      println("source")
-      ruleAnalyzer.sourceCacheExprs.foreach(a => println(a.desc))
-      println("source final")
-      ruleAnalyzer.sourceFinalCacheExprs.foreach(a => println(a.desc))
-      println("target")
-      ruleAnalyzer.targetCacheExprs.foreach(a => println(a.desc))
-      println("target final")
-      ruleAnalyzer.targetFinalCacheExprs.foreach(a => println(a.desc))
-      println("groupby")
-      ruleAnalyzer.sourceGroupbyExprs.foreach(println)
-      ruleAnalyzer.targetGroupbyExprs.foreach(println)
-    }
-
-
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/pom.xml
----------------------------------------------------------------------
diff --git a/griffin-measure/pom.xml b/griffin-measure/pom.xml
deleted file mode 100644
index 01475da..0000000
--- a/griffin-measure/pom.xml
+++ /dev/null
@@ -1,197 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <!--<parent>-->
-        <!--<artifactId>griffin-parent</artifactId>-->
-        <!--<groupId>com.ebay.oss</groupId>-->
-        <!--<version>0.1.0-SNAPSHOT</version>-->
-    <!--</parent>-->
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>org.apache.griffin</groupId>
-    <artifactId>griffin-measure</artifactId>
-    <packaging>pom</packaging>
-    <version>0.0.1-SNAPSHOT</version>
-    <modules>
-        <module>griffin-measure-batch</module>
-    </modules>
-
-    <properties>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
-
-        <scala.version>2.10.6</scala.version>
-        <spark.version>1.6.0</spark.version>
-        <scala.binary.version>2.10</scala.binary.version>
-
-        <avro.version>1.7.7</avro.version>
-        <jackson.version>2.8.7</jackson.version>
-        <scalaj.version>2.3.0</scalaj.version>
-        <junit.version>4.11</junit.version>
-        <scalatest.version>2.2.4</scalatest.version>
-        <slf4j.version>1.7.21</slf4j.version>
-        <log4j.version>1.2.16</log4j.version>
-    </properties>
-
-    <dependencies>
-        <!--scala-->
-        <dependency>
-            <groupId>org.scala-lang</groupId>
-            <artifactId>scala-library</artifactId>
-            <version>${scala.version}</version>
-        </dependency>
-
-        <!--spark, spark streaming, spark hive-->
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-streaming_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-streaming-kafka_${scala.binary.version}</artifactId>
-            <version>${spark.version}</version>
-        </dependency>
-
-        <!--jackson-->
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
-            <version>${jackson.version}</version>
-        </dependency>
-
-        <!--scalaj for http request-->
-        <dependency>
-            <groupId>org.scalaj</groupId>
-            <artifactId>scalaj-http_${scala.binary.version}</artifactId>
-            <version>${scalaj.version}</version>
-        </dependency>
-
-        <!--avro-->
-        <dependency>
-            <groupId>com.databricks</groupId>
-            <artifactId>spark-avro_${scala.binary.version}</artifactId>
-            <version>2.0.1</version>
-        </dependency>
-        <!--csv-->
-        <dependency>
-            <groupId>com.databricks</groupId>
-            <artifactId>spark-csv_${scala.binary.version}</artifactId>
-            <version>1.5.0</version>
-        </dependency>
-
-        <!--log4j-->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>${slf4j.version}</version>
-        </dependency>
-        <!--<dependency>-->
-            <!--<groupId>org.slf4j</groupId>-->
-            <!--<artifactId>slf4j-simple</artifactId>-->
-            <!--<version>${slf4j.version}</version>-->
-        <!--</dependency>-->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>${slf4j.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
-            <version>${log4j.version}</version>
-        </dependency>
-
-        <!--junit-->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>${junit.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <!--scala test-->
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.binary.version}</artifactId>
-            <version>${scalatest.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <configuration>
-                    <!-- or whatever version you use -->
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <version>2.15.2</version>
-                <executions>
-                    <execution>
-                        <id>compile</id>
-                        <goals>
-                            <goal>compile</goal>
-                        </goals>
-                        <phase>compile</phase>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.5.1</version>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-models/data/test/dataFile/users_info_src.avro
----------------------------------------------------------------------
diff --git a/griffin-models/data/test/dataFile/users_info_src.avro b/griffin-models/data/test/dataFile/users_info_src.avro
deleted file mode 100644
index 615f538..0000000
Binary files a/griffin-models/data/test/dataFile/users_info_src.avro and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-models/data/test/dataFile/users_info_target.avro
----------------------------------------------------------------------
diff --git a/griffin-models/data/test/dataFile/users_info_target.avro b/griffin-models/data/test/dataFile/users_info_target.avro
deleted file mode 100644
index c2b267b..0000000
Binary files a/griffin-models/data/test/dataFile/users_info_target.avro and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-models/data/test/recordFile/_RESULT_ACCU
----------------------------------------------------------------------
diff --git a/griffin-models/data/test/recordFile/_RESULT_ACCU b/griffin-models/data/test/recordFile/_RESULT_ACCU
deleted file mode 100644
index 77cb87b..0000000
--- a/griffin-models/data/test/recordFile/_RESULT_ACCU
+++ /dev/null
@@ -1,12 +0,0 @@
-//========== 1. Test Accuracy model result with request file: data\test\reqJson\accuAvroTest.json ==========
-match percentage: 80.0 %
-
-Map(id -> 44444, name -> Snoooooopy, age -> 16, desc -> dooooog)
-
-//========== 2. Test Accuracy model result with request file: data\test\reqJson\accuUsersInfo.json ==========
-match percentage: 94.0 %
-
-Map(first_name -> Tom049, email -> tomajerrya044@dc.org, post_code -> 94022, user_id -> 10044, last_name -> Jerrya, address -> 244 DisneyCity, phone -> 10000044)
-Map(first_name -> Tom049, email -> tomajerrya049@dc.org, post_code -> 94022, user_id -> 10050, last_name -> Jerrya, address -> 249 DisneyCity, phone -> 10000049)
-Map(first_name -> Tom039, email -> tomajerrya039@dc.org, post_code -> 94022, user_id -> 10039, last_name -> Jerrya, address -> 239 DisneyCity123, phone -> 10000039)
-

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-models/data/test/reqJson/accuUsersInfo.json
----------------------------------------------------------------------
diff --git a/griffin-models/data/test/reqJson/accuUsersInfo.json b/griffin-models/data/test/reqJson/accuUsersInfo.json
deleted file mode 100644
index 02d5dd2..0000000
--- a/griffin-models/data/test/reqJson/accuUsersInfo.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-  "source": "users_info_src",
-  "target": "users_info_target",
-  "accuracyMapping": [
-    {
-      "sourceColId": 0,
-      "sourceColName": "user_id",
-      "targetColId": 0,
-      "targetColName": "user_id",
-      "isPK": true
-    },
-    {
-      "sourceColId": 1,
-      "sourceColName": "first_name",
-      "targetColId": 1,
-      "targetColName": "first_name",
-      "isPK": false
-    },
-    {
-      "sourceColId": 2,
-      "sourceColName": "last_name",
-      "targetColId": 2,
-      "targetColName": "last_name",
-      "isPK": false
-    },
-    {
-      "sourceColId": 3,
-      "sourceColName": "address",
-      "targetColId": 3,
-      "targetColName": "address",
-      "isPK": false
-    },
-    {
-      "sourceColId": 4,
-      "sourceColName": "email",
-      "targetColId": 4,
-      "targetColName": "email",
-      "isPK": false
-    },
-    {
-      "sourceColId": 5,
-      "sourceColName": "phone",
-      "targetColId": 5,
-      "targetColName": "phone",
-      "isPK": false
-    },
-    {
-      "sourceColId": 6,
-      "sourceColName": "post_code",
-      "targetColId": 6,
-      "targetColName": "post_code",
-      "isPK": false
-    }
-  ]
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-models/src/test/scala/modelTest/AccuTest.scala
----------------------------------------------------------------------
diff --git a/griffin-models/src/test/scala/modelTest/AccuTest.scala b/griffin-models/src/test/scala/modelTest/AccuTest.scala
deleted file mode 100644
index b7ebed8..0000000
--- a/griffin-models/src/test/scala/modelTest/AccuTest.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-package modelTest
-
-import org.apache.griffin.accuracy._
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import org.apache.spark.{SparkConf, SparkContext}
-import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
-import org.apache.spark.sql.{DataFrame, SQLContext}
-import java.io.{FileInputStream, FileOutputStream}
-
-import org.apache.griffin.dataLoaderUtils.{DataLoaderFactory, FileLoaderUtil}
-
-import scala.collection.mutable.MutableList
-import org.junit.runner.RunWith
-import org.scalatest.junit.JUnitRunner
-@RunWith(classOf[JUnitRunner])
-class AccuTest extends FunSuite with Matchers with BeforeAndAfter {
-
-  val dataFilePath = FileLoaderUtil.convertPath("data/test/dataFile/")
-  val reqJsonPath = FileLoaderUtil.convertPath("data/test/reqJson/")
-  val recordFilePath = FileLoaderUtil.convertPath("data/test/recordFile/")
-  val recordFileName = "_RESULT_ACCU"
-
-  case class AccuData() {
-    var cnt: Int = _
-    var reqJson: String = _
-    var configure: AccuracyConfEntity = _
-    var dataFrameSrc: DataFrame = _
-    var dataFrameTgt: DataFrame = _
-    var result: ((Long, Long), List[String]) = _
-  }
-  val accuDatas = MutableList[AccuData]()
-
-  var sc: SparkContext = _
-
-  before {
-    val conf = new SparkConf().setMaster("local[*]").setAppName("AccTest")
-    sc = new SparkContext(conf)
-    val sqlContext = new SQLContext(sc)
-    val mapper = new ObjectMapper()
-    mapper.registerModule(DefaultScalaModule)
-
-    var cnt = 1;
-    val accTests = List("accuAvroTest.json", "accuUsersInfo.json")
-    for (tf <- accTests) {
-      val reqJson = reqJsonPath + tf
-      val accuData = new AccuData()
-      accuData.cnt = cnt
-      accuData.reqJson = reqJson
-      val input = new FileInputStream(reqJson)
-      accuData.configure = mapper.readValue(input, classOf[AccuracyConfEntity])
-      val dataLoader = DataLoaderFactory.getDataLoader(sqlContext, DataLoaderFactory.avro, dataFilePath)
-      val dfs = dataLoader.getAccuDataFrame(accuData.configure)
-      accuData.dataFrameSrc = dfs._1
-      accuData.dataFrameTgt = dfs._2
-      accuDatas += accuData
-      cnt += 1
-    }
-  }
-
-  test("test accuracy requests") {
-    for (accuData <- accuDatas) {
-      //-- algorithm --
-      accuData.result = Accu.calcAccu(accuData.configure, accuData.dataFrameSrc, accuData.dataFrameTgt)
-    }
-  }
-
-  after {
-    val out = new FileOutputStream(recordFilePath + recordFileName)
-    for (accuData <- accuDatas) {
-      //output
-      out.write(("//" + "=" * 10).getBytes("utf-8"))
-      out.write((s" ${accuData.cnt}. Test Accuracy model result with request file: ${accuData.reqJson} ").getBytes("utf-8"))
-      out.write(("=" * 10 + "\n").getBytes("utf-8"))
-
-      val ((missCount, srcCount), missedList) = accuData.result
-      val rslt = s"match percentage: ${((1 - missCount.toDouble / srcCount) * 100)} %"
-      val rcds = missedList.mkString("\n")
-      val rcd = rslt + "\n\n" + rcds + "\n\n";
-
-      out.write(rcd.getBytes("utf-8"))
-    }
-    out.close()
-    sc.stop()
-  }
-}



[2/3] incubator-griffin git commit: clean files

Posted by gu...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParser.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParser.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParser.scala
deleted file mode 100644
index 6070189..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParser.scala
+++ /dev/null
@@ -1,298 +0,0 @@
-package org.apache.griffin.measure.batch.rule
-
-import org.apache.griffin.measure.batch.rule.expr._
-
-import scala.util.parsing.combinator._
-
-case class RuleParser() extends JavaTokenParsers with Serializable {
-
-  /**
-    * BNF representation for grammar as below:
-    *
-    * <rule> ::= <logical-statement> [WHEN <logical-statement>]
-    * rule: mapping-rule [WHEN when-rule]
-    * - mapping-rule: the first level opr should better not be OR | NOT, otherwise it can't automatically find the groupby column
-    * - when-rule: only contain the general info of data source, not the special info of each data row
-    *
-    * <logical-statement> ::= [NOT] <logical-expression> [(AND | OR) <logical-expression>]+ | "(" <logical-statement> ")"
-    * logical-statement: return boolean value
-    * logical-operator: "AND" | "&&", "OR" | "||", "NOT" | "!"
-    *
-    * <logical-expression> ::= <math-expr> (<compare-opr> <math-expr> | <range-opr> <range-expr>)
-    * logical-expression example: $source.id = $target.id, $source.page_id IN ('3214', '4312', '60821')
-    *
-    * <compare-opr> ::= "=" | "!=" | "<" | ">" | "<=" | ">="
-    * <range-opr> ::= ["NOT"] "IN" | "BETWEEN"
-    * <range-expr> ::= "(" [<math-expr>] [, <math-expr>]+ ")"
-    * range-expr example: ('3214', '4312', '60821'), (10, 15), ()
-    *
-    * <math-expr> ::= [<unary-opr>] <math-factor> [<binary-opr> <math-factor>]+
-    * math-expr example: $source.price * $target.count, "hello" + " " + "world" + 123
-    *
-    * <binary-opr> ::= "+" | "-" | "*" | "/" | "%"
-    * <unary-opr> ::= "+" | "-"
-    *
-    * <math-factor> ::= <literal> | <selection> | "(" <math-expr> ")"
-    *
-    * <selection> ::= <selection-head> [ <field-sel> | <function-operation> | <index-field-range-sel> | <filter-sel> ]+
-    * selection example: $source.price, $source.json(), $source['state'], $source.numList[3], $target.json().mails['org' = 'apache'].names[*]
-    *
-    * <selection-head> ::= $source | $target
-    *
-    * <field-sel> ::= "." <field-string>
-    *
-    * <function-operation> ::= "." <function-name> "(" <arg> [, <arg>]+ ")"
-    * <function-name> ::= <name-string>
-    * <arg> ::= <math-expr>
-    *
-    * <index-field-range-sel> ::= "[" <index-field-range> [, <index-field-range>]+ "]"
-    * <index-field-range> ::= <index-field> | (<index-field>, <index-field>) | "*"
-    * index-field-range: 2 means the 3rd item, (0, 3) means first 4 items, * means all items, 'age' means item 'age'
-    * <index-field> ::= <index> | <field-quote> | <all-selection>
-    * index: 0 ~ n means position from start, -1 ~ -n means position from end
-    * <field-quote> ::= ' <field-string> ' | " <field-string> "
-    *
-    * <filter-sel> ::= "[" <field-quote> <filter-compare-opr> <math-expr> "]"
-    * <filter-compare-opr> ::= "=" | "!=" | "<" | ">" | "<=" | ">="
-    * filter-sel example: ['name' = 'URL'], $source.man['age' > $source.graduate_age + 5 ]
-    *
-    * When <math-expr> in the selection, it mustn't contain the different <selection-head>, for example:
-    * $source.tags[1+2]             valid
-    * $source.tags[$source.first]   valid
-    * $source.tags[$target.first]   invalid
-    * -- Such job is for validation, not for parser
-    *
-    *
-    * <literal> ::= <literal-string> | <literal-number> | <literal-time> | <literal-boolean>
-    * <literal-string> ::= <any-string>
-    * <literal-number> ::= <integer> | <double>
-    * <literal-time> ::= <integer> ("d"|"h"|"m"|"s"|"ms")
-    * <literal-boolean> ::= true | false
-    *
-    */
-
-  object Keyword {
-    def WhenKeywords: Parser[String] = """(?i)when""".r
-    def UnaryLogicalKeywords: Parser[String] = """(?i)not""".r
-    def BinaryLogicalKeywords: Parser[String] = """(?i)and|or""".r
-    def RangeKeywords: Parser[String] = """(?i)(not\s+)?(in|between)""".r
-    def DataSourceKeywords: Parser[String] = """(?i)\$(source|target)""".r
-    def Keywords: Parser[String] = WhenKeywords | UnaryLogicalKeywords | BinaryLogicalKeywords | RangeKeywords | DataSourceKeywords
-  }
-  import Keyword._
-
-  object Operator {
-    def NotLogicalOpr: Parser[String] = """(?i)not""".r | "!"
-    def AndLogicalOpr: Parser[String] = """(?i)and""".r | "&&"
-    def OrLogicalOpr: Parser[String] = """(?i)or""".r | "||"
-    def CompareOpr: Parser[String] = """!?==?""".r | """<=?""".r | """>=?""".r
-    def RangeOpr: Parser[String] = RangeKeywords
-
-    def UnaryMathOpr: Parser[String] = "+" | "-"
-    def BinaryMathOpr1: Parser[String] = "*" | "/" | "%"
-    def BinaryMathOpr2: Parser[String] = "+" | "-"
-
-    def FilterCompareOpr: Parser[String] = """!?==?""".r | """<=?""".r | """>=?""".r
-
-    def SqBracketPair: (Parser[String], Parser[String]) = ("[", "]")
-    def BracketPair: (Parser[String], Parser[String]) = ("(", ")")
-    def Dot: Parser[String] = "."
-    def AllSelection: Parser[String] = "*"
-    def SQuote: Parser[String] = "'"
-    def DQuote: Parser[String] = "\""
-    def Comma: Parser[String] = ","
-  }
-  import Operator._
-
-  object SomeString {
-    def AnyString: Parser[String] = """[^'\"{}\[\]()=<>.$@,;+\-*/\\]*""".r
-    def SimpleFieldString: Parser[String] = """\w+""".r
-    def FieldString: Parser[String] = """[\w\s]+""".r
-    def NameString: Parser[String] = """[a-zA-Z_]\w*""".r
-  }
-  import SomeString._
-
-  object SomeNumber {
-    def IntegerNumber: Parser[String] = """[+\-]?\d+""".r
-    def DoubleNumber: Parser[String] = """[+\-]?(\.\d+|\d+\.\d*)""".r
-    def IndexNumber: Parser[String] = IntegerNumber
-  }
-  import SomeNumber._
-
-  // -- literal --
-  def literal: Parser[LiteralExpr] = literialString | literialTime | literialNumber | literialBoolean
-  def literialString: Parser[LiteralStringExpr] = (SQuote ~> AnyString <~ SQuote | DQuote ~> AnyString <~ DQuote) ^^ { LiteralStringExpr(_) }
-  def literialNumber: Parser[LiteralNumberExpr] = (DoubleNumber | IntegerNumber) ^^ { LiteralNumberExpr(_) }
-  def literialTime: Parser[LiteralTimeExpr] = """(\d+(d|h|m|s|ms))+""".r ^^ { LiteralTimeExpr(_) }
-  def literialBoolean: Parser[LiteralBooleanExpr] = ("""(?i)true""".r | """(?i)false""".r) ^^ { LiteralBooleanExpr(_) }
-
-  // -- selection --
-  // <selection> ::= <selection-head> [ <field-sel> | <function-operation> | <index-field-range-sel> | <filter-sel> ]+
-  def selection: Parser[SelectionExpr] = selectionHead ~ rep(selector) ^^ {
-    case head ~ selectors => SelectionExpr(head, selectors)
-  }
-  def selector: Parser[SelectExpr] = (fieldSelect | functionOperation | indexFieldRangeSelect | filterSelect)
-
-  def selectionHead: Parser[SelectionHead] = DataSourceKeywords ^^ { SelectionHead(_) }
-  // <field-sel> ::= "." <field-string>
-  def fieldSelect: Parser[IndexFieldRangeSelectExpr] = Dot ~> SimpleFieldString ^^ {
-    case field => IndexFieldRangeSelectExpr(FieldDesc(field) :: Nil)
-  }
-  // <function-operation> ::= "." <function-name> "(" <arg> [, <arg>]+ ")"
-  def functionOperation: Parser[FunctionOperationExpr] = Dot ~ NameString ~ BracketPair._1 ~ repsep(argument, Comma) ~ BracketPair._2 ^^ {
-    case _ ~ func ~ _ ~ args ~ _ => FunctionOperationExpr(func, args)
-  }
-  def argument: Parser[MathExpr] = mathExpr
-  // <index-field-range-sel> ::= "[" <index-field-range> [, <index-field-range>]+ "]"
-  def indexFieldRangeSelect: Parser[IndexFieldRangeSelectExpr] = SqBracketPair._1 ~> rep1sep(indexFieldRange, Comma) <~ SqBracketPair._2 ^^ {
-    case ifrs => IndexFieldRangeSelectExpr(ifrs)
-  }
-  // <index-field-range> ::= <index-field> | (<index-field>, <index-field>) | "*"
-  def indexFieldRange: Parser[FieldDescOnly] = indexField | BracketPair._1 ~ indexField ~ Comma ~ indexField ~ BracketPair._2 ^^ {
-    case _ ~ if1 ~ _ ~ if2 ~ _ => FieldRangeDesc(if1, if2)
-  }
-  // <index-field> ::= <index> | <field-quote> | <all-selection>
-  // *here it can parse <math-expr>, but for simple situation, not supported now*
-  def indexField: Parser[FieldDescOnly] = IndexNumber ^^ { IndexDesc(_) } | fieldQuote | AllSelection ^^ { AllFieldsDesc(_) }
-  // <field-quote> ::= ' <field-string> ' | " <field-string> "
-  def fieldQuote: Parser[FieldDesc] = (SQuote ~> FieldString <~ SQuote | DQuote ~> FieldString <~ DQuote) ^^ { FieldDesc(_) }
-  // <filter-sel> ::= "[" <field-quote> <filter-compare-opr> <math-expr> "]"
-  def filterSelect: Parser[FilterSelectExpr] = SqBracketPair._1 ~> fieldQuote ~ FilterCompareOpr ~ mathExpr <~ SqBracketPair._2 ^^ {
-    case field ~ compare ~ value => FilterSelectExpr(field, compare, value)
-  }
-
-  // -- math --
-  // <math-factor> ::= <literal> | <selection> | "(" <math-expr> ")"
-  def mathFactor: Parser[MathExpr] = (literal | selection | BracketPair._1 ~> mathExpr <~ BracketPair._2) ^^ { MathFactorExpr(_) }
-  // <math-expr> ::= [<unary-opr>] <math-factor> [<binary-opr> <math-factor>]+
-  // <unary-opr> ::= "+" | "-"
-  def unaryMathExpr: Parser[MathExpr] = rep(UnaryMathOpr) ~ mathFactor ^^ {
-    case Nil ~ a => a
-    case list ~ a => UnaryMathExpr(list, a)
-  }
-  // <binary-opr> ::= "+" | "-" | "*" | "/" | "%"
-  def binaryMathExpr1: Parser[MathExpr] = unaryMathExpr ~ rep(BinaryMathOpr1 ~ unaryMathExpr) ^^ {
-    case a ~ Nil => a
-    case a ~ list => BinaryMathExpr(a, list.map(c => (c._1, c._2)))
-  }
-  def binaryMathExpr2: Parser[MathExpr] = binaryMathExpr1 ~ rep(BinaryMathOpr2 ~ binaryMathExpr1) ^^ {
-    case a ~ Nil => a
-    case a ~ list => BinaryMathExpr(a, list.map(c => (c._1, c._2)))
-  }
-  def mathExpr: Parser[MathExpr] = binaryMathExpr2
-
-  // -- logical expression --
-  // <range-expr> ::= "(" [<math-expr>] [, <math-expr>]+ ")"
-  def rangeExpr: Parser[RangeDesc] = BracketPair._1 ~> repsep(mathExpr, Comma) <~ BracketPair._2 ^^ { RangeDesc(_) }
-  // <logical-expression> ::= <math-expr> (<compare-opr> <math-expr> | <range-opr> <range-expr>)
-  def logicalExpr: Parser[LogicalExpr] = mathExpr ~ CompareOpr ~ mathExpr ^^ {
-    case left ~ opr ~ right => LogicalCompareExpr(left, opr, right)
-  } | mathExpr ~ RangeOpr ~ rangeExpr ^^ {
-    case left ~ opr ~ range => LogicalRangeExpr(left, opr, range)
-  }
-
-  // -- logical statement --
-  def logicalFactor: Parser[LogicalExpr] = logicalExpr | BracketPair._1 ~> logicalStatement <~ BracketPair._2
-  def notLogicalStatement: Parser[LogicalExpr] = rep(NotLogicalOpr) ~ logicalFactor ^^ {
-    case Nil ~ a => a
-    case list ~ a => UnaryLogicalExpr(list, a)
-  }
-  def andLogicalStatement: Parser[LogicalExpr] = notLogicalStatement ~ rep(AndLogicalOpr ~ notLogicalStatement) ^^ {
-    case a ~ Nil => a
-    case a ~ list => BinaryLogicalExpr(a, list.map(c => (c._1, c._2)))
-  }
-  def orLogicalStatement: Parser[LogicalExpr] = andLogicalStatement ~ rep(OrLogicalOpr ~ andLogicalStatement) ^^ {
-    case a ~ Nil => a
-    case a ~ list => BinaryLogicalExpr(a, list.map(c => (c._1, c._2)))
-  }
-  // <logical-statement> ::= [NOT] <logical-expression> [(AND | OR) <logical-expression>]+ | "(" <logical-statement> ")"
-  def logicalStatement: Parser[LogicalExpr] = orLogicalStatement
-
-  // -- rule --
-  // <rule> ::= <logical-statement> [WHEN <logical-statement>]
-  def rule: Parser[StatementExpr] = logicalStatement ~ opt(WhenKeywords ~> logicalStatement) ^^ {
-    case ls ~ Some(ws) => WhenClauseStatementExpr(ls, ws)
-    case ls ~ _ => SimpleStatementExpr(ls)
-  }
-
-  // for complie only
-//  case class NullStatementExpr(expression: String) extends StatementExpr {
-//    def genValue(values: Map[String, Any]): Option[Any] = None
-//    def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = Nil
-//  }
-//  def statementsExpr = mathExpr ^^ { NullStatementExpr(_) }
-
-
-//
-//  // basic
-//  val anyString: Parser[String] = """[^'{}\[\]()=<>.$@;+\-*/\\\"]*""".r
-//  val variable: Parser[String] = """[a-zA-Z_]\w*""".r
-//  val number: Parser[String] = """[+\-]?\d+""".r
-//  val time: Parser[String] = """\d+(y|M|w|d|h|m|s|ms)""".r
-//
-//  val numPosition: Parser[String] = """\d+""".r
-//  val anyPosition: Parser[String] = "*"
-//
-//  val filterOpr: Parser[String] = "=" | "!=" | ">" | "<" | ">=" | "<="
-//
-//  val opr1: Parser[String] = "*" | "/" | "%"
-//  val opr2: Parser[String] = "+" | "-"
-//
-//  val assignOpr: Parser[String] = "="
-//  val compareOpr: Parser[String] = "==" | "!=" | ">" | "<" | ">=" | "<="
-//  val mappingOpr: Parser[String] = "==="
-//
-//  val exprSep: Parser[String] = ";"
-//
-//  // simple
-//  def variableString: Parser[VariableExpr] = (("'" ~> anyString <~ "'") | ("\"" ~> anyString <~ "\"")) ^^ { VariableStringExpr(_) }
-//  def constString: Parser[ConstExpr] = (("'" ~> anyString <~ "'") | ("\"" ~> anyString <~ "\"")) ^^ { ConstStringExpr(_) }
-//  def constValue: Parser[ConstExpr] = time ^^ { ConstTimeExpr(_) } | number ^^ { ConstNumberExpr(_)} | constString
-//  def variableValue: Parser[VariableExpr] = variable ^^ { VariableStringExpr(_) }
-//  def quoteVariableValue: Parser[QuoteVariableExpr] = "${" ~> variable <~ "}" ^^ { QuoteVariableExpr(_) }
-//  def position: Parser[SelectExpr] = anyPosition ^^ { AnyPositionExpr(_) } | """\d+""".r ^^ { NumPositionExpr(_) } | (("'" ~> anyString <~ "'") | ("\"" ~> anyString <~ "\"")) ^^ { StringPositionExpr(_) }
-//  def argument: Parser[ConstExpr] = constValue
-//  def annotationExpr: Parser[AnnotationExpr] = "@" ~> variable ^^ { AnnotationExpr(_) }
-//
-//  // selector
-//  def filterOpration: Parser[SelectExpr] = (variableString ~ filterOpr ~ constString) ^^ {
-//    case v ~ opr ~ c => FilterOprExpr(opr, v, c)
-//  }
-//  def positionExpr: Parser[SelectExpr] = "[" ~> (filterOpration | position) <~ "]"
-//  def functionExpr: Parser[SelectExpr] = "." ~ variable ~ "(" ~ repsep(argument, ",") ~ ")" ^^ {
-//    case _ ~ v ~ _ ~ args ~ _ => FunctionExpr(v, args)
-//  }
-//  def selectorExpr: Parser[SelectExpr] = positionExpr | functionExpr
-//
-//  // data
-//  def selectorsExpr: Parser[DataExpr] = quoteVariableValue ~ rep(selectorExpr) ^^ {
-//    case q ~ tails => SelectionExpr(q, tails)
-//  }
-//
-//  // calculation
-//  def factor: Parser[ElementExpr] = (constValue | selectorsExpr | "(" ~> expr <~ ")") ^^ { FactorExpr(_) }
-//  def term: Parser[ElementExpr] = factor ~ rep(opr1 ~ factor) ^^ {
-//    case a ~ Nil => a
-//    case a ~ list => CalculationExpr(a, list.map(c => (c._1, c._2)))
-//  }
-//  def expr: Parser[ElementExpr] = term ~ rep(opr2 ~ term) ^^ {
-//    case a ~ Nil => a
-//    case a ~ list => CalculationExpr(a, list.map(c => (c._1, c._2)))
-//  }
-//
-//  // statement
-//  def assignExpr: Parser[StatementExpr] = variableValue ~ assignOpr ~ expr ^^ {
-//    case v ~ opr ~ c => AssignExpr(opr, v, c)
-//  }
-//  def conditionExpr: Parser[StatementExpr] = rep(annotationExpr) ~ expr ~ compareOpr ~ expr ^^ {
-//    case anos ~ le ~ opr ~ re => ConditionExpr(opr, le, re, anos)
-//  }
-//  def mappingExpr: Parser[StatementExpr] = rep(annotationExpr) ~ expr ~ mappingOpr ~ expr ^^ {
-//    case anos ~ le ~ opr ~ re => MappingExpr(opr, le, re, anos)
-//  }
-//  def statementExpr: Parser[StatementExpr] = assignExpr | conditionExpr | mappingExpr
-//
-//  // statements
-//  def statementsExpr: Parser[StatementExpr] = repsep(statementExpr, exprSep) ^^ { StatementsExpr(_) }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParserDescription.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParserDescription.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParserDescription.scala
deleted file mode 100644
index 18810b3..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleParserDescription.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
-  * BNF representation for grammar as below:
-  *
-  * <rule> ::= <logical-statement> [WHEN <logical-statement>]
-  * rule: mapping-rule [WHEN when-rule]
-  * - mapping-rule: the first level opr should better not be OR | NOT, otherwise it can't automatically find the groupby column
-  * - when-rule: only contain the general info of data source, not the special info of each data row
-  *
-  * <logical-statement> ::= [NOT] <logical-expression> [(AND | OR) <logical-expression>]+ | "(" <logical-statement> ")"
-  * logical-statement: return boolean value
-  * logical-operator: "AND" | "&&", "OR" | "||", "NOT" | "!"
-  *
-  * <logical-expression> ::= <math-expr> (<compare-opr> <math-expr> | <range-opr> <range-expr>)
-  * logical-expression example: $source.id = $target.id, $source.page_id IN ('3214', '4312', '60821')
-  *
-  * <compare-opr> ::= "=" | "!=" | "<" | ">" | "<=" | ">="
-  * <range-opr> ::= ["NOT"] "IN" | "BETWEEN"
-  * <range-expr> ::= "(" [<math-expr>] [, <math-expr>]+ ")"
-  * range-expr example: ('3214', '4312', '60821'), (10, 15), ()
-  *
-  * <math-expr> ::= [<unary-opr>] <math-factor> [<binary-opr> <math-factor>]+
-  * math-expr example: $source.price * $target.count, "hello" + " " + "world" + 123
-  *
-  * <binary-opr> ::= "+" | "-" | "*" | "/" | "%"
-  * <unary-opr> ::= "+" | "-"
-  *
-  * <math-factor> ::= <literal> | <selection> | "(" <math-expr> ")"
-  *
-  * <selection> ::= <selection-head> [ <field-sel> | <function-operation> | <index-field-range-sel> | <filter-sel> ]+
-  * selection example: $source.price, $source.json(), $source['state'], $source.numList[3], $target.json().mails['org' = 'apache'].names[*]
-  *
-  * <selection-head> ::= $source | $target
-  *
-  * <field-sel> ::= "." <field-string>
-  *
-  * <function-operation> ::= "." <function-name> "(" <arg> [, <arg>]+ ")"
-  * <function-name> ::= <name-string>
-  * <arg> ::= <math-expr>
-  *
-  * <index-field-range-sel> ::= "[" <index-field-range> [, <index-field-range>]+ "]"
-  * <index-field-range> ::= <index-field> | (<index-field>, <index-field>) | "*"
-  * index-field-range: 2 means the 3rd item, (0, 3) means first 4 items, * means all items, 'age' means item 'age'
-  * <index-field> ::= <index> | <field-quote> | <all-selection>
-  * index: 0 ~ n means position from start, -1 ~ -n means position from end
-  * <field-quote> ::= ' <field-string> ' | " <field-string> "
-  *
-  * <filter-sel> ::= "[" <field-quote> <filter-compare-opr> <math-expr> "]"
-  * <filter-compare-opr> ::= "=" | "!=" | "<" | ">" | "<=" | ">="
-  * filter-sel example: ['name' = 'URL'], $source.man['age' > $source.graduate_age + 5 ]
-  *
-  * When <math-expr> in the selection, it mustn't contain the different <selection-head>, for example:
-  * $source.tags[1+2]             valid
-  * $source.tags[$source.first]   valid
-  * $source.tags[$target.first]   invalid
-  * -- Such job is for validation, not for parser
-  *
-  *
-  * <literal> ::= <literal-string> | <literal-number> | <literal-time> | <literal-boolean>
-  * <literal-string> ::= <any-string>
-  * <literal-number> ::= <integer> | <double>
-  * <literal-time> ::= <integer> ("d"|"h"|"m"|"s"|"ms")
-  * <literal-boolean> ::= true | false
-  *
-  */
-
-
-/**
-  * BACK_UP
-  *
-  * <index-field-range-sel> ::= "[" <index-field-range> [, <index-field-range>]+ "]"
-  * <index-field-range> ::= <index-field> | (<index-field>, <index-field>) | "*"
-  * index-field-range: 2 means the 3rd item, (0, 3) means first 4 items, * means all items, 'age' means item 'age'
-  * <index-field> ::= <index> | <field-quote> | <math-result>
-  * <field-quote> ::= ' <any-string> ' | \" <any-string> \"
-  * <index> ::= <integer>
-  * index: 0 ~ n means position from start, -1 ~ -n means position from end
-  * <math-result> ::= "${" <math-expr> "}"
-  *
-  * <filter-sel> ::= "[" <field-quote> <filter-compare-opr> <filter-value> "]"
-  * <filter-compare-opr> ::= "=" | "!=" | "<" | ">" | "<=" | ">="
-  * <filter-value> ::= <literal> | <math-result>
-  * filter-sel example: ['name' = 'URL'], $source.man['age' > ${ $source.graduate_age + 5 }]
-  *
-  */

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/AnalyzableExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/AnalyzableExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/AnalyzableExpr.scala
deleted file mode 100644
index 0d54707..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/AnalyzableExpr.scala
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-
-trait AnalyzableExpr extends Serializable {
-  def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = Nil
-  def getWhenClauseExpr(): Option[LogicalExpr] = None
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Cacheable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Cacheable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Cacheable.scala
deleted file mode 100644
index e062376..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Cacheable.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait Cacheable extends DataSourceable {
-  protected def cacheUnit: Boolean = false
-  def cacheable(ds: String): Boolean = {
-    cacheUnit && !conflict() && ((ds.isEmpty && dataSources.isEmpty) || (ds.nonEmpty && contains(ds)))
-  }
-  protected def getCacheExprs(ds: String): Iterable[Cacheable]
-
-  protected def persistUnit: Boolean = false
-  def persistable(ds: String): Boolean = {
-    persistUnit && ((ds.isEmpty && dataSources.isEmpty) || (ds.nonEmpty && contains(ds)))
-  }
-  protected def getPersistExprs(ds: String): Iterable[Cacheable]
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Calculatable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Calculatable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Calculatable.scala
deleted file mode 100644
index 8018c19..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Calculatable.scala
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait Calculatable extends Serializable {
-
-  def calculate(values: Map[String, Any]): Option[Any]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/DataSourceable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/DataSourceable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/DataSourceable.scala
deleted file mode 100644
index f18798a..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/DataSourceable.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait DataSourceable extends Serializable {
-  val dataSources: Set[String]
-  protected def conflict(): Boolean = dataSources.size > 1
-  def contains(ds: String): Boolean = dataSources.contains(ds)
-  def dataSourceOpt: Option[String] = {
-    if (dataSources.size == 1) Some(dataSources.head) else None
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Describable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Describable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Describable.scala
deleted file mode 100644
index 38758a2..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Describable.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait Describable extends Serializable {
-
-  val desc: String
-
-  protected def describe(v: Any): String = {
-    v match {
-      case s: Describable => s"${s.desc}"
-      case s: String => s"'${s}'"
-      case a => s"${a}"
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Expr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Expr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Expr.scala
deleted file mode 100644
index d7810aa..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/Expr.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait Expr extends Serializable with Describable with Cacheable with Calculatable {
-
-  protected val _defaultId: String = ExprIdCounter.emptyId
-
-  val _id = ExprIdCounter.genId(_defaultId)
-
-  protected def getSubCacheExprs(ds: String): Iterable[Expr] = Nil
-  final def getCacheExprs(ds: String): Iterable[Expr] = {
-    if (cacheable(ds)) getSubCacheExprs(ds).toList :+ this else getSubCacheExprs(ds)
-  }
-
-  protected def getSubFinalCacheExprs(ds: String): Iterable[Expr] = Nil
-  final def getFinalCacheExprs(ds: String): Iterable[Expr] = {
-    if (cacheable(ds)) Nil :+ this else getSubFinalCacheExprs(ds)
-  }
-
-  protected def getSubPersistExprs(ds: String): Iterable[Expr] = Nil
-  final def getPersistExprs(ds: String): Iterable[Expr] = {
-    if (persistable(ds)) getSubPersistExprs(ds).toList :+ this else getSubPersistExprs(ds)
-  }
-
-  final def calculate(values: Map[String, Any]): Option[Any] = {
-    values.get(_id) match {
-      case Some(v) => Some(v)
-      case _ => calculateOnly(values)
-    }
-  }
-  protected def calculateOnly(values: Map[String, Any]): Option[Any]
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprDescOnly.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprDescOnly.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprDescOnly.scala
deleted file mode 100644
index 0bb5085..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprDescOnly.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait ExprDescOnly extends Describable {
-
-}
-
-
-case class SelectionHead(expr: String) extends ExprDescOnly {
-  private val headRegex = """\$(\w+)""".r
-  val head: String = expr match {
-    case headRegex(v) => v.toLowerCase
-    case _ => expr
-  }
-  val desc: String = "$" + head
-}
-
-case class RangeDesc(elements: Iterable[MathExpr]) extends ExprDescOnly {
-  val desc: String = {
-    val rangeDesc = elements.map(_.desc).mkString(", ")
-    s"(${rangeDesc})"
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprIdCounter.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprIdCounter.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprIdCounter.scala
deleted file mode 100644
index 56e7daa..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/ExprIdCounter.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-import java.util.concurrent.atomic.AtomicLong
-
-import scala.collection.mutable.{Set => MutableSet}
-
-object ExprIdCounter {
-
-  private val idCounter: AtomicLong = new AtomicLong(0L)
-
-  private val existIdSet: MutableSet[String] = MutableSet.empty[String]
-
-  private val invalidIdRegex = """^\d+$""".r
-
-  val emptyId: String = ""
-
-  def genId(defaultId: String): String = {
-    defaultId match {
-      case emptyId => increment.toString
-      case invalidIdRegex() => increment.toString
-//      case defId if (exist(defId)) => s"${increment}#${defId}"
-      case defId if (exist(defId)) => s"${defId}"
-      case _ => {
-        insertUserId(defaultId)
-        defaultId
-      }
-    }
-  }
-
-  private def exist(id: String): Boolean = {
-    existIdSet.contains(id)
-  }
-
-  private def insertUserId(id: String): Unit = {
-    existIdSet += id
-  }
-
-  private def increment(): Long = {
-    idCounter.incrementAndGet()
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/FieldDescOnly.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/FieldDescOnly.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/FieldDescOnly.scala
deleted file mode 100644
index f13f15a..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/FieldDescOnly.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-import scala.util.{Success, Try}
-
-trait FieldDescOnly extends Describable with DataSourceable {
-
-}
-
-case class IndexDesc(expr: String) extends FieldDescOnly {
-  val index: Int = {
-    Try(expr.toInt) match {
-      case Success(v) => v
-      case _ => throw new Exception(s"${expr} is invalid index")
-    }
-  }
-  val desc: String = describe(index)
-  val dataSources: Set[String] = Set.empty[String]
-}
-
-case class FieldDesc(expr: String) extends FieldDescOnly {
-  val field: String = expr
-  val desc: String = describe(field)
-  val dataSources: Set[String] = Set.empty[String]
-}
-
-case class AllFieldsDesc(expr: String) extends FieldDescOnly {
-  val allFields: String = expr
-  val desc: String = allFields
-  val dataSources: Set[String] = Set.empty[String]
-}
-
-case class FieldRangeDesc(startField: FieldDescOnly, endField: FieldDescOnly) extends FieldDescOnly {
-  val desc: String = {
-    (startField, endField) match {
-      case (f1: IndexDesc, f2: IndexDesc) => s"(${f1.desc}, ${f2.desc})"
-      case _ => throw new Exception("invalid field range description")
-    }
-  }
-  val dataSources: Set[String] = Set.empty[String]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LiteralExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LiteralExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LiteralExpr.scala
deleted file mode 100644
index 020ddc2..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LiteralExpr.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-import scala.util.{Failure, Success, Try}
-
-trait LiteralExpr extends Expr {
-  val value: Option[Any]
-  def calculateOnly(values: Map[String, Any]): Option[Any] = value
-  val dataSources: Set[String] = Set.empty[String]
-}
-
-case class LiteralStringExpr(expr: String) extends LiteralExpr {
-  val value: Option[String] = Some(expr)
-  val desc: String = value.getOrElse("")
-}
-
-case class LiteralNumberExpr(expr: String) extends LiteralExpr {
-  val value: Option[Any] = {
-    if (expr.contains(".")) {
-      Try (expr.toDouble) match {
-        case Success(v) => Some(v)
-        case _ => throw new Exception(s"${expr} is invalid number")
-      }
-    } else {
-      Try (expr.toLong) match {
-        case Success(v) => Some(v)
-        case _ => throw new Exception(s"${expr} is invalid number")
-      }
-    }
-  }
-  val desc: String = value.getOrElse("").toString
-}
-
-case class LiteralTimeExpr(expr: String) extends LiteralExpr {
-  final val TimeRegex = """(\d+)(d|h|m|s|ms)""".r
-  val value: Option[Long] = {
-    Try {
-      expr match {
-        case TimeRegex(time, unit) => {
-          val t = time.toLong
-          unit match {
-            case "d" => t * 24 * 60 * 60 * 1000
-            case "h" => t * 60 * 60 * 1000
-            case "m" => t * 60 * 1000
-            case "s" => t * 1000
-            case "ms" => t
-            case _ => throw new Exception(s"${expr} is invalid time format")
-          }
-        }
-        case _ => throw new Exception(s"${expr} is invalid time format")
-      }
-    } match {
-      case Success(v) => Some(v)
-      case Failure(ex) => throw ex
-    }
-  }
-  val desc: String = expr
-}
-
-case class LiteralBooleanExpr(expr: String) extends LiteralExpr {
-  final val TrueRegex = """(?i)true""".r
-  final val FalseRegex = """(?i)false""".r
-  val value: Option[Boolean] = expr match {
-    case TrueRegex() => Some(true)
-    case FalseRegex() => Some(false)
-    case _ => throw new Exception(s"${expr} is invalid boolean")
-  }
-  val desc: String = value.getOrElse("").toString
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LogicalExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LogicalExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LogicalExpr.scala
deleted file mode 100644
index 74e80e3..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/LogicalExpr.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-import org.apache.griffin.measure.batch.utils.CalculationUtil._
-
-trait LogicalExpr extends Expr with AnalyzableExpr {
-  override def cacheUnit: Boolean = true
-}
-
-case class LogicalCompareExpr(left: MathExpr, compare: String, right: MathExpr) extends LogicalExpr {
-  private val (eqOpr, neqOpr, btOpr, bteOpr, ltOpr, lteOpr) = ("""==?""".r, """!==?""".r, ">", ">=", "<", "<=")
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val (lv, rv) = (left.calculate(values), right.calculate(values))
-    compare match {
-      case this.eqOpr() => lv === rv
-      case this.neqOpr() => lv =!= rv
-      case this.btOpr => lv > rv
-      case this.bteOpr => lv >= rv
-      case this.ltOpr => lv < rv
-      case this.lteOpr => lv <= rv
-      case _ => None
-    }
-  }
-  val desc: String = s"${left.desc} ${compare} ${right.desc}"
-  val dataSources: Set[String] = left.dataSources ++ right.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    left.getCacheExprs(ds) ++ right.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    left.getFinalCacheExprs(ds) ++ right.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    left.getPersistExprs(ds) ++ right.getPersistExprs(ds)
-  }
-
-  override def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = {
-    if (compare == "=" || compare == "==") {
-      (left.dataSourceOpt, right.dataSourceOpt) match {
-        case (Some(dsPair._1), Some(dsPair._2)) => (left, right) :: Nil
-        case (Some(dsPair._2), Some(dsPair._1)) => (right, left) :: Nil
-        case _ => Nil
-      }
-    } else Nil
-  }
-}
-
-case class LogicalRangeExpr(left: MathExpr, rangeOpr: String, range: RangeDesc) extends LogicalExpr {
-  private val (inOpr, ninOpr, btwnOpr, nbtwnOpr) = ("""(?i)in""".r, """(?i)not\s+in""".r, """(?i)between""".r, """(?i)not\s+between""".r)
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val (lv, rvs) = (left.calculate(values), range.elements.map(_.calculate(values)))
-    rangeOpr match {
-      case this.inOpr() => lv in rvs
-      case this.ninOpr() => lv not_in rvs
-      case this.btwnOpr() => lv between rvs
-      case this.nbtwnOpr() => lv not_between rvs
-      case _ => None
-    }
-  }
-  val desc: String = s"${left.desc} ${rangeOpr} ${range.desc}"
-  val dataSources: Set[String] = left.dataSources ++ range.elements.flatMap(_.dataSources).toSet
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    left.getCacheExprs(ds) ++ range.elements.flatMap(_.getCacheExprs(ds))
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    left.getFinalCacheExprs(ds) ++ range.elements.flatMap(_.getFinalCacheExprs(ds))
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    left.getPersistExprs(ds) ++ range.elements.flatMap(_.getPersistExprs(ds))
-  }
-}
-
-// -- logical statement --
-//case class LogicalFactorExpr(self: LogicalExpr) extends LogicalExpr {
-//  def calculate(values: Map[String, Any]): Option[Any] = self.calculate(values)
-//  val desc: String = self.desc
-//}
-
-case class UnaryLogicalExpr(oprList: Iterable[String], factor: LogicalExpr) extends LogicalExpr {
-  private val notOpr = """(?i)not|!""".r
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val fv = factor.calculate(values)
-    oprList.foldRight(fv) { (opr, v) =>
-      opr match {
-        case this.notOpr() => !v
-        case _ => None
-      }
-    }
-  }
-  val desc: String = oprList.foldRight(factor.desc) { (prev, ex) => s"${prev}${ex}" }
-  val dataSources: Set[String] = factor.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    factor.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    factor.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    factor.getPersistExprs(ds)
-  }
-
-  override def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = {
-    val notOprList = oprList.filter { opr =>
-      opr match {
-        case this.notOpr() => true
-        case _ => false
-      }
-    }
-    if (notOprList.size % 2 == 0) factor.getGroupbyExprPairs(dsPair) else Nil
-  }
-}
-
-case class BinaryLogicalExpr(first: LogicalExpr, others: Iterable[(String, LogicalExpr)]) extends LogicalExpr {
-  private val (andOpr, orOpr) = ("""(?i)and|&&""".r, """(?i)or|\|\|""".r)
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val fv = first.calculate(values)
-    others.foldLeft(fv) { (v, pair) =>
-      val (opr, next) = pair
-      val nv = next.calculate(values)
-      opr match {
-        case this.andOpr() => v && nv
-        case this.orOpr() => v || nv
-        case _ => None
-      }
-    }
-  }
-  val desc: String = others.foldLeft(first.desc) { (ex, next) => s"${ex} ${next._1} ${next._2.desc}" }
-  val dataSources: Set[String] = first.dataSources ++ others.flatMap(_._2.dataSources).toSet
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    first.getCacheExprs(ds) ++ others.flatMap(_._2.getCacheExprs(ds))
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    first.getFinalCacheExprs(ds) ++ others.flatMap(_._2.getFinalCacheExprs(ds))
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    first.getPersistExprs(ds) ++ others.flatMap(_._2.getPersistExprs(ds))
-  }
-
-  override def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = {
-    if (others.isEmpty) first.getGroupbyExprPairs(dsPair)
-    else {
-      val isAnd = others.exists(_._1 match {
-        case this.andOpr() => true
-        case _ => false
-      })
-      if (isAnd) {
-        first.getGroupbyExprPairs(dsPair) ++ others.flatMap(_._2.getGroupbyExprPairs(dsPair))
-      } else Nil
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/MathExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/MathExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/MathExpr.scala
deleted file mode 100644
index db09a0c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/MathExpr.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-import org.apache.griffin.measure.batch.utils.CalculationUtil._
-
-trait MathExpr extends Expr {
-
-}
-
-case class MathFactorExpr(self: Expr) extends MathExpr {
-  def calculateOnly(values: Map[String, Any]): Option[Any] = self.calculate(values)
-  val desc: String = self.desc
-  val dataSources: Set[String] = self.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    self.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    self.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    self.getPersistExprs(ds)
-  }
-}
-
-case class UnaryMathExpr(oprList: Iterable[String], factor: Expr) extends MathExpr {
-  private val (posOpr, negOpr) = ("+", "-")
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val fv = factor.calculate(values)
-    oprList.foldRight(fv) { (opr, v) =>
-      opr match {
-        case this.posOpr => v
-        case this.negOpr => -v
-        case _ => None
-      }
-    }
-  }
-  val desc: String = oprList.foldRight(factor.desc) { (prev, ex) => s"${prev}${ex}" }
-  val dataSources: Set[String] = factor.dataSources
-  override def cacheUnit: Boolean = true
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    factor.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    factor.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    factor.getPersistExprs(ds)
-  }
-}
-
-case class BinaryMathExpr(first: MathExpr, others: Iterable[(String, MathExpr)]) extends MathExpr {
-  private val (addOpr, subOpr, mulOpr, divOpr, modOpr) = ("+", "-", "*", "/", "%")
-  def calculateOnly(values: Map[String, Any]): Option[Any] = {
-    val fv = first.calculate(values)
-    others.foldLeft(fv) { (v, pair) =>
-      val (opr, next) = pair
-      val nv = next.calculate(values)
-      opr match {
-        case this.addOpr => v + nv
-        case this.subOpr => v - nv
-        case this.mulOpr => v * nv
-        case this.divOpr => v / nv
-        case this.modOpr => v % nv
-        case _ => None
-      }
-    }
-  }
-  val desc: String = others.foldLeft(first.desc) { (ex, next) => s"${ex} ${next._1} ${next._2.desc}" }
-  val dataSources: Set[String] = first.dataSources ++ others.flatMap(_._2.dataSources).toSet
-  override def cacheUnit: Boolean = true
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    first.getCacheExprs(ds) ++ others.flatMap(_._2.getCacheExprs(ds))
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    first.getFinalCacheExprs(ds) ++ others.flatMap(_._2.getFinalCacheExprs(ds))
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    first.getPersistExprs(ds) ++ others.flatMap(_._2.getPersistExprs(ds))
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/SelectExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/SelectExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/SelectExpr.scala
deleted file mode 100644
index 52ebe21..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/SelectExpr.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-trait SelectExpr extends Expr {
-  def calculateOnly(values: Map[String, Any]): Option[Any] = None
-}
-
-case class IndexFieldRangeSelectExpr(fields: Iterable[FieldDescOnly]) extends SelectExpr {
-  val desc: String = s"[${fields.map(_.desc).mkString(",")}]"
-  val dataSources: Set[String] = Set.empty[String]
-}
-
-case class FunctionOperationExpr(func: String, args: Iterable[MathExpr]) extends SelectExpr {
-  val desc: String = s".${func}(${args.map(_.desc).mkString(",")})"
-  val dataSources: Set[String] = args.flatMap(_.dataSources).toSet
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = args.flatMap(_.getCacheExprs(ds))
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = args.flatMap(_.getFinalCacheExprs(ds))
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = args.flatMap(_.getPersistExprs(ds))
-}
-
-case class FilterSelectExpr(field: FieldDesc, compare: String, value: MathExpr) extends SelectExpr {
-  val desc: String = s"[${field.desc}${compare}${value.desc}]"
-  val dataSources: Set[String] = value.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = value.getCacheExprs(ds)
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = value.getFinalCacheExprs(ds)
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = value.getPersistExprs(ds)
-}
-
-// -- selection --
-case class SelectionExpr(head: SelectionHead, selectors: Iterable[SelectExpr]) extends Expr {
-  def calculateOnly(values: Map[String, Any]): Option[Any] = values.get(_id)
-
-  val desc: String = {
-    val argsString = selectors.map(_.desc).mkString("")
-    s"${head.desc}${argsString}"
-  }
-  val dataSources: Set[String] = {
-    val selectorDataSources = selectors.flatMap(_.dataSources).toSet
-    selectorDataSources + head.head
-  }
-
-  override def cacheUnit: Boolean = true
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    selectors.flatMap(_.getCacheExprs(ds))
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    selectors.flatMap(_.getFinalCacheExprs(ds))
-  }
-
-  override def persistUnit: Boolean = true
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    selectors.flatMap(_.getPersistExprs(ds))
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/StatementExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/StatementExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/StatementExpr.scala
deleted file mode 100644
index 3ae5139..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr/StatementExpr.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr
-
-
-trait StatementExpr extends Expr with AnalyzableExpr {
-  def valid(values: Map[String, Any]): Boolean = true
-  override def cacheUnit: Boolean = true
-}
-
-case class SimpleStatementExpr(expr: LogicalExpr) extends StatementExpr {
-  def calculateOnly(values: Map[String, Any]): Option[Any] = expr.calculate(values)
-  val desc: String = expr.desc
-  val dataSources: Set[String] = expr.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    expr.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    expr.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    expr.getPersistExprs(ds)
-  }
-
-  override def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = expr.getGroupbyExprPairs(dsPair)
-}
-
-case class WhenClauseStatementExpr(expr: LogicalExpr, whenExpr: LogicalExpr) extends StatementExpr {
-  def calculateOnly(values: Map[String, Any]): Option[Any] = expr.calculate(values)
-  val desc: String = s"${expr.desc} when ${whenExpr.desc}"
-
-  override def valid(values: Map[String, Any]): Boolean = {
-    whenExpr.calculate(values) match {
-      case Some(r: Boolean) => r
-      case _ => false
-    }
-  }
-
-  val dataSources: Set[String] = expr.dataSources ++ whenExpr.dataSources
-  override def getSubCacheExprs(ds: String): Iterable[Expr] = {
-    expr.getCacheExprs(ds) ++ whenExpr.getCacheExprs(ds)
-  }
-  override def getSubFinalCacheExprs(ds: String): Iterable[Expr] = {
-    expr.getFinalCacheExprs(ds) ++ whenExpr.getFinalCacheExprs(ds)
-  }
-  override def getSubPersistExprs(ds: String): Iterable[Expr] = {
-    expr.getPersistExprs(ds) ++ whenExpr.getPersistExprs(ds)
-  }
-
-  override def getGroupbyExprPairs(dsPair: (String, String)): Seq[(MathExpr, MathExpr)] = expr.getGroupbyExprPairs(dsPair)
-  override def getWhenClauseExpr(): Option[LogicalExpr] = Some(whenExpr)
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/AnnotationExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/AnnotationExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/AnnotationExpr.scala
deleted file mode 100644
index 2152bf9..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/AnnotationExpr.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-case class AnnotationExpr(expression: String) extends Expr with Calculatable {
-
-  val Key = """^(?i)Key$""".r
-
-  def isKey: Boolean = {
-    expression match {
-      case Key() => true
-      case _ => false
-    }
-  }
-
-  def genValue(values: Map[String, Any]): Option[String] = Some(expression)
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Calculatable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Calculatable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Calculatable.scala
deleted file mode 100644
index 9b8a577..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Calculatable.scala
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait Calculatable extends Serializable {
-
-  def genValue(values: Map[String, Any]): Option[Any]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ConstExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ConstExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ConstExpr.scala
deleted file mode 100644
index c8e623b..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ConstExpr.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-import scala.util.{Success, Try}
-
-trait ConstExpr extends Expr with Calculatable {
-
-  val value: Any
-
-}
-
-
-case class ConstStringExpr(expression: String) extends ConstExpr {
-
-  val value: String = expression
-  def genValue(values: Map[String, Any]): Option[String] = Some(value)
-
-}
-
-case class ConstTimeExpr(expression: String) extends ConstExpr {
-
-  val TimeRegex = """(\d+)(y|M|w|d|h|m|s|ms)""".r
-
-  val value: Long = expression match {
-    case TimeRegex(time, unit) => {
-      val t = time.toLong
-      val r = unit match {
-        case "y" => t * 365 * 30 * 24 * 60 * 60 * 1000
-        case "M" => t * 30 * 24 * 60 * 60 * 1000
-        case "w" => t * 7 * 24 * 60 * 60 * 1000
-        case "d" => t * 24 * 60 * 60 * 1000
-        case "h" => t * 60 * 60 * 1000
-        case "m" => t * 60 * 1000
-        case "s" => t * 1000
-        case "ms" => t
-        case _ => t
-      }
-      r
-    }
-    case _ => 0L
-  }
-
-  def genValue(values: Map[String, Any]): Option[Long] = Some(value)
-
-}
-
-case class ConstNumberExpr(expression: String) extends ConstExpr {
-
-  val value: Long = {
-    Try {
-      expression.toLong
-    } match {
-      case Success(v) => v
-      case _ => 0L
-    }
-  }
-
-  def genValue(values: Map[String, Any]): Option[Long] = Some(value)
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/DataExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/DataExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/DataExpr.scala
deleted file mode 100644
index e0f24ef..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/DataExpr.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait DataExpr extends Expr with Calculatable with Recordable {
-
-  def head: QuoteVariableExpr
-  def args: Iterable[SelectExpr]
-
-}
-
-
-case class SelectionExpr(head: QuoteVariableExpr, args: Iterable[SelectExpr]) extends {
-
-  val recordName = {
-    val argsString = args.map(_.recordName).mkString("")
-    s"${head.recordName}${argsString}"
-  }
-
-  override protected val _defaultId = recordName
-
-} with DataExpr {
-
-  val expression: String = ""
-
-  def genValue(values: Map[String, Any]): Option[Any] = values.get(_id)
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ElementExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ElementExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ElementExpr.scala
deleted file mode 100644
index 999c1ee..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ElementExpr.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-import org.apache.griffin.measure.batch.utils.CalculationUtil._
-
-trait ElementExpr extends Expr with ExprAnalyzable with Calculatable {
-
-}
-
-
-// self: const | selection | calculation
-case class FactorExpr(self: Expr with Calculatable) extends ElementExpr {
-
-  val expression: String = self.expression
-
-  def genValue(values: Map[String, Any]): Option[Any] = self.genValue(values)
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = {
-    self match {
-      case expr: DataExpr => if (expr.head.name == dataSign) expr :: Nil else Nil
-      case expr: CalculationExpr => expr.getDataRelatedExprs(dataSign)
-      case _ => Nil
-    }
-  }
-
-}
-
-case class CalculationExpr(first: ElementExpr, others: Iterable[(String, ElementExpr)]) extends ElementExpr {
-
-  val expression: String = others.foldLeft(first.expression) { (ex, next) => s"${ex} ${next._1} ${next._2}" }
-
-  def genValue(values: Map[String, Any]): Option[Any] = {
-    others.foldLeft(first.genValue(values)) { (v, next) =>
-      val (opr, ele) = next
-      val eleValue = ele.genValue(values)
-      opr match {
-        case "+" => v + eleValue
-        case "-" => v - eleValue
-        case "*" => v * eleValue
-        case "/" => v / eleValue
-        case "%" => v % eleValue
-        case _ => v
-      }
-    }
-  }
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = {
-    others.foldLeft(first.getDataRelatedExprs(dataSign)) { (origin, next) =>
-      origin ++ next._2.getDataRelatedExprs(dataSign)
-    }
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Expr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Expr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Expr.scala
deleted file mode 100644
index 17adf16..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Expr.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait Expr extends Serializable {
-
-  val expression: String
-
-  protected val _defaultId: String = ""
-
-  val _id = ExprIdCounter.genId(_defaultId)
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprAnalyzable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprAnalyzable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprAnalyzable.scala
deleted file mode 100644
index 29c47cd..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprAnalyzable.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-
-trait ExprAnalyzable extends Serializable {
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprIdCounter.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprIdCounter.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprIdCounter.scala
deleted file mode 100644
index c24f617..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/ExprIdCounter.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-import java.util.concurrent.atomic.AtomicLong
-import scala.collection.mutable.{Set => MutableSet}
-
-object ExprIdCounter {
-
-  private val idCounter: AtomicLong = new AtomicLong(0L)
-
-  private val existIdSet: MutableSet[String] = MutableSet.empty[String]
-
-  private val invalidIdRegex = """^\d+$""".r
-
-  def genId(defaultId: String): String = {
-    defaultId match {
-      case "" => increment.toString
-      case invalidIdRegex() => increment.toString
-//      case defId if (exist(defId)) => s"${increment}#${defId}"
-      case defId if (exist(defId)) => s"${defId}"
-      case _ => {
-        insertUserId(defaultId)
-        defaultId
-      }
-    }
-  }
-
-  private def exist(id: String): Boolean = {
-    existIdSet.contains(id)
-  }
-
-  private def insertUserId(id: String): Unit = {
-    existIdSet += id
-  }
-
-  private def increment(): Long = {
-    idCounter.incrementAndGet()
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Recordable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Recordable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Recordable.scala
deleted file mode 100644
index 27e5a48..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/Recordable.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-
-trait Recordable extends Serializable {
-
-  val recordName: String
-
-  protected def value2RecordString(v: Any): String = {
-    v match {
-      case s: String => s"'${s}'"
-      case a => s"${a}"
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/SelectExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/SelectExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/SelectExpr.scala
deleted file mode 100644
index 4dae38d..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/SelectExpr.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait SelectExpr extends Expr with Recordable {
-
-}
-
-
-case class NumPositionExpr(expression: String) extends SelectExpr {
-
-  val index: Int = expression.toInt
-
-  val recordName = s"[${value2RecordString(index)}]"
-
-}
-
-case class StringPositionExpr(expression: String) extends SelectExpr {
-
-  val field: String = expression
-
-  val recordName = s"[${value2RecordString(field)}]"
-
-}
-
-case class AnyPositionExpr(expression: String) extends SelectExpr {
-
-  val recordName = s"[${expression}]"
-
-}
-
-case class FilterOprExpr(expression: String, left: VariableExpr, right: ConstExpr) extends SelectExpr {
-
-  val field: String = left.name
-  val value: Any = right.value
-
-  val recordName = {
-    s"[${value2RecordString(field)}${expression}${value2RecordString(value)}]"
-  }
-
-}
-
-case class FunctionExpr(expression: String, args: Iterable[ConstExpr]) extends SelectExpr {
-
-  val recordName = {
-    val argsStr = args.map(value2RecordString(_)).mkString(",")
-    s".${expression}(${argsStr})"
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementAnalyzable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementAnalyzable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementAnalyzable.scala
deleted file mode 100644
index 57968e9..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementAnalyzable.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-
-trait StatementAnalyzable extends Serializable {
-
-  def getAssigns(): Iterable[AssignExpr] = Nil
-
-  def getConditions(): Iterable[ConditionExpr] = Nil
-
-  def getMappings(): Iterable[MappingExpr] = Nil
-
-  def getKeyMappings(): Iterable[MappingExpr] = Nil
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementExpr.scala
deleted file mode 100644
index 6669f83..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/StatementExpr.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait StatementExpr extends Expr with StatementAnalyzable with ExprAnalyzable with Calculatable {
-
-}
-
-case class AssignExpr(expression: String, left: VariableExpr, right: ElementExpr) extends StatementExpr {
-
-  def genValue(values: Map[String, Any]): Option[Any] = right.genValue(values)
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = right.getDataRelatedExprs(dataSign)
-
-  override def getAssigns(): Iterable[AssignExpr] = this :: Nil
-
-}
-
-case class ConditionExpr(expression: String, left: ElementExpr, right: ElementExpr, annotations: Iterable[AnnotationExpr]) extends StatementExpr {
-
-  def genValue(values: Map[String, Any]): Option[Boolean] = {
-    Some(true) // fixme: not done, need calculation
-  }
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = {
-    left.getDataRelatedExprs(dataSign) ++ right.getDataRelatedExprs(dataSign)
-  }
-
-  override def getConditions(): Iterable[ConditionExpr] = this :: Nil
-
-}
-
-case class MappingExpr(expression: String, left: ElementExpr, right: ElementExpr, annotations: Iterable[AnnotationExpr]) extends StatementExpr {
-
-  def isKey: Boolean = {
-    annotations.exists { e =>
-      e match {
-        case a: AnnotationExpr => a.isKey
-        case _ => false
-      }
-    }
-  }
-
-  def genValue(values: Map[String, Any]): Option[Boolean] = {
-    (left.genValue(values), right.genValue(values)) match {
-      case (Some(v1), Some(v2)) => Some(v1 == v2)
-      case (None, None) => Some(true)
-      case _ => Some(false)
-    }
-  }
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = {
-    left.getDataRelatedExprs(dataSign) ++ right.getDataRelatedExprs(dataSign)
-  }
-
-  override def getMappings(): Iterable[MappingExpr] = this :: Nil
-  override def getKeyMappings(): Iterable[MappingExpr] = if (isKey) this :: Nil else Nil
-
-}
-
-case class StatementsExpr(statements: Iterable[StatementExpr]) extends StatementExpr {
-
-  val expression: String = statements.map(_.expression).mkString("\n")
-
-  def genValue(values: Map[String, Any]): Option[Any] = None
-
-  def getDataRelatedExprs(dataSign: String): Iterable[DataExpr] = {
-    statements.flatMap(_.getDataRelatedExprs(dataSign))
-  }
-
-  override def getAssigns(): Iterable[AssignExpr] = {
-    statements.flatMap(_.getAssigns())
-  }
-
-  override def getConditions(): Iterable[ConditionExpr] = {
-    statements.flatMap(_.getConditions())
-  }
-
-  override def getMappings(): Iterable[MappingExpr] = {
-    statements.flatMap(_.getMappings())
-  }
-
-  override def getKeyMappings(): Iterable[MappingExpr] = {
-    statements.flatMap(_.getKeyMappings())
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/VariableExpr.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/VariableExpr.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/VariableExpr.scala
deleted file mode 100644
index 87424fc..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/expr_old/VariableExpr.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.apache.griffin.measure.batch.rule.expr_old
-
-trait VariableExpr extends Expr with Recordable {
-
-  val name: String
-
-}
-
-
-case class VariableStringExpr(expression: String) extends VariableExpr {
-
-  val name = expression
-
-  val recordName = name
-
-}
-
-case class QuoteVariableExpr(expression: String) extends VariableExpr with Calculatable {
-
-  val name = expression
-
-  val recordName = name
-
-  def genValue(values: Map[String, Any]): Option[Any] = values.get(name)
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/CalculationUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/CalculationUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/CalculationUtil.scala
deleted file mode 100644
index 2d10fb4..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/CalculationUtil.scala
+++ /dev/null
@@ -1,259 +0,0 @@
-package org.apache.griffin.measure.batch.utils
-
-import scala.util.{Success, Try}
-
-
-object CalculationUtil {
-
-  implicit def option2CalculationValue(v: Option[_]): CalculationValue = CalculationValue(v)
-
-  case class CalculationValue(value: Option[_]) extends Serializable {
-
-    def + (other: Option[_]): Option[_] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: String), Some(v2)) => Some(v1 + v2.toString)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 + v2.toString.toByte)
-          case (Some(v1: Short), Some(v2)) => Some(v1 + v2.toString.toShort)
-          case (Some(v1: Int), Some(v2)) => Some(v1 + v2.toString.toInt)
-          case (Some(v1: Long), Some(v2)) => Some(v1 + v2.toString.toLong)
-          case (Some(v1: Float), Some(v2)) => Some(v1 + v2.toString.toFloat)
-          case (Some(v1: Double), Some(v2)) => Some(v1 + v2.toString.toDouble)
-          case (None, Some(v2)) => other
-          case _ => value
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => value
-      }
-    }
-
-    def - (other: Option[_]): Option[_] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: Byte), Some(v2)) => Some(v1 - v2.toString.toByte)
-          case (Some(v1: Short), Some(v2)) => Some(v1 - v2.toString.toShort)
-          case (Some(v1: Int), Some(v2)) => Some(v1 - v2.toString.toInt)
-          case (Some(v1: Long), Some(v2)) => Some(v1 - v2.toString.toLong)
-          case (Some(v1: Float), Some(v2)) => Some(v1 - v2.toString.toFloat)
-          case (Some(v1: Double), Some(v2)) => Some(v1 - v2.toString.toDouble)
-          case _ => value
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => value
-      }
-    }
-
-    def * (other: Option[_]): Option[_] = {
-      Try {
-        (value, other) match {
-          case (Some(s1: String), Some(n2: Int)) => Some(s1 * n2)
-          case (Some(s1: String), Some(n2: Long)) => Some(s1 * n2.toInt)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 * v2.toString.toByte)
-          case (Some(v1: Short), Some(v2)) => Some(v1 * v2.toString.toShort)
-          case (Some(v1: Int), Some(v2)) => Some(v1 * v2.toString.toInt)
-          case (Some(v1: Long), Some(v2)) => Some(v1 * v2.toString.toLong)
-          case (Some(v1: Float), Some(v2)) => Some(v1 * v2.toString.toFloat)
-          case (Some(v1: Double), Some(v2)) => Some(v1 * v2.toString.toDouble)
-          case _ => value
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => value
-      }
-    }
-
-    def / (other: Option[_]): Option[_] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: Byte), Some(v2)) => Some(v1 / v2.toString.toByte)
-          case (Some(v1: Short), Some(v2)) => Some(v1 / v2.toString.toShort)
-          case (Some(v1: Int), Some(v2)) => Some(v1 / v2.toString.toInt)
-          case (Some(v1: Long), Some(v2)) => Some(v1 / v2.toString.toLong)
-          case (Some(v1: Float), Some(v2)) => Some(v1 / v2.toString.toFloat)
-          case (Some(v1: Double), Some(v2)) => Some(v1 / v2.toString.toDouble)
-          case _ => value
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => value
-      }
-    }
-
-    def % (other: Option[_]): Option[_] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: Byte), Some(v2)) => Some(v1 % v2.toString.toByte)
-          case (Some(v1: Short), Some(v2)) => Some(v1 % v2.toString.toShort)
-          case (Some(v1: Int), Some(v2)) => Some(v1 % v2.toString.toInt)
-          case (Some(v1: Long), Some(v2)) => Some(v1 % v2.toString.toLong)
-          case (Some(v1: Float), Some(v2)) => Some(v1 % v2.toString.toFloat)
-          case (Some(v1: Double), Some(v2)) => Some(v1 % v2.toString.toDouble)
-          case _ => value
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => value
-      }
-    }
-
-    def unary_- (): Option[_] = {
-      value match {
-        case Some(v: String) => Some(v.reverse.toString)
-        case Some(v: Boolean) => Some(!v)
-        case Some(v: Byte) => Some(-v)
-        case Some(v: Short) => Some(-v)
-        case Some(v: Int) => Some(-v)
-        case Some(v: Long) => Some(-v)
-        case Some(v: Float) => Some(-v)
-        case Some(v: Double) => Some(-v)
-        case Some(v) => Some(v)
-        case _ => None
-      }
-    }
-
-
-    def === (other: Option[_]): Option[Boolean] = {
-      (value, other) match {
-        case (Some(v1), Some(v2)) => Some(v1 == v2)
-        case _ => Some(false)
-      }
-    }
-
-    def =!= (other: Option[_]): Option[Boolean] = {
-      (value, other) match {
-        case (Some(v1), Some(v2)) => Some(v1 != v2)
-        case _ => Some(true)
-      }
-    }
-
-    def > (other: Option[_]): Option[Boolean] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: String), Some(v2: String)) => Some(v1 > v2)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case (Some(v1: Short), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case (Some(v1: Int), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case (Some(v1: Long), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case (Some(v1: Float), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case (Some(v1: Double), Some(v2)) => Some(v1 > v2.toString.toDouble)
-          case _ => Some(false)
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => None
-      }
-    }
-
-    def >= (other: Option[_]): Option[Boolean] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: String), Some(v2: String)) => Some(v1 >= v2)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case (Some(v1: Short), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case (Some(v1: Int), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case (Some(v1: Long), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case (Some(v1: Float), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case (Some(v1: Double), Some(v2)) => Some(v1 >= v2.toString.toDouble)
-          case _ => Some(false)
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => None
-      }
-    }
-
-    def < (other: Option[_]): Option[Boolean] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: String), Some(v2: String)) => Some(v1 < v2)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case (Some(v1: Short), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case (Some(v1: Int), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case (Some(v1: Long), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case (Some(v1: Float), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case (Some(v1: Double), Some(v2)) => Some(v1 < v2.toString.toDouble)
-          case _ => Some(false)
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => None
-      }
-    }
-
-    def <= (other: Option[_]): Option[Boolean] = {
-      Try {
-        (value, other) match {
-          case (Some(v1: String), Some(v2: String)) => Some(v1 <= v2)
-          case (Some(v1: Byte), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case (Some(v1: Short), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case (Some(v1: Int), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case (Some(v1: Long), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case (Some(v1: Float), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case (Some(v1: Double), Some(v2)) => Some(v1 <= v2.toString.toDouble)
-          case _ => Some(false)
-        }
-      } match {
-        case Success(opt) => opt
-        case _ => None
-      }
-    }
-
-
-    def in (other: Iterable[Option[_]]): Option[Boolean] = {
-      Some(other.foldLeft(false) { (res, next) =>
-        res || ===(next).getOrElse(false)
-      })
-    }
-
-    def not_in (other: Iterable[Option[_]]): Option[Boolean] = {
-      Some(other.foldLeft(true) { (res, next) =>
-        res && =!=(next).getOrElse(false)
-      })
-    }
-
-    def between (other: Iterable[Option[_]]): Option[Boolean] = {
-      if (other.size < 2) None else {
-        val (begin, end) = (other.head, other.tail.head)
-        (>=(begin), <=(end)) match {
-          case (Some(b1), Some(b2)) => Some(b1 && b2)
-          case _ => None
-        }
-      }
-    }
-
-    def not_between (other: Iterable[Option[_]]): Option[Boolean] = {
-      if (other.size < 2) None else {
-        val (begin, end) = (other.head, other.tail.head)
-        (<(begin), >(end)) match {
-          case (Some(b1), Some(b2)) => Some(b1 || b2)
-          case _ => None
-        }
-      }
-    }
-
-    def unary_! (): Option[Boolean] = {
-      value match {
-        case Some(v: Boolean) => Some(!v)
-        case Some(v) => Some(false)
-        case _ => None
-      }
-    }
-
-    def && (other: Option[_]): Option[Boolean] = {
-      (value, other) match {
-        case (Some(b1: Boolean), Some(b2: Boolean)) => Some(b1 && b2)
-        case _ => None
-      }
-    }
-
-    def || (other: Option[_]): Option[Boolean] = {
-      (value, other) match {
-        case (Some(b1: Boolean), Some(b2: Boolean)) => Some(b1 || b2)
-        case _ => None
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HdfsUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HdfsUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HdfsUtil.scala
deleted file mode 100644
index b48478a..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HdfsUtil.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.griffin.measure.batch.utils
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FSDataInputStream, FSDataOutputStream, FileSystem, Path}
-
-object HdfsUtil {
-
-  private val seprator = "/"
-
-  private val conf = new Configuration()
-  conf.set("dfs.support.append", "true")
-
-  private val dfs = FileSystem.get(conf)
-
-  def existPath(filePath: String): Boolean = {
-    val path = new Path(filePath)
-    dfs.exists(path)
-  }
-
-  def createFile(filePath: String): FSDataOutputStream = {
-    val path = new Path(filePath)
-    if (dfs.exists(path)) dfs.delete(path, true)
-    return dfs.create(path)
-  }
-
-  def appendOrCreateFile(filePath: String): FSDataOutputStream = {
-    val path = new Path(filePath)
-    if (dfs.exists(path)) dfs.append(path) else createFile(filePath)
-  }
-
-  def openFile(filePath: String): FSDataInputStream = {
-    val path = new Path(filePath)
-    dfs.open(path)
-  }
-
-  def writeContent(filePath: String, message: String): Unit = {
-    val out = createFile(filePath)
-    out.write(message.getBytes("utf-8"))
-    out.close
-  }
-
-  def appendContent(filePath: String, message: String): Unit = {
-    val out = appendOrCreateFile(filePath)
-    out.write(message.getBytes("utf-8"))
-    out.close
-  }
-
-  def createEmptyFile(filePath: String): Unit = {
-    val out = createFile(filePath)
-    out.close
-  }
-
-
-  def getHdfsFilePath(parentPath: String, fileName: String): String = {
-    if (parentPath.endsWith(seprator)) parentPath + fileName else parentPath + seprator + fileName
-  }
-
-  def deleteHdfsPath(dirPath: String): Unit = {
-    val path = new Path(dirPath)
-    if (dfs.exists(path)) dfs.delete(path, true)
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HttpUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HttpUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HttpUtil.scala
deleted file mode 100644
index 747d0fa..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/HttpUtil.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.apache.griffin.measure.batch.utils
-
-import scalaj.http._
-
-object HttpUtil {
-
-  val GET_REGEX = """^(?i)get$""".r
-  val POST_REGEX = """^(?i)post$""".r
-  val PUT_REGEX = """^(?i)put$""".r
-  val DELETE_REGEX = """^(?i)delete$""".r
-
-  def postData(url: String, params: Map[String, Object], headers: Map[String, Object], data: String): String = {
-    val response = Http(url).params(convertObjMap2StrMap(params)).headers(convertObjMap2StrMap(headers)).postData(data).asString
-    response.code.toString
-  }
-
-  def httpRequest(url: String, method: String, params: Map[String, Object], headers: Map[String, Object], data: String): String = {
-    val httpReq = Http(url).params(convertObjMap2StrMap(params)).headers(convertObjMap2StrMap(headers))
-    method match {
-      case POST_REGEX() => httpReq.postData(data).asString.code.toString
-      case PUT_REGEX() => httpReq.put(data).asString.code.toString
-      case _ => "wrong method"
-    }
-  }
-
-  private def convertObjMap2StrMap(map: Map[String, Object]): Map[String, String] = {
-    map.map(pair => pair._1 -> pair._2.toString)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/JsonUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/JsonUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/JsonUtil.scala
deleted file mode 100644
index cdd470a..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/JsonUtil.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.griffin.measure.batch.utils
-
-import java.io.InputStream
-
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-
-import scala.reflect._
-
-object JsonUtil {
-  val mapper = new ObjectMapper()
-  mapper.registerModule(DefaultScalaModule)
-  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-
-  def toJson(value: Map[Symbol, Any]): String = {
-    toJson(value map { case (k,v) => k.name -> v})
-  }
-
-  def toJson(value: Any): String = {
-    mapper.writeValueAsString(value)
-  }
-
-  def toMap[V](json:String)(implicit m: Manifest[V]) = fromJson[Map[String,V]](json)
-
-  def fromJson[T: ClassTag](json: String)(implicit m : Manifest[T]): T = {
-    mapper.readValue[T](json, classTag[T].runtimeClass.asInstanceOf[Class[T]])
-  }
-
-  def fromJson[T: ClassTag](is: InputStream)(implicit m : Manifest[T]): T = {
-    mapper.readValue[T](is, classTag[T].runtimeClass.asInstanceOf[Class[T]])
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/StringParseUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/StringParseUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/StringParseUtil.scala
deleted file mode 100644
index 7f2b355..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/utils/StringParseUtil.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.measure.batch.utils
-
-object StringParseUtil {
-
-  def sepStrings(str: String, sep: String): Iterable[String] = {
-    val strings = str.split(sep)
-    strings.map(_.trim)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/test/resources/config.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/test/resources/config.json b/griffin-measure/griffin-measure-batch/src/test/resources/config.json
deleted file mode 100644
index 65e0ed9..0000000
--- a/griffin-measure/griffin-measure-batch/src/test/resources/config.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "name": "accu1",
-  "type": "accuracy",
-
-  "source": {
-    "type": "avro",
-    "version": "1.7",
-    "config": {
-      "file.name": "src/test/resources/users_info_src.avro"
-    }
-  },
-
-  "target": {
-    "type": "avro",
-    "version": "1.7",
-    "config": {
-      "file.name": "src/test/resources/users_info_target.avro"
-    }
-  },
-
-  "evaluateRule": {
-    "sampleRatio": 1,
-    "rules": "$source.user_id > 10020 AND $source.user_id + 5 = $target.user_id + (2 + 3) AND $source.first_name + 12 = $target.first_name + (10 + 2) AND $source.last_name = $target.last_name AND $source.address = $target.address AND $source.email = $target.email AND $source.phone = $target.phone AND $source.post_code = $target.post_code"
-  }
-}
\ No newline at end of file



[3/3] incubator-griffin git commit: clean files

Posted by gu...@apache.org.
clean files

Author: Liu <ll...@ebay.com>
Author: Liu <ll...@lm-shc-16501428.corp.ebay.com>
Author: Liu <ll...@lm-shc-16501428.dhcp>
Author: bhlx3lyx7 <bh...@163.com>

Closes #36 from bhlx3lyx7/master.


Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/c4340cea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/c4340cea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/c4340cea

Branch: refs/heads/master
Commit: c4340cea10cb0acfb16dbc10f699ec8aeaf10c1e
Parents: a011781
Author: Liu <ll...@ebay.com>
Authored: Tue May 23 11:35:51 2017 +0800
Committer: Liu <ll...@lm-shc-16501428.corp.ebay.com>
Committed: Tue May 23 11:35:51 2017 +0800

----------------------------------------------------------------------
 griffin-measure/griffin-measure-batch/pom.xml   |  16 -
 .../src/main/resources/config-old.json          |  45 ---
 .../src/main/resources/config.json              |  29 --
 .../src/main/resources/env.json                 |  29 --
 .../src/main/resources/log4j.properties         |   5 -
 .../griffin/measure/batch/Application.scala     |  87 ------
 .../measure/batch/algo/AccuracyAlgo.scala       |   6 -
 .../griffin/measure/batch/algo/Algo.scala       |  16 -
 .../measure/batch/algo/BatchAccuracyAlgo.scala  | 144 ---------
 .../measure/batch/algo/core/AccuracyCore.scala  |  78 -----
 .../measure/batch/config/params/AllParam.scala  |  14 -
 .../measure/batch/config/params/Param.scala     |   7 -
 .../batch/config/params/env/CleanerParam.scala  |  10 -
 .../batch/config/params/env/EnvParam.scala      |  13 -
 .../batch/config/params/env/PersistParam.scala  |  12 -
 .../batch/config/params/env/SparkParam.scala    |  13 -
 .../config/params/user/DataConnectorParam.scala |  13 -
 .../config/params/user/EvaluateRuleParam.scala  |  12 -
 .../batch/config/params/user/UserParam.scala    |  15 -
 .../batch/config/reader/ParamFileReader.scala   |  20 --
 .../config/reader/ParamHdfsFileReader.scala     |  20 --
 .../config/reader/ParamRawStringReader.scala    |  17 --
 .../batch/config/reader/ParamReader.scala       |  12 -
 .../config/reader/ParamReaderFactory.scala      |  22 --
 .../config/validator/AllParamValidator.scala    |  16 -
 .../batch/config/validator/ParamValidator.scala |  12 -
 .../batch/connector/AvroDataConnector.scala     |  79 -----
 .../measure/batch/connector/CacheDataUtil.scala |  63 ----
 .../measure/batch/connector/DataConnector.scala |  16 -
 .../batch/connector/DataConnectorFactory.scala  |  32 --
 .../batch/connector/HiveDataConnector.scala     | 101 -------
 .../griffin/measure/batch/log/Loggable.scala    |  25 --
 .../measure/batch/persist/HdfsPersist.scala     | 119 --------
 .../measure/batch/persist/HttpPersist.scala     |  44 ---
 .../measure/batch/persist/MultiPersists.scala   |  27 --
 .../griffin/measure/batch/persist/Persist.scala |  23 --
 .../measure/batch/persist/PersistFactory.scala  |  30 --
 .../measure/batch/result/AccuracyResult.scala   |  26 --
 .../griffin/measure/batch/result/Result.scala   |  14 -
 .../measure/batch/result/ResultInfo.scala       |  33 --
 .../measure/batch/rule/RuleAnalyzer.scala       |  28 --
 .../measure/batch/rule/RuleAnalyzerOld.scala    |  26 --
 .../measure/batch/rule/RuleFactory.scala        |  34 ---
 .../griffin/measure/batch/rule/RuleParser.scala | 298 -------------------
 .../batch/rule/RuleParserDescription.scala      |  84 ------
 .../batch/rule/expr/AnalyzableExpr.scala        |   7 -
 .../measure/batch/rule/expr/Cacheable.scala     |  15 -
 .../measure/batch/rule/expr/Calculatable.scala  |   7 -
 .../batch/rule/expr/DataSourceable.scala        |  10 -
 .../measure/batch/rule/expr/Describable.scala   |  15 -
 .../griffin/measure/batch/rule/expr/Expr.scala  |  33 --
 .../measure/batch/rule/expr/ExprDescOnly.scala  |  22 --
 .../measure/batch/rule/expr/ExprIdCounter.scala |  42 ---
 .../measure/batch/rule/expr/FieldDescOnly.scala |  40 ---
 .../measure/batch/rule/expr/LiteralExpr.scala   |  68 -----
 .../measure/batch/rule/expr/LogicalExpr.scala   | 149 ----------
 .../measure/batch/rule/expr/MathExpr.scala      |  79 -----
 .../measure/batch/rule/expr/SelectExpr.scala    |  53 ----
 .../measure/batch/rule/expr/StatementExpr.scala |  50 ----
 .../batch/rule/expr_old/AnnotationExpr.scala    |  16 -
 .../batch/rule/expr_old/Calculatable.scala      |   7 -
 .../measure/batch/rule/expr_old/ConstExpr.scala |  59 ----
 .../measure/batch/rule/expr_old/DataExpr.scala  |  27 --
 .../batch/rule/expr_old/ElementExpr.scala       |  52 ----
 .../measure/batch/rule/expr_old/Expr.scala      |  11 -
 .../batch/rule/expr_old/ExprAnalyzable.scala    |   8 -
 .../batch/rule/expr_old/ExprIdCounter.scala     |  39 ---
 .../batch/rule/expr_old/Recordable.scala        |  15 -
 .../batch/rule/expr_old/SelectExpr.scala        |  48 ---
 .../rule/expr_old/StatementAnalyzable.scala     |  14 -
 .../batch/rule/expr_old/StatementExpr.scala     |  85 ------
 .../batch/rule/expr_old/VariableExpr.scala      |  26 --
 .../measure/batch/utils/CalculationUtil.scala   | 259 ----------------
 .../griffin/measure/batch/utils/HdfsUtil.scala  |  62 ----
 .../griffin/measure/batch/utils/HttpUtil.scala  |  30 --
 .../griffin/measure/batch/utils/JsonUtil.scala  |  32 --
 .../measure/batch/utils/StringParseUtil.scala   |  10 -
 .../src/test/resources/config.json              |  25 --
 .../src/test/resources/config1.json             |  27 --
 .../src/test/resources/env.json                 |  27 --
 .../src/test/resources/env1.json                |  21 --
 .../src/test/resources/log4j.properties         |   5 -
 .../src/test/resources/users_info_src.avro      | Bin 3850 -> 0 bytes
 .../src/test/resources/users_info_src.dat       |  50 ----
 .../src/test/resources/users_info_target.avro   | Bin 3852 -> 0 bytes
 .../src/test/resources/users_info_target.dat    |  50 ----
 .../batch/algo/BatchAccuracyAlgoTest.scala      | 219 --------------
 .../config/reader/ParamFileReaderTest.scala     |  35 ---
 .../measure/batch/rule/RuleParserTest.scala     |  77 -----
 griffin-measure/pom.xml                         | 197 ------------
 .../data/test/dataFile/users_info_src.avro      | Bin 3799 -> 0 bytes
 .../data/test/dataFile/users_info_target.avro   | Bin 3799 -> 0 bytes
 .../data/test/recordFile/_RESULT_ACCU           |  12 -
 .../data/test/reqJson/accuUsersInfo.json        |  55 ----
 .../src/test/scala/modelTest/AccuTest.scala     |  86 ------
 95 files changed, 4061 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/pom.xml
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/pom.xml b/griffin-measure/griffin-measure-batch/pom.xml
deleted file mode 100644
index 6fd72ed..0000000
--- a/griffin-measure/griffin-measure-batch/pom.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>griffin-measure</artifactId>
-        <groupId>org.apache.griffin</groupId>
-        <version>0.0.1-SNAPSHOT</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>griffin-measure-batch</artifactId>
-    <version>0.0.1-SNAPSHOT</version>
-
-
-</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/resources/config-old.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/resources/config-old.json b/griffin-measure/griffin-measure-batch/src/main/resources/config-old.json
deleted file mode 100644
index 63dee69..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/resources/config-old.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
-  "name": "accu1",
-  "type": "accuracy",
-
-  "source": {
-    "connector": {
-      "type": "hive",
-      "version": "1.2",
-      "config": {
-        "table.name": "users_info_src",
-        "partitions": "dt=20170410, hour=14"
-      }
-    }
-  },
-
-  "target": {
-    "connector": {
-      "type": "hive",
-      "version": "1.2",
-      "config": {
-        "database": "default",
-        "table.name": "users_info_target",
-        "partitions": "dt=20170410, hour=14; dt=20170410, hour=15"
-      }
-    }
-  },
-
-  "evaluateRule": {
-    "sampleRatio": 1,
-    "assertion": {
-      "type": "DSL-griffin",
-      "rules": [
-        {
-          "rule": "@Key ${source}['user_id'] === ${target}['user_id']"
-        },
-        {
-          "rule": "${source}['first_name'] === ${target}['first_name']; ${source}['last_name'] === ${target}['last_name']; ${source}['address'] === ${target}['address']"
-        },
-        {
-          "rule": "${source}['email'] === ${target}['email']; ${source}['phone'] === ${target}['phone']; ${source}['post_code'] === ${target}['post_code']"
-        }
-      ]
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/resources/config.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/resources/config.json b/griffin-measure/griffin-measure-batch/src/main/resources/config.json
deleted file mode 100644
index edd2e6a..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/resources/config.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "name": "accu1",
-  "type": "accuracy",
-
-  "source": {
-    "type": "hive",
-    "version": "1.2",
-    "config": {
-      "database": "default",
-      "table.name": "users_info_src",
-      "partitions": "dt=23123, hour=432; dt=35464, hour=4657"
-    }
-  },
-
-  "target": {
-    "type": "hive",
-    "version": "1.2",
-    "config": {
-      "database": "default",
-      "table.name": "users_info_target",
-      "partitions": "dt=23123, hour=432; dt=35464, hour=4657"
-    }
-  },
-
-  "evaluateRule": {
-    "sampleRatio": 0.2,
-    "rules": "$source.user_id = $target.user_id AND $source.first_name = $target.first_name AND $source.last_name = $target.last_name AND $source.address = $target.address AND $source.email = $target.email AND $source.phone = $target.phone AND $source.post_code = $target.post_code"
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/resources/env.json
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/resources/env.json b/griffin-measure/griffin-measure-batch/src/main/resources/env.json
deleted file mode 100644
index 2e72601..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/resources/env.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "spark": {
-    "log.level": "INFO",
-    "checkpoint.dir": "hdfs:///griffin/batch/cp",
-    "config": {}
-  },
-
-  "persist": [
-    {
-      "type": "hdfs",
-      "config": {
-        "path": "hdfs:///griffin/streaming/persist",
-        "max.persist.lines": 10000,
-        "max.lines.per.file": 10000
-      }
-    },
-    {
-      "type": "http",
-      "config": {
-        "method": "post",
-        "api": "http://phxbark4dq-360935.stratus.phx.ebay.com:8080/"
-      }
-    }
-  ],
-
-  "cleaner": {
-
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/resources/log4j.properties b/griffin-measure/griffin-measure-batch/src/main/resources/log4j.properties
deleted file mode 100644
index bd31e15..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-log4j.rootLogger=INFO, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.Target=System.out
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%c] - %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala
deleted file mode 100644
index 6fe6bcb..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/Application.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-package org.apache.griffin.measure.batch
-
-import org.apache.griffin.measure.batch.algo.{Algo, BatchAccuracyAlgo}
-import org.apache.griffin.measure.batch.config.params._
-import org.apache.griffin.measure.batch.config.params.env._
-import org.apache.griffin.measure.batch.config.params.user._
-import org.apache.griffin.measure.batch.config.reader._
-import org.apache.griffin.measure.batch.config.validator.AllParamValidator
-import org.apache.griffin.measure.batch.log.Loggable
-
-import scala.util.{Failure, Success, Try}
-
-object Application extends Loggable {
-
-  def main(args: Array[String]): Unit = {
-    info(args.toString)
-    if (args.length < 2) {
-      error("Usage: class <env-param> <user-param> [List of String split by comma: raw | local | hdfs(default)]")
-      sys.exit(-1)
-    }
-
-    val envParamFile = args(0)
-    val userParamFile = args(1)
-    val (envFsType, userFsType) = if (args.length > 2) {
-      val fsTypes = args(2).trim.split(",")
-      if (fsTypes.length == 1) (fsTypes(0).trim, fsTypes(0).trim)
-      else if (fsTypes.length >= 2) (fsTypes(0).trim, fsTypes(1).trim)
-      else ("hdfs", "hdfs")
-    } else ("hdfs", "hdfs")
-
-    info(envParamFile)
-    info(userParamFile)
-
-    // read param files
-    val envParam = readParamFile[EnvParam](envParamFile, envFsType) match {
-      case Success(p) => p
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-2)
-      }
-    }
-    val userParam = readParamFile[UserParam](userParamFile, userFsType) match {
-      case Success(p) => p
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-2)
-      }
-    }
-    val allParam: AllParam = AllParam(envParam, userParam)
-
-    // validate param files
-    validateParams(allParam) match {
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-3)
-      }
-      case _ => {
-        info("params validation pass")
-      }
-    }
-
-    // choose algorithm   // fixme: not done, need to choose algorithm by param
-    val dqType = allParam.userParam.dqType
-    val algo: Algo = BatchAccuracyAlgo(allParam)
-
-    algo.run match {
-      case Failure(ex) => {
-        error(ex.getMessage)
-        sys.exit(-4)
-      }
-      case _ => {
-        info("calculation finished")
-      }
-    }
-  }
-
-  private def readParamFile[T <: Param](file: String, fsType: String)(implicit m : Manifest[T]): Try[T] = {
-    val paramReader = ParamReaderFactory.getParamReader(file, fsType)
-    paramReader.readConfig[T]
-  }
-
-  private def validateParams(allParam: AllParam): Try[Boolean] = {
-    val allParamValidator = AllParamValidator()
-    allParamValidator.validate(allParam)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala
deleted file mode 100644
index d22add2..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/AccuracyAlgo.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.apache.griffin.measure.batch.algo
-
-
-trait AccuracyAlgo extends Algo {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala
deleted file mode 100644
index d7047ee..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/Algo.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.apache.griffin.measure.batch.algo
-
-import org.apache.griffin.measure.batch.config.params.env._
-import org.apache.griffin.measure.batch.config.params.user._
-import org.apache.griffin.measure.batch.log.Loggable
-
-import scala.util.Try
-
-trait Algo extends Loggable with Serializable {
-
-  val envParam: EnvParam
-  val userParam: UserParam
-
-  def run(): Try[_]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala
deleted file mode 100644
index 24564a4..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/BatchAccuracyAlgo.scala
+++ /dev/null
@@ -1,144 +0,0 @@
-package org.apache.griffin.measure.batch.algo
-
-import java.util.Date
-
-import org.apache.griffin.measure.batch.algo.core.AccuracyCore
-import org.apache.griffin.measure.batch.config.params.AllParam
-import org.apache.griffin.measure.batch.connector._
-import org.apache.griffin.measure.batch.rule._
-import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.griffin.measure.batch.persist._
-import org.apache.griffin.measure.batch.result._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.{SparkConf, SparkContext}
-
-import scala.util.{Failure, Success, Try}
-
-
-case class BatchAccuracyAlgo(allParam: AllParam) extends AccuracyAlgo {
-  val envParam = allParam.envParam
-  val userParam = allParam.userParam
-
-  def run(): Try[_] = {
-    Try {
-      val metricName = userParam.name
-
-      val conf = new SparkConf().setAppName(metricName)
-      val sc = new SparkContext(conf)
-      val sqlContext = new HiveContext(sc)
-
-      // start time
-      val startTime = new Date().getTime()
-
-      // get persists
-      val persist: Persist = PersistFactory(envParam.persistParams, metricName).getPersists(startTime)
-
-      // get spark application id
-      val applicationId = sc.applicationId
-
-      // start
-      persist.start(applicationId)
-
-      // rules
-      val ruleFactory = RuleFactory(userParam.evaluateRuleParam)
-      val rule: StatementExpr = ruleFactory.generateRule()
-      val ruleAnalyzer: RuleAnalyzer = RuleAnalyzer(rule)
-
-      // global cache data
-      val globalCachedData = CacheDataUtil.genCachedMap(None, ruleAnalyzer.globalCacheExprs, Map[String, Any]())
-      val globalFinalCachedData = CacheDataUtil.filterCachedMap(ruleAnalyzer.globalFinalCacheExprs, globalCachedData)
-
-      // data connector
-      val sourceDataConnector: DataConnector =
-        DataConnectorFactory.getDataConnector(sqlContext, userParam.sourceParam,
-          ruleAnalyzer.sourceGroupbyExprs, ruleAnalyzer.sourceCacheExprs,
-          ruleAnalyzer.sourceFinalCacheExprs, globalFinalCachedData) match {
-          case Success(cntr) => {
-            if (cntr.available) cntr
-            else throw new Exception("source data connection error!")
-          }
-          case Failure(ex) => throw ex
-        }
-      val targetDataConnector: DataConnector =
-        DataConnectorFactory.getDataConnector(sqlContext, userParam.targetParam,
-          ruleAnalyzer.targetGroupbyExprs, ruleAnalyzer.targetCacheExprs,
-          ruleAnalyzer.targetFinalCacheExprs, globalFinalCachedData) match {
-          case Success(cntr) => {
-            if (cntr.available) cntr
-            else throw new Exception("target data connection error!")
-          }
-          case Failure(ex) => throw ex
-        }
-
-      // get metadata
-//      val sourceMetaData: Iterable[(String, String)] = sourceDataConnector.metaData() match {
-//        case Success(md) => md
-//        case _ => throw new Exception("source metadata error!")
-//      }
-//      val targetMetaData: Iterable[(String, String)] = targetDataConnector.metaData() match {
-//        case Success(md) => md
-//        case _ => throw new Exception("target metadata error!")
-//      }
-
-      // get data
-      val sourceData: RDD[(Product, Map[String, Any])] = sourceDataConnector.data() match {
-        case Success(dt) => dt
-        case Failure(ex) => throw ex
-      }
-      val targetData: RDD[(Product, Map[String, Any])] = targetDataConnector.data() match {
-        case Success(dt) => dt
-        case Failure(ex) => throw ex
-      }
-
-      // accuracy algorithm
-      val (accuResult, missingRdd, matchingRdd) = accuracy(sourceData, targetData, ruleAnalyzer)
-
-      // end time
-      val endTime = new Date().getTime
-      persist.log(endTime, s"calculation using time: ${endTime - startTime} ms")
-
-      // persist result
-      persist.result(endTime, accuResult)
-      val missingRecords = missingRdd.map(record2String(_))
-      persist.missRecords(missingRecords)
-
-      // persist end time
-      val persistEndTime = new Date().getTime
-      persist.log(persistEndTime, s"persist using time: ${persistEndTime - endTime} ms")
-
-      // finish
-      persist.finish()
-
-      // context stop
-      sc.stop
-
-    }
-  }
-
-  def wrapInitData(data: Map[String, Any]): (Map[String, Any], Map[String, Any]) = {
-    (data, Map[String, Any]())
-  }
-
-  def accuracy(sourceData: RDD[(Product, Map[String, Any])], targetData: RDD[(Product, Map[String, Any])], ruleAnalyzer: RuleAnalyzer
-              ): (AccuracyResult, RDD[(Product, (Map[String, Any], Map[String, Any]))], RDD[(Product, (Map[String, Any], Map[String, Any]))]) = {
-
-    // 1. wrap data
-    val sourceWrappedData: RDD[(Product, (Map[String, Any], Map[String, Any]))] = sourceData.map(r => (r._1, wrapInitData(r._2)))
-    val targetWrappedData: RDD[(Product, (Map[String, Any], Map[String, Any]))] = targetData.map(r => (r._1, wrapInitData(r._2)))
-
-    // 2. cogroup
-    val allKvs = sourceWrappedData.cogroup(targetWrappedData)
-
-    // 3. accuracy calculation
-    val (accuResult, missingRdd, matchingRdd) = AccuracyCore.accuracy(allKvs, ruleAnalyzer)
-
-    (accuResult, missingRdd, matchingRdd)
-  }
-
-  def record2String(rec: (Product, (Map[String, Any], Map[String, Any]))): String = {
-    val (key, (data, info)) = rec
-    s"${data} [${info}]"
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala
deleted file mode 100644
index 2989538..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/algo/core/AccuracyCore.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package org.apache.griffin.measure.batch.algo.core
-
-import org.apache.griffin.measure.batch.rule.RuleAnalyzer
-import org.apache.griffin.measure.batch.result._
-import org.apache.spark.rdd.RDD
-
-
-object AccuracyCore {
-
-  type V = Map[String, Any]
-  type T = Map[String, Any]
-
-  def accuracy(allKvs: RDD[(Product, (Iterable[(V, T)], Iterable[(V, T)]))], ruleAnalyzer: RuleAnalyzer
-              ): (AccuracyResult, RDD[(Product, (V, T))], RDD[(Product, (V, T))]) = {
-    val result: RDD[(Long, Long, List[(Product, (V, T))], List[(Product, (V, T))])] = allKvs.map { kv =>
-      val (key, (sourceDatas, targetDatas)) = kv
-
-      // result: (missCount, matchCount, missDataList, matchDataList)
-      val rslt = sourceDatas.foldLeft((0L, 0L, List[(Product, (V, T))](), List[(Product, (V, T))]())) { (sr, sourcePair) =>
-        val matchResult = if (targetDatas.isEmpty) {
-          (false, Map[String, Any]((MismatchInfo.key -> "no target")))
-        } else {
-          targetDatas.foldLeft((false, Map[String, Any]())) { (tr, targetPair) =>
-            if (tr._1) tr
-            else matchData(sourcePair, targetPair, ruleAnalyzer)
-          }
-        }
-
-        if (matchResult._1) {
-          val matchItem = (key, sourcePair)
-          (sr._1, sr._2 + 1, sr._3, sr._4 :+ matchItem)
-        } else {
-          val missItem = (key, (sourcePair._1, sourcePair._2 ++ matchResult._2))
-          (sr._1 + 1, sr._2, sr._3 :+ missItem, sr._4)
-        }
-      }
-
-      rslt
-    }
-
-    val missRdd = result.flatMap(_._3)
-    val matchRdd = result.flatMap(_._4)
-
-    def seq(cnt: (Long, Long), rcd: (Long, Long, Any, Any)): (Long, Long) = {
-      (cnt._1 + rcd._1, cnt._2 + rcd._2)
-    }
-    def comb(c1: (Long, Long), c2: (Long, Long)): (Long, Long) = {
-      (c1._1 + c2._1, c1._2 + c2._2)
-    }
-    val countPair = result.aggregate((0L, 0L))(seq, comb)
-
-    (AccuracyResult(countPair._1, (countPair._1 + countPair._2)), missRdd, matchRdd)
-  }
-
-  private def matchData(source: (V, T), target: (V, T), ruleAnalyzer: RuleAnalyzer): (Boolean, T) = {
-
-    // 1. merge source and target cached data
-    val mergedData: Map[String, Any] = mergeData(source, target)
-
-    // 2. check valid
-    if (ruleAnalyzer.rule.valid(mergedData)) {
-      // 3. substitute the cached data into statement, get the statement value
-      // currently we can not get the mismatch reason, we need to add such information to figure out how it mismatches
-      ((ruleAnalyzer.rule.calculate(mergedData) match {
-        case Some(b: Boolean) => b
-        case _ => false
-      }), Map[String, Any]((MismatchInfo.key -> s"not matched with ${target._1}")))
-    } else {
-      (false, Map[String, Any]((MismatchInfo.key -> s"not valid to compare with ${target._1}")))
-    }
-
-  }
-
-  private def mergeData(source: (V, T), target: (V, T)): Map[String, Any] = {
-    source._1 ++ target._1
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala
deleted file mode 100644
index b5618a8..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/AllParam.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.griffin.measure.batch.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.env._
-import org.apache.griffin.measure.batch.config.params.user._
-
-// simply composite of env and user params, for convenient usage
-@JsonInclude(Include.NON_NULL)
-case class AllParam( @JsonProperty("env") envParam: EnvParam,
-                     @JsonProperty("user") userParam: UserParam
-                   ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala
deleted file mode 100644
index ce94c5c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/Param.scala
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.griffin.measure.batch.config.params
-
-trait Param extends Serializable {
-
-  def validate(): Boolean = true
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala
deleted file mode 100644
index a29004c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/CleanerParam.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.env
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class CleanerParam() extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala
deleted file mode 100644
index 79e2575..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/EnvParam.scala
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.env
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class EnvParam( @JsonProperty("spark") sparkParam: SparkParam,
-                     @JsonProperty("persist") persistParams: List[PersistParam],
-                     @JsonProperty("cleaner") cleanerParam: CleanerParam
-                   ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala
deleted file mode 100644
index 655f19b..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/PersistParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.env
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class PersistParam( @JsonProperty("type") persistType: String,
-                         @JsonProperty("config") config: Map[String, Any]
-                       ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala
deleted file mode 100644
index ba6f9a6..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/env/SparkParam.scala
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.env
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class SparkParam( @JsonProperty("log.level") logLevel: String,
-                       @JsonProperty("checkpoint.dir") cpDir: String,
-                       @JsonProperty("config") config: Map[String, Any]
-                     ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala
deleted file mode 100644
index 7a8b2b2..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/DataConnectorParam.scala
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.user
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class DataConnectorParam( @JsonProperty("type") conType: String,
-                               @JsonProperty("version") version: String,
-                               @JsonProperty("config") config: Map[String, Any]
-                             ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala
deleted file mode 100644
index edf0a38..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/EvaluateRuleParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.user
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class EvaluateRuleParam( @JsonProperty("sampleRatio") sampleRatio: Double,
-                              @JsonProperty("rules") rules: String
-                            ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala
deleted file mode 100644
index be0d3c8..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/params/user/UserParam.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.griffin.measure.batch.config.params.user
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import org.apache.griffin.measure.batch.config.params.Param
-
-@JsonInclude(Include.NON_NULL)
-case class UserParam(@JsonProperty("name") name: String,
-                     @JsonProperty("type") dqType: String,
-                     @JsonProperty("source") sourceParam: DataConnectorParam,
-                     @JsonProperty("target") targetParam: DataConnectorParam,
-                     @JsonProperty("evaluateRule") evaluateRuleParam: EvaluateRuleParam
-                    ) extends Param {
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala
deleted file mode 100644
index e30513f..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamFileReader.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.apache.griffin.measure.batch.config.params.Param
-import org.apache.griffin.measure.batch.utils.JsonUtil
-
-import scala.util.Try
-
-case class ParamFileReader(file: String) extends ParamReader {
-
-  def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = {
-    Try {
-      val source = scala.io.Source.fromFile(file)
-      val lines = source.mkString
-      val param = JsonUtil.fromJson[T](lines)
-      source.close
-      param
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala
deleted file mode 100644
index 473b428..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamHdfsFileReader.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.apache.griffin.measure.batch.config.params.Param
-import org.apache.griffin.measure.batch.utils.JsonUtil
-import org.apache.griffin.measure.batch.utils.HdfsUtil
-
-import scala.util.Try
-
-case class ParamHdfsFileReader(filePath: String) extends ParamReader {
-
-  def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = {
-    Try {
-      val source = HdfsUtil.openFile(filePath)
-      val param = JsonUtil.fromJson[T](source)
-      source.close
-      param
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala
deleted file mode 100644
index 1e931de..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamRawStringReader.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.apache.griffin.measure.batch.config.params.Param
-import org.apache.griffin.measure.batch.utils.JsonUtil
-
-import scala.util.Try
-
-case class ParamRawStringReader(rawString: String) extends ParamReader {
-
-  def readConfig[T <: Param](implicit m : Manifest[T]): Try[T] = {
-    Try {
-      val param = JsonUtil.fromJson[T](rawString)
-      param
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala
deleted file mode 100644
index 2f42b09..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReader.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.apache.griffin.measure.batch.log.Loggable
-import org.apache.griffin.measure.batch.config.params.Param
-
-import scala.util.Try
-
-trait ParamReader extends Loggable with Serializable {
-
-  def readConfig[T <: Param](implicit m : Manifest[T]): Try[T]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala
deleted file mode 100644
index dd77e8c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/reader/ParamReaderFactory.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.apache.griffin.measure.batch.config.reader
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
-
-
-object ParamReaderFactory {
-
-  val RawStringRegex = """^(?i)raw$""".r
-  val LocalFsRegex = """^(?i)local$""".r
-  val HdfsFsRegex = """^(?i)hdfs$""".r
-
-  def getParamReader(filePath: String, fsType: String): ParamReader = {
-    fsType match {
-      case RawStringRegex() => ParamRawStringReader(filePath)
-      case LocalFsRegex() => ParamFileReader(filePath)
-      case HdfsFsRegex() => ParamHdfsFileReader(filePath)
-      case _ => ParamHdfsFileReader(filePath)
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/AllParamValidator.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/AllParamValidator.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/AllParamValidator.scala
deleted file mode 100644
index b0ab8b5..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/AllParamValidator.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.apache.griffin.measure.batch.config.validator
-
-import org.apache.griffin.measure.batch.config.params.Param
-
-import scala.util.Try
-
-case class AllParamValidator() extends ParamValidator {
-
-  def validate[T <: Param](param: Param): Try[Boolean] = {
-    Try {
-      // fixme: not done, need to validate param
-      true
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/ParamValidator.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/ParamValidator.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/ParamValidator.scala
deleted file mode 100644
index 9dc9e60..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/config/validator/ParamValidator.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.measure.batch.config.validator
-
-import org.apache.griffin.measure.batch.log.Loggable
-import org.apache.griffin.measure.batch.config.params.Param
-
-import scala.util.Try
-
-trait ParamValidator extends Loggable with Serializable {
-
-  def validate[T <: Param](param: Param): Try[Boolean]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/AvroDataConnector.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/AvroDataConnector.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/AvroDataConnector.scala
deleted file mode 100644
index d271e17..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/AvroDataConnector.scala
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.apache.griffin.measure.batch.connector
-
-import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Row, SQLContext}
-import com.databricks.spark.avro._
-
-import scala.util.{Success, Try}
-import java.nio.file.{Files, Paths}
-
-import org.apache.griffin.measure.batch.utils.HdfsUtil
-
-case class AvroDataConnector(sqlContext: SQLContext, config: Map[String, Any],
-                             groupbyExprs: Seq[MathExpr], cacheExprs: Iterable[Expr],
-                             finalCacheExprs: Iterable[Expr], globalFinalCacheMap: Map[String, Any]
-                            ) extends DataConnector {
-
-  val FilePath = "file.path"
-  val FileName = "file.name"
-
-  val filePath = config.getOrElse(FilePath, "").toString
-  val fileName = config.getOrElse(FileName, "").toString
-
-  val concreteFileFullPath = if (pathPrefix) s"${filePath}${fileName}" else fileName
-
-  private def pathPrefix(): Boolean = {
-    filePath.nonEmpty
-  }
-
-  private def fileExist(): Boolean = {
-    HdfsUtil.existPath(concreteFileFullPath)
-  }
-
-  def available(): Boolean = {
-    (!concreteFileFullPath.isEmpty) && fileExist
-  }
-
-  def metaData(): Try[Iterable[(String, String)]] = {
-    Try {
-      val st = sqlContext.read.format("com.databricks.spark.avro").load(concreteFileFullPath).schema
-      st.fields.map(f => (f.name, f.dataType.typeName))
-    }
-  }
-
-  def data(): Try[RDD[(Product, Map[String, Any])]] = {
-    Try {
-      loadDataFile.map { row =>
-        // generate cache data
-        val cacheData: Map[String, Any] = cacheExprs.foldLeft(globalFinalCacheMap) { (cachedMap, expr) =>
-          CacheDataUtil.genCachedMap(Some(row), expr, cachedMap)
-        }
-        val finalCacheData = CacheDataUtil.filterCachedMap(finalCacheExprs, cacheData)
-
-        // get groupby data
-        val groupbyData: Seq[AnyRef] = groupbyExprs.flatMap { expr =>
-          expr.calculate(finalCacheData) match {
-            case Some(v) => Some(v.asInstanceOf[AnyRef])
-            case _ => None
-          }
-        }
-        val key = toTuple(groupbyData)
-
-        (key, finalCacheData)
-      }
-    }
-  }
-
-  private def loadDataFile() = {
-    sqlContext.read.format("com.databricks.spark.avro").load(concreteFileFullPath)
-  }
-
-  private def toTuple[A <: AnyRef](as: Seq[A]): Product = {
-    if (as.size > 0) {
-      val tupleClass = Class.forName("scala.Tuple" + as.size)
-      tupleClass.getConstructors.apply(0).newInstance(as: _*).asInstanceOf[Product]
-    } else None
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/CacheDataUtil.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/CacheDataUtil.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/CacheDataUtil.scala
deleted file mode 100644
index 884df5d..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/CacheDataUtil.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.griffin.measure.batch.connector
-
-import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.spark.sql.Row
-
-import scala.util.{Success, Try}
-
-object CacheDataUtil {
-
-  // for now, one expr only get one value, not supporting one expr get multiple values
-  private def getCacheData(data: Option[Any], expr: Expr, cachedMap: Map[String, Any]): Option[Any] = {
-    Try {
-      expr match {
-        case selection: SelectionExpr => {
-          selection.selectors.foldLeft(data) { (dt, selector) =>
-            getCacheData(dt, selector, cachedMap)
-          }
-        }
-        case selector: IndexFieldRangeSelectExpr => {
-          data match {
-            case Some(row: Row) => {
-              if (selector.fields.size == 1) {
-                selector.fields.head match {
-                  case i: IndexDesc => Some(row.getAs[Any](i.index))
-                  case f: FieldDesc => Some(row.getAs[Any](f.field))
-                  case _ => None
-                }
-              } else None
-            }
-            case _ => None
-          }
-        }
-        case _ => expr.calculate(cachedMap)
-      }
-    } match {
-      case Success(v) => v
-      case _ => None
-    }
-  }
-
-  def genCachedMap(data: Option[Any], expr: Expr, initialCachedMap: Map[String, Any]): Map[String, Any] = {
-    val valueOpt = getCacheData(data, expr, initialCachedMap)
-    if (valueOpt.nonEmpty) {
-      initialCachedMap + (expr._id -> valueOpt.get)
-    } else initialCachedMap
-  }
-
-  def genCachedMap(data: Option[Any], exprs: Iterable[Expr], initialCachedMap: Map[String, Any]): Map[String, Any] = {
-    exprs.foldLeft(initialCachedMap) { (cachedMap, expr) =>
-      CacheDataUtil.genCachedMap(None, expr, cachedMap)
-    }
-  }
-
-  def filterCachedMap(exprs: Iterable[Expr], cachedMap: Map[String, Any]): Map[String, Any] = {
-    exprs.foldLeft(Map[String, Any]()) { (newMap, expr) =>
-      val valueOpt = expr.calculate(cachedMap)
-      if (valueOpt.nonEmpty) {
-        newMap + (expr._id -> valueOpt.get)
-      } else newMap
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnector.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnector.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnector.scala
deleted file mode 100644
index 9cc9be6..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnector.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.apache.griffin.measure.batch.connector
-
-import org.apache.spark.rdd.RDD
-
-import scala.util.Try
-
-
-trait DataConnector extends Serializable {
-
-  def available(): Boolean
-
-  def metaData(): Try[Iterable[(String, String)]]
-
-  def data(): Try[RDD[(Product, Map[String, Any])]]
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnectorFactory.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnectorFactory.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnectorFactory.scala
deleted file mode 100644
index ffa8d9c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/DataConnectorFactory.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.griffin.measure.batch.connector
-
-import org.apache.griffin.measure.batch.config.params.user._
-import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.spark.sql.SQLContext
-
-import scala.util.Try
-
-object DataConnectorFactory {
-
-  val HiveRegex = """^(?i)hive$""".r
-  val AvroRegex = """^(?i)avro$""".r
-
-  def getDataConnector(sqlContext: SQLContext,
-                       dataConnectorParam: DataConnectorParam,
-                       groupbyExprs: Seq[MathExpr],
-                       cacheExprs: Iterable[Expr],
-                       finalCacheExprs: Iterable[Expr],
-                       globalFinalCacheMap: Map[String, Any]
-                      ): Try[DataConnector] = {
-    val conType = dataConnectorParam.conType
-    val version = dataConnectorParam.version
-    Try {
-      conType match {
-        case HiveRegex() => HiveDataConnector(sqlContext, dataConnectorParam.config, groupbyExprs, cacheExprs, finalCacheExprs, globalFinalCacheMap)
-        case AvroRegex() => AvroDataConnector(sqlContext, dataConnectorParam.config, groupbyExprs, cacheExprs, finalCacheExprs, globalFinalCacheMap)
-        case _ => throw new Exception("connector creation error!")
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/HiveDataConnector.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/HiveDataConnector.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/HiveDataConnector.scala
deleted file mode 100644
index 5c80e30..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/connector/HiveDataConnector.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-package org.apache.griffin.measure.batch.connector
-
-import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Row, SQLContext}
-
-import scala.util.{Success, Try}
-
-case class HiveDataConnector(sqlContext: SQLContext, config: Map[String, Any],
-                             groupbyExprs: Seq[MathExpr], cacheExprs: Iterable[Expr],
-                             finalCacheExprs: Iterable[Expr], globalFinalCacheMap: Map[String, Any]
-                            ) extends DataConnector {
-
-  val Database = "database"
-  val TableName = "table.name"
-  val Partitions = "partitions"
-
-  val database = config.getOrElse(Database, "").toString
-  val tableName = config.getOrElse(TableName, "").toString
-  val partitionsString = config.getOrElse(Partitions, "").toString
-
-  val concreteTableName = if (dbPrefix) s"${database}.${tableName}" else tableName
-  val partitions = partitionsString.split(";").map(s => s.split(",").map(_.trim))
-
-  private def dbPrefix(): Boolean = {
-    database.nonEmpty && !database.equals("default")
-  }
-
-  def available(): Boolean = {
-    (!tableName.isEmpty) && {
-      Try {
-        if (dbPrefix) {
-          sqlContext.tables(database).filter(tableExistsSql).collect.size
-        } else {
-          sqlContext.tables().filter(tableExistsSql).collect.size
-        }
-      } match {
-        case Success(s) => s > 0
-        case _ => false
-      }
-    }
-  }
-
-  def metaData(): Try[Iterable[(String, String)]] = {
-    Try {
-      val originRows = sqlContext.sql(metaDataSql).map(r => (r.getString(0), r.getString(1))).collect
-      val partitionPos: Int = originRows.indexWhere(pair => pair._1.startsWith("# "))
-      if (partitionPos < 0) originRows
-      else originRows.take(partitionPos)
-    }
-  }
-
-  def data(): Try[RDD[(Product, Map[String, Any])]] = {
-    Try {
-      sqlContext.sql(dataSql).map { row =>
-        // generate cache data
-        val cacheData: Map[String, Any] = cacheExprs.foldLeft(globalFinalCacheMap) { (cachedMap, expr) =>
-          CacheDataUtil.genCachedMap(Some(row), expr, cachedMap)
-        }
-        val finalCacheData = CacheDataUtil.filterCachedMap(finalCacheExprs, cacheData)
-
-        // get groupby data
-        val groupbyData: Seq[AnyRef] = groupbyExprs.flatMap { expr =>
-          expr.calculate(finalCacheData) match {
-            case Some(v) => Some(v.asInstanceOf[AnyRef])
-            case _ => None
-          }
-        }
-        val key = toTuple(groupbyData)
-
-        (key, finalCacheData)
-      }
-    }
-  }
-
-  private def tableExistsSql(): String = {
-//    s"SHOW TABLES LIKE '${concreteTableName}'"    // this is hive sql, but not work for spark sql
-    s"tableName LIKE '${tableName}'"
-  }
-
-  private def metaDataSql(): String = {
-    s"DESCRIBE ${concreteTableName}"
-  }
-
-  private def dataSql(): String = {
-    val clauses = partitions.map { prtn =>
-      val cls = prtn.mkString(" AND ")
-      if (cls.isEmpty) s"SELECT * FROM ${concreteTableName}"
-      else s"SELECT * FROM ${concreteTableName} WHERE ${cls}"
-    }
-    clauses.mkString(" UNION ALL ")
-  }
-
-  private def toTuple[A <: AnyRef](as: Seq[A]): Product = {
-    if (as.size > 0) {
-      val tupleClass = Class.forName("scala.Tuple" + as.size)
-      tupleClass.getConstructors.apply(0).newInstance(as: _*).asInstanceOf[Product]
-    } else None
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/log/Loggable.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/log/Loggable.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/log/Loggable.scala
deleted file mode 100644
index f73e86c..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/log/Loggable.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.apache.griffin.measure.batch.log
-
-import org.slf4j.LoggerFactory
-
-trait Loggable {
-
-  @transient private lazy val logger = LoggerFactory.getLogger(getClass)
-
-  protected def info(msg: String): Unit = {
-    logger.info(msg)
-  }
-
-  protected def debug(msg: String): Unit = {
-    logger.debug(msg)
-  }
-
-  protected def warn(msg: String): Unit = {
-    logger.warn(msg)
-  }
-
-  protected def error(msg: String): Unit = {
-    logger.error(msg)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HdfsPersist.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HdfsPersist.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HdfsPersist.scala
deleted file mode 100644
index 7bff3b6..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HdfsPersist.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-package org.apache.griffin.measure.batch.persist
-
-import java.util.Date
-
-import org.apache.griffin.measure.batch.result._
-import org.apache.griffin.measure.batch.utils.HdfsUtil
-import org.apache.spark.rdd.RDD
-
-
-case class HdfsPersist(config: Map[String, Any], metricName: String, timeStamp: Long) extends Persist {
-
-  val Path = "path"
-  val MaxPersistLines = "max.persist.lines"
-  val MaxLinesPerFile = "max.lines.per.file"
-
-  val path = config.getOrElse(Path, "").toString
-  val maxPersistLines = config.getOrElse(MaxPersistLines, -1).toString.toLong
-  val maxLinesPerFile = config.getOrElse(MaxLinesPerFile, 10000).toString.toLong
-
-  val separator = "/"
-
-  val StartFile = filePath("_START")
-  val FinishFile = filePath("_FINISH")
-  val ResultFile = filePath("_RESULT")
-
-  val MissRecFile = filePath("_MISSREC")      // optional
-
-  val LogFile = filePath("_LOG")
-
-  var _init = true
-  private def isInit = {
-    val i = _init
-    _init = false
-    i
-  }
-
-  def available(): Boolean = {
-    (path.nonEmpty) && (maxPersistLines < Int.MaxValue)
-  }
-
-  private def persistHead: String = {
-    val dt = new Date(timeStamp)
-    s"================ log of ${dt} ================\n"
-  }
-
-  private def timeHead(rt: Long): String = {
-    val dt = new Date(rt)
-    s"--- ${dt} ---\n"
-  }
-
-  protected def getFilePath(parentPath: String, fileName: String): String = {
-    if (parentPath.endsWith(separator)) parentPath + fileName else parentPath + separator + fileName
-  }
-
-  protected def filePath(file: String): String = {
-    getFilePath(path, s"${metricName}/${timeStamp}/${file}")
-  }
-
-  protected def withSuffix(path: String, suffix: String): String = {
-    s"${path}.${suffix}"
-  }
-
-  def start(msg: String): Unit = {
-    HdfsUtil.writeContent(StartFile, msg)
-  }
-  def finish(): Unit = {
-    HdfsUtil.createEmptyFile(FinishFile)
-  }
-
-  def result(rt: Long, result: Result): Unit = {
-    val resStr = result match {
-      case ar: AccuracyResult => {
-        s"match percentage: ${ar.matchPercentage}\ntotal count: ${ar.getTotal}\nmiss count: ${ar.getMiss}, match count: ${ar.getMatch}"
-      }
-      case _ => {
-        s"result: ${result}"
-      }
-    }
-    HdfsUtil.writeContent(ResultFile, timeHead(rt) + resStr)
-    log(rt, resStr)
-
-    info(resStr)
-  }
-
-  // need to avoid string too long
-  def missRecords(records: RDD[String]): Unit = {
-    val recordCount = records.count
-    val count = if (maxPersistLines < 0) recordCount else scala.math.min(maxPersistLines, recordCount)
-    if (count > 0) {
-      val groupCount = ((count - 1) / maxLinesPerFile + 1).toInt
-      if (groupCount <= 1) {
-        val recs = records.take(count.toInt)
-        persistRecords(MissRecFile, recs)
-      } else {
-        val groupedRecords: RDD[(Long, Iterable[String])] =
-          records.zipWithIndex.flatMap { r =>
-            val gid = r._2 / maxLinesPerFile
-            if (gid < groupCount) Some((gid, r._1)) else None
-          }.groupByKey()
-        groupedRecords.foreach { group =>
-          val (gid, recs) = group
-          val hdfsPath = if (gid == 0) MissRecFile else withSuffix(MissRecFile, gid.toString)
-          persistRecords(hdfsPath, recs)
-        }
-      }
-    }
-  }
-
-  private def persistRecords(hdfsPath: String, records: Iterable[String]): Unit = {
-    val recStr = records.mkString("\n")
-    HdfsUtil.appendContent(hdfsPath, recStr)
-  }
-
-  def log(rt: Long, msg: String): Unit = {
-    val logStr = (if (isInit) persistHead else "") + timeHead(rt) + s"${msg}\n\n"
-    HdfsUtil.appendContent(LogFile, logStr)
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HttpPersist.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HttpPersist.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HttpPersist.scala
deleted file mode 100644
index 5765927..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/HttpPersist.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.apache.griffin.measure.batch.persist
-
-import org.apache.griffin.measure.batch.result._
-import org.apache.griffin.measure.batch.utils.{HttpUtil, JsonUtil}
-import org.apache.spark.rdd.RDD
-
-case class HttpPersist(config: Map[String, Any], metricName: String, timeStamp: Long) extends Persist {
-
-  val Api = "api"
-  val Method = "method"
-
-  val api = config.getOrElse(Api, "").toString
-  val method = config.getOrElse(Method, "post").toString
-
-  def available(): Boolean = {
-    api.nonEmpty
-  }
-
-  def start(msg: String): Unit = {}
-  def finish(): Unit = {}
-
-  def result(rt: Long, result: Result): Unit = {
-    result match {
-      case ar: AccuracyResult => {
-        val dataMap = Map[String, Any](("metricName" -> metricName), ("timestamp" -> timeStamp), ("value" -> ar.matchPercentage))
-        val data = JsonUtil.toJson(dataMap)
-
-        // post
-        val params = Map[String, Object]()
-        val header = Map[String, Object](("content-type" -> "application/json"))
-        val status = HttpUtil.httpRequest(api, method, params, header, data)
-        info(s"${method} to ${api} response status: ${status}")
-      }
-      case _ => {
-        info(s"result: ${result}")
-      }
-    }
-  }
-
-  def missRecords(records: RDD[String]): Unit = {}
-
-  def log(rt: Long, msg: String): Unit = {}
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/MultiPersists.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/MultiPersists.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/MultiPersists.scala
deleted file mode 100644
index 2fa6942..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/MultiPersists.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.apache.griffin.measure.batch.persist
-
-import org.apache.griffin.measure.batch.result._
-import org.apache.griffin.measure.batch.utils.{HttpUtil, JsonUtil}
-import org.apache.spark.rdd.RDD
-
-case class MultiPersists(persists: Iterable[Persist]) extends Persist {
-
-  val timeStamp: Long = persists match {
-    case Nil => 0
-    case _ => persists.head.timeStamp
-  }
-
-  val config: Map[String, Any] = Map[String, Any]()
-
-  def available(): Boolean = { persists.exists(_.available()) }
-
-  def start(msg: String): Unit = { persists.foreach(_.start(msg)) }
-  def finish(): Unit = {persists.foreach(_.finish())}
-
-  def result(rt: Long, result: Result): Unit = { persists.foreach(_.result(rt, result)) }
-
-  def missRecords(records: RDD[String]): Unit = { persists.foreach(_.missRecords(records)) }
-
-  def log(rt: Long, msg: String): Unit = { persists.foreach(_.log(rt, msg)) }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/Persist.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/Persist.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/Persist.scala
deleted file mode 100644
index 7398c24..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/Persist.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.apache.griffin.measure.batch.persist
-
-import org.apache.griffin.measure.batch.log.Loggable
-import org.apache.griffin.measure.batch.result._
-import org.apache.spark.rdd.RDD
-
-
-trait Persist extends Loggable with Serializable {
-  val timeStamp: Long
-
-  val config: Map[String, Any]
-
-  def available(): Boolean
-
-  def start(msg: String): Unit
-  def finish(): Unit
-
-  def result(rt: Long, result: Result): Unit
-
-  def missRecords(records: RDD[String]): Unit
-
-  def log(rt: Long, msg: String): Unit
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/PersistFactory.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/PersistFactory.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/PersistFactory.scala
deleted file mode 100644
index 84dc4ce..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/persist/PersistFactory.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.apache.griffin.measure.batch.persist
-
-import org.apache.griffin.measure.batch.config.params.env._
-
-import scala.util.{Success, Try}
-
-
-case class PersistFactory(persistParams: Iterable[PersistParam], metricName: String) extends Serializable {
-
-  val HDFS_REGEX = """^(?i)hdfs$""".r
-  val HTTP_REGEX = """^(?i)http$""".r
-
-  def getPersists(timeStamp: Long): MultiPersists = {
-    MultiPersists(persistParams.flatMap(param => getPersist(timeStamp, param)))
-  }
-
-  private def getPersist(timeStamp: Long, persistParam: PersistParam): Option[Persist] = {
-    val persistConfig = persistParam.config
-    val persistTry = persistParam.persistType match {
-      case HDFS_REGEX() => Try(HdfsPersist(persistConfig, metricName, timeStamp))
-      case HTTP_REGEX() => Try(HttpPersist(persistConfig, metricName, timeStamp))
-      case _ => throw new Exception("not supported persist type")
-    }
-    persistTry match {
-      case Success(persist) if (persist.available) => Some(persist)
-      case _ => None
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/AccuracyResult.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/AccuracyResult.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/AccuracyResult.scala
deleted file mode 100644
index 55505ac..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/AccuracyResult.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.apache.griffin.measure.batch.result
-
-
-case class AccuracyResult(miss: Long, total: Long) extends Result {
-
-  type T = AccuracyResult
-
-  def update(delta: T): T = {
-    AccuracyResult(delta.miss, total)
-  }
-
-  def eventual(): Boolean = {
-    this.miss <= 0
-  }
-
-  def differsFrom(other: T): Boolean = {
-    (this.miss != other.miss) || (this.total != other.total)
-  }
-
-  def getMiss = miss
-  def getTotal = total
-  def getMatch = total - miss
-
-  def matchPercentage: Double = if (getTotal <= 0) 0 else getMatch.toDouble / getTotal * 100
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/Result.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/Result.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/Result.scala
deleted file mode 100644
index 8d529db..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/Result.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.griffin.measure.batch.result
-
-
-trait Result extends Serializable {
-
-  type T <: Result
-
-  def update(delta: T): T
-
-  def eventual(): Boolean
-
-  def differsFrom(other: T): Boolean
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/ResultInfo.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/ResultInfo.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/ResultInfo.scala
deleted file mode 100644
index a430318..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/result/ResultInfo.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.griffin.measure.batch.result
-
-
-sealed trait ResultInfo {
-  type T
-  val key: String
-  val tp: String
-  def wrap(value: T) = (key -> value)
-}
-
-final case object TimeGroupInfo extends ResultInfo {
-  type T = Long
-  val key = "__time__"
-  val tp = "bigint"
-}
-
-final case object NextFireTimeInfo extends ResultInfo {
-  type T = Long
-  val key = "__next_fire_time__"
-  val tp = "bigint"
-}
-
-final case object MismatchInfo extends ResultInfo {
-  type T = String
-  val key = "__mismatch__"
-  val tp = "string"
-}
-
-final case object ErrorInfo extends ResultInfo {
-  type T = String
-  val key = "__error__"
-  val tp = "string"
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzer.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzer.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzer.scala
deleted file mode 100644
index 78665f9..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzer.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-package org.apache.griffin.measure.batch.rule
-
-import org.apache.griffin.measure.batch.rule.expr._
-
-case class RuleAnalyzer(rule: StatementExpr) extends Serializable {
-
-  val GlobalData = ""
-  val SourceData = "source"
-  val TargetData = "target"
-
-  val globalCacheExprs: Iterable[Expr] = rule.getCacheExprs(GlobalData)
-  val sourceCacheExprs: Iterable[Expr] = rule.getCacheExprs(SourceData)
-  val targetCacheExprs: Iterable[Expr] = rule.getCacheExprs(TargetData)
-
-  val sourcePersistExprs: Iterable[Expr] = rule.getPersistExprs(SourceData)
-  val targetPersistExprs: Iterable[Expr] = rule.getPersistExprs(TargetData)
-
-  val globalFinalCacheExprs: Iterable[Expr] = rule.getFinalCacheExprs(GlobalData).toSet
-  val sourceFinalCacheExprs: Iterable[Expr] = rule.getFinalCacheExprs(SourceData).toSet ++ sourcePersistExprs.toSet
-  val targetFinalCacheExprs: Iterable[Expr] = rule.getFinalCacheExprs(TargetData).toSet ++ targetPersistExprs.toSet
-
-  val groupbyExprPairs: Seq[(MathExpr, MathExpr)] = rule.getGroupbyExprPairs((SourceData, TargetData))
-  val sourceGroupbyExprs: Seq[MathExpr] = groupbyExprPairs.map(_._1)
-  val targetGroupbyExprs: Seq[MathExpr] = groupbyExprPairs.map(_._2)
-
-  val whenClauseExpr: Option[LogicalExpr] = rule.getWhenClauseExpr
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzerOld.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzerOld.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzerOld.scala
deleted file mode 100644
index dd90c81..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleAnalyzerOld.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.apache.griffin.measure.batch.rule
-
-//import org.apache.griffin.measure.batch.rule.expr._
-import org.apache.griffin.measure.batch.rule.expr_old._
-
-case class RuleAnalyzerOld(rule: StatementExpr) extends Serializable {
-
-  val SourceData = "source"
-  val TargetData = "target"
-
-//  val sourceCacheExprs: Iterable[Expr] = rule.getCacheExprs(SourceData)
-//  val targetCacheExprs: Iterable[Expr] = rule.getCacheExprs(TargetData)
-
-  val sourceDataExprs: Iterable[DataExpr] = rule.getDataRelatedExprs(SourceData)
-  val targetDataExprs: Iterable[DataExpr] = rule.getDataRelatedExprs(TargetData)
-
-  val keyMappings: Iterable[MappingExpr] = rule.getKeyMappings()
-
-  val sourceDataKeyExprs: Seq[DataExpr] = keyMappings.flatMap(_.getDataRelatedExprs(SourceData)).toSeq
-  val targetDatakeyExprs: Seq[DataExpr] = keyMappings.flatMap(_.getDataRelatedExprs(TargetData)).toSeq
-
-  val assigns: Iterable[AssignExpr] = rule.getAssigns()
-  val conditions: Iterable[ConditionExpr] = rule.getConditions()
-  val mappings: Iterable[MappingExpr] = rule.getMappings()
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/c4340cea/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleFactory.scala
----------------------------------------------------------------------
diff --git a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleFactory.scala b/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleFactory.scala
deleted file mode 100644
index cc3e8b3..0000000
--- a/griffin-measure/griffin-measure-batch/src/main/scala/org/apache/griffin/measure/batch/rule/RuleFactory.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.apache.griffin.measure.batch.rule
-
-import org.apache.griffin.measure.batch.config.params.user._
-
-import scala.util.Failure
-//import org.apache.griffin.measure.batch.rule.expr_old._
-import org.apache.griffin.measure.batch.rule.expr._
-
-import scala.util.{Success, Try}
-
-
-case class RuleFactory(evaluateRuleParam: EvaluateRuleParam) {
-
-  val ruleParser: RuleParser = RuleParser()
-
-  def generateRule(): StatementExpr = {
-    val rules = evaluateRuleParam.rules
-    val statement = parseExpr(rules) match {
-      case Success(se) => se
-      case Failure(ex) => throw ex
-    }
-    statement
-  }
-
-  private def parseExpr(rules: String): Try[StatementExpr] = {
-    Try {
-      val result = ruleParser.parseAll(ruleParser.rule, rules)
-      if (result.successful) result.get
-      else throw new Exception("parse rule error!")
-//      throw new Exception("parse rule error!")
-    }
-  }
-
-}