You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/03/03 12:16:55 UTC

[49/51] [partial] incubator-griffin git commit: Init move

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mongodb-org-3.2.repo
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mongodb-org-3.2.repo b/docker/griffin-base/config/mongodb-org-3.2.repo
new file mode 100755
index 0000000..da73c93
--- /dev/null
+++ b/docker/griffin-base/config/mongodb-org-3.2.repo
@@ -0,0 +1,5 @@
+[mongodb-org-3.2]
+name=MongoDB Repository
+baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.2/x86_64/
+gpgcheck=0
+enabled=1

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_init.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_init.sh b/docker/griffin-base/config/mysql_init.sh
new file mode 100755
index 0000000..de68ffa
--- /dev/null
+++ b/docker/griffin-base/config/mysql_init.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+sed -i.bak s/^.*"hive-txn-schema-0.13.0.mysql.sql".*/"SOURCE \/apache\/hive\/scripts\/metastore\/upgrade\/mysql\/hive-txn-schema-0.13.0.mysql.sql;"/ /apache/hive/scripts/metastore/upgrade/mysql/hive-schema-1.2.0.mysql.sql
+
+mysql -u root -p123456 < $GRIFFIN_HOME/mysql_init_metastore.sql
+
+rm $GRIFFIN_HOME/mysql_init_metastore.sql

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_init_metastore.sql
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_init_metastore.sql b/docker/griffin-base/config/mysql_init_metastore.sql
new file mode 100755
index 0000000..3477a06
--- /dev/null
+++ b/docker/griffin-base/config/mysql_init_metastore.sql
@@ -0,0 +1,9 @@
+CREATE DATABASE metastore;
+USE metastore;
+SOURCE /apache/hive/scripts/metastore/upgrade/mysql/hive-schema-1.2.0.mysql.sql;
+
+
+CREATE USER 'hive'@'localhost' IDENTIFIED BY '123456';
+REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost';
+GRANT SELECT,INSERT,UPDATE,DELETE,LOCK TABLES,EXECUTE ON metastore.* TO 'hive'@'localhost';
+FLUSH PRIVILEGES;

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_secure.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_secure.sh b/docker/griffin-base/config/mysql_secure.sh
new file mode 100755
index 0000000..aebcd75
--- /dev/null
+++ b/docker/griffin-base/config/mysql_secure.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+#
+# Automate mysql secure installation for debian-baed systems
+# 
+#  - You can set a password for root accounts.
+#  - You can remove root accounts that are accessible from outside the local host.
+#  - You can remove anonymous-user accounts.
+#  - You can remove the test database (which by default can be accessed by all users, even anonymous users), 
+#    and privileges that permit anyone to access databases with names that start with test_. 
+#  For details see documentation: http://dev.mysql.com/doc/refman/5.7/en/mysql-secure-installation.html
+#
+# @version 13.08.2014 00:39 +03:00
+# Tested on Debian 7.6 (wheezy)
+#
+# Usage:
+#  Setup mysql root password:  ./mysql_secure.sh 'your_new_root_password'
+#  Change mysql root password: ./mysql_secure.sh 'your_old_root_password' 'your_new_root_password'"
+#
+
+# Delete package expect when script is done
+# 0 - No; 
+# 1 - Yes.
+PURGE_EXPECT_WHEN_DONE=0
+
+#
+# Check the bash shell script is being run by root
+#
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+
+#
+# Check input params
+#
+if [ -n "${1}" -a -z "${2}" ]; then
+    # Setup root password
+    CURRENT_MYSQL_PASSWORD=''
+    NEW_MYSQL_PASSWORD="${1}"
+elif [ -n "${1}" -a -n "${2}" ]; then
+    # Change existens root password
+    CURRENT_MYSQL_PASSWORD="${1}"
+    NEW_MYSQL_PASSWORD="${2}"
+else
+    echo "Usage:"
+    echo "  Setup mysql root password: ${0} 'your_new_root_password'"
+    echo "  Change mysql root password: ${0} 'your_old_root_password' 'your_new_root_password'"
+    exit 1
+fi
+
+#
+# Check is expect package installed
+#
+if [ $(dpkg-query -W -f='${Status}' expect 2>/dev/null | grep -c "ok installed") -eq 0 ]; then
+    echo "Can't find expect. Trying install it..."
+    yum -y install expect
+
+fi
+
+SECURE_MYSQL=$(expect -c "
+set timeout 3
+spawn mysql_secure_installation
+expect \"Enter current password for root (enter for none):\"
+send \"$CURRENT_MYSQL_PASSWORD\r\"
+expect \"root password?\"
+send \"y\r\"
+expect \"New password:\"
+send \"$NEW_MYSQL_PASSWORD\r\"
+expect \"Re-enter new password:\"
+send \"$NEW_MYSQL_PASSWORD\r\"
+expect \"Remove anonymous users?\"
+send \"y\r\"
+expect \"Disallow root login remotely?\"
+send \"n\r\"
+expect \"Remove test database and access to it?\"
+send \"y\r\"
+expect \"Reload privilege tables now?\"
+send \"y\r\"
+expect eof
+")
+
+#
+# Execution mysql_secure_installation
+#
+echo "${SECURE_MYSQL}"
+
+if [ "${PURGE_EXPECT_WHEN_DONE}" -eq 1 ]; then
+    # Uninstalling expect package
+    yum -y purge expect
+fi
+
+exit 0

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/spark-conf-mod.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/spark-conf-mod.sh b/docker/griffin-base/config/spark-conf-mod.sh
new file mode 100755
index 0000000..f309f5e
--- /dev/null
+++ b/docker/griffin-base/config/spark-conf-mod.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+
+cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
+echo "SPARK_MASTER_WEBUI_PORT=8082" >> $SPARK_HOME/conf/spark-env.sh

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/tomcat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/tomcat b/docker/griffin-base/config/tomcat
new file mode 100755
index 0000000..e195bd9
--- /dev/null
+++ b/docker/griffin-base/config/tomcat
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# description: Tomcat Start Stop Restart
+# processname: tomcat
+# chkconfig: 234 20 80
+JAVA_HOME=/usr/java/latest
+export JAVA_HOME
+PATH=$JAVA_HOME/bin:$PATH
+export PATH
+JAVA_OPTS="-Dfile.encoding=UTF-8 -Xms128m -Xmx1024m -XX:PermSize=128m -XX:MaxPermSize=512m"
+export JAVA_OPTS
+echo $JAVA_OPTS
+CATALINA_HOME=/apache/tomcat
+RETVAL=$?
+case "$1" in
+start)
+$CATALINA_HOME/bin/startup.sh
+;;
+restart)
+$CATALINA_HOME/bin/shutdown.sh
+$CATALINA_HOME/bin/startup.sh
+;;
+stop)
+$CATALINA_HOME/bin/shutdown.sh
+esac
+exit 0

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/dataFile/users_info_src.dat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/dataFile/users_info_src.dat b/docker/griffin-base/griffin/dataFile/users_info_src.dat
new file mode 100755
index 0000000..ce49443
--- /dev/null
+++ b/docker/griffin-base/griffin/dataFile/users_info_src.dat
@@ -0,0 +1,50 @@
+10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000001|94022
+10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000002|94022
+10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
+10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
+10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
+10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
+10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
+10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
+10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
+10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
+10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
+10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
+10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
+10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
+10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
+10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
+10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
+10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
+10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
+10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
+10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
+10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
+10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
+10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
+10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
+10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
+10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
+10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
+10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
+10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
+10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
+10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
+10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
+10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
+10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
+10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
+10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
+10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
+10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
+10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
+10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
+10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
+10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
+10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
+10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
+10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
+10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
+10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
+10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
+10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/dataFile/users_info_target.dat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/dataFile/users_info_target.dat b/docker/griffin-base/griffin/dataFile/users_info_target.dat
new file mode 100755
index 0000000..07a6b40
--- /dev/null
+++ b/docker/griffin-base/griffin/dataFile/users_info_target.dat
@@ -0,0 +1,50 @@
+10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000101|94022
+10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000102|94022
+10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
+10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
+10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
+10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
+10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
+10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
+10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
+10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
+10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
+10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
+10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
+10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
+10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
+10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
+10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
+10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
+10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
+10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
+10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
+10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
+10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
+10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
+10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
+10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
+10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
+10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
+10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
+10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
+10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
+10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
+10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
+10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
+10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
+10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
+10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
+10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
+10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
+10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
+10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
+10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
+10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
+10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
+10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
+10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
+10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
+10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
+10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
+10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hadoop-end.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hadoop-end.sh b/docker/griffin-base/griffin/hadoop-end.sh
new file mode 100755
index 0000000..432bf8f
--- /dev/null
+++ b/docker/griffin-base/griffin/hadoop-end.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/sbin/stop-dfs.sh
+$HADOOP_PREFIX/sbin/stop-yarn.sh
+service sshd stop

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hadoop-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hadoop-start.sh b/docker/griffin-base/griffin/hadoop-start.sh
new file mode 100755
index 0000000..8e88406
--- /dev/null
+++ b/docker/griffin-base/griffin/hadoop-start.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+HOSTNAME=sandbox
+echo $(head -1 /etc/hosts | cut -f1) $HOSTNAME >> /etc/hosts
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-after-hive.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-after-hive.sh b/docker/griffin-base/griffin/hd-after-hive.sh
new file mode 100755
index 0000000..75e8ccd
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-after-hive.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+
+hadoop fs -touchz /user/hive/warehouse/users_info_src/_SUCCESS
+hadoop fs -touchz /user/hive/warehouse/users_info_target/_SUCCESS

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-before-hive.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-before-hive.sh b/docker/griffin-base/griffin/hd-before-hive.sh
new file mode 100755
index 0000000..c932d7a
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-before-hive.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+hadoop fs -mkdir /user/hive/warehouse/users_info_src
+hadoop fs -put $GRIFFIN_HOME/dataFile/users_info_src.dat /user/hive/warehouse/users_info_src/
+hadoop fs -mkdir /user/hive/warehouse/users_info_target
+hadoop fs -put $GRIFFIN_HOME/dataFile/users_info_target.dat /user/hive/warehouse/users_info_target/

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-test-json.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-test-json.sh b/docker/griffin-base/griffin/hd-test-json.sh
new file mode 100755
index 0000000..8fa0aec
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-test-json.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+hadoop fs -mkdir /user/test
+hadoop fs -put $GRIFFIN_HOME/jsonFile/accu_config.json /user/test/
+hadoop fs -put $GRIFFIN_HOME/jsonFile/vali_config.json /user/test/
+hadoop fs -mkdir /user/test/output

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hive-init.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hive-init.sh b/docker/griffin-base/griffin/hive-init.sh
new file mode 100755
index 0000000..70fab77
--- /dev/null
+++ b/docker/griffin-base/griffin/hive-init.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#mysql service
+service mysqld start
+
+$GRIFFIN_HOME/mysql_secure.sh 123456 && rm $GRIFFIN_HOME/mysql_secure.sh
+$GRIFFIN_HOME/mysql_init.sh && rm $GRIFFIN_HOME/mysql_init.sh
+
+#hive metastore service
+hive --service metastore &
+
+#insert hive table
+hive -f $GRIFFIN_HOME/hive-input.hql
+
+rm $GRIFFIN_HOME/hive-input.hql

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hive-input.hql
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hive-input.hql b/docker/griffin-base/griffin/hive-input.hql
new file mode 100755
index 0000000..9f33879
--- /dev/null
+++ b/docker/griffin-base/griffin/hive-input.hql
@@ -0,0 +1,25 @@
+CREATE TABLE users_info_src (
+  user_id bigint,
+  first_name string,
+  last_name string,
+  address string,
+  email string,
+  phone string,
+  post_code string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;
+
+
+
+CREATE TABLE users_info_target (
+  user_id bigint,
+  first_name string,
+  last_name string,
+  address string,
+  email string,
+  phone string,
+  post_code string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/jsonFile/accu_config.json
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/jsonFile/accu_config.json b/docker/griffin-base/griffin/jsonFile/accu_config.json
new file mode 100755
index 0000000..0b24d8b
--- /dev/null
+++ b/docker/griffin-base/griffin/jsonFile/accu_config.json
@@ -0,0 +1,63 @@
+{
+  "source": "users_info_src",
+  "target": "users_info_target",
+  "accuracyMapping": [
+    {
+      "sourceColId": 0,
+      "sourceColName": "user_id",
+      "targetColId": 0,
+      "targetColName": "user_id",
+      "matchFunction": "true",
+      "isPK": true
+    },
+    {
+      "sourceColId": 1,
+      "sourceColName": "first_name",
+      "targetColId": 1,
+      "targetColName": "first_name",
+      "matchFunction": "false",
+      "isPK": false
+    },
+    {
+      "sourceColId": 2,
+      "sourceColName": "last_name",
+      "targetColId": 2,
+      "targetColName": "last_name",
+      "matchFunction": "false",
+      "isPK": false
+    },
+    {
+      "sourceColId": 3,
+      "sourceColName": "address",
+      "targetColId": 3,
+      "targetColName": "address",
+      "matchFunction": "false",
+      "isPK": false
+    },
+    {
+      "sourceColId": 4,
+      "sourceColName": "email",
+      "targetColId": 4,
+      "targetColName": "email",
+      "matchFunction": "false",
+      "isPK": false
+    },
+    {
+      "sourceColId": 5,
+      "sourceColName": "phone",
+      "targetColId": 5,
+      "targetColName": "phone",
+      "matchFunction": "false",
+      "isPK": false
+    },
+    {
+      "sourceColId": 6,
+      "sourceColName": "post_code",
+      "targetColId": 6,
+      "targetColName": "post_code",
+      "matchFunction": "false",
+      "isPK": false
+    }
+  ]
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/jsonFile/vali_config.json
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/jsonFile/vali_config.json b/docker/griffin-base/griffin/jsonFile/vali_config.json
new file mode 100755
index 0000000..80dcf41
--- /dev/null
+++ b/docker/griffin-base/griffin/jsonFile/vali_config.json
@@ -0,0 +1,135 @@
+{
+	"dataSet": "users_info_src",
+	"validityReq": [
+		{
+			"colId": 0,
+			"colName": "user_id",
+			"metrics": [
+				{
+					"name": 5
+				},
+				{
+					"name": 6
+				},
+				{
+					"name": 7
+				},
+				{
+					"name": 8
+				},
+				{
+					"name": 1
+				}
+			]
+		},
+		{
+			"colId": 1,
+			"colName": "first_name",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		},
+		{
+			"colId": 2,
+			"colName": "last_name",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		},
+		{
+			"colId": 3,
+			"colName": "address",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		},
+		{
+			"colId": 4,
+			"colName": "email",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		},
+		{
+			"colId": 5,
+			"colName": "phone",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		},
+		{
+			"colId": 6,
+			"colName": "post_code",
+			"metrics": [
+				{
+					"name": 1
+				},
+				{
+					"name": 2
+				},
+				{
+					"name": 3
+				},
+				{
+					"name": 4
+				}
+			]
+		}
+	]
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/pre-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/pre-start.sh b/docker/griffin-base/griffin/pre-start.sh
new file mode 100755
index 0000000..7ea925d
--- /dev/null
+++ b/docker/griffin-base/griffin/pre-start.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+#sleep 10 seconds
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#create directories
+hadoop fs -mkdir /tmp
+hadoop fs -mkdir /user/hive
+hadoop fs -mkdir /user/hive/warehouse
+hadoop fs -chmod g+w /tmp
+hadoop fs -chmod g+w /user/hive/warehouse
+
+#prepare mongod configure
+mongod -f /etc/mongod.conf

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/start-up.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/start-up.sh b/docker/griffin-base/start-up.sh
new file mode 100755
index 0000000..0aed01a
--- /dev/null
+++ b/docker/griffin-base/start-up.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+: ${TOMCAT_HOME:=/apache/tomcat}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
+
+#spark start
+$SPARK_HOME/sbin/start-all.sh
+
+#start mongodb
+mongod -f /etc/mongod.conf
+
+#mysql service
+service mysqld start
+
+#hive metastore service
+hive --service metastore &
+
+
+/bin/bash -c "bash"

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/adduser.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/adduser.sh b/docker/griffin-base/user/adduser.sh
new file mode 100755
index 0000000..3d6f362
--- /dev/null
+++ b/docker/griffin-base/user/adduser.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ROOT_UID=0
+SUCCESS=0
+E_USEREXISTS=70
+
+# Run as root, of course. (this might not be necessary, because we have to run the script somehow with root anyway)
+if [ "$UID" -ne "$ROOT_UID" ]
+then
+  echo "Must be root to run this script."
+  exit $E_NOTROOT
+fi  
+
+#test, if both argument are there
+if [ $# -eq 2 ]; then
+username=$1
+pass=$2
+
+	# Check if user already exists.
+	grep -q "$username" /etc/passwd
+	if [ $? -eq $SUCCESS ] 
+	then	
+	echo "User $username does already exist."
+  	echo "please chose another username."
+	exit $E_USEREXISTS
+	fi  
+
+
+	useradd -d /home/"$username" -m -g users -s /bin/bash "$username"
+
+	echo "the account is setup"
+
+	echo "$username:$pass" | chpasswd
+
+else
+        echo  " this programm needs 2 arguments you have given $# "
+        echo  " you have to call the script $0 username and the pass "
+fi
+
+exit 0

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/env.txt
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/env.txt b/docker/griffin-base/user/env.txt
new file mode 100644
index 0000000..e6ee39e
--- /dev/null
+++ b/docker/griffin-base/user/env.txt
@@ -0,0 +1,13 @@
+#user define enviroment
+export GRIFFIN_HOME=/home/griffin
+
+export JAVA_HOME=/usr/java/latest
+export HADOOP_HOME=/usr/local/hadoop
+export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+export SPARK_HOME=/usr/local/spark
+export HIVE_HOME=/apache/hive
+export TOMCAT_HOME=/apache/tomcat
+
+export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$HIVE_HOME/bin:$TOMCAT_HOME/bin:$PATH
+
+export HADOOP_USER_CLASSPATH_FIRST=true

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/sudouser.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/sudouser.sh b/docker/griffin-base/user/sudouser.sh
new file mode 100755
index 0000000..3678da9
--- /dev/null
+++ b/docker/griffin-base/user/sudouser.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+if [ $# -eq 1 ]; then
+username=$1
+echo "$username  ALL=(ALL:ALL) ALL" >> /etc/sudoers
+fi

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/griffin/Dockerfile b/docker/griffin/Dockerfile
new file mode 100755
index 0000000..647c138
--- /dev/null
+++ b/docker/griffin/Dockerfile
@@ -0,0 +1,27 @@
+FROM griffin-base-env:latest
+MAINTAINER com.ebay.oss
+
+
+#input files
+ADD script/*.sh $GRIFFIN_HOME/
+ADD ./*.sh $GRIFFIN_HOME/
+RUN mkdir $GRIFFIN_HOME/tmp $GRIFFIN_HOME/temp $GRIFFIN_HOME/log
+RUN chmod -R 755 $GRIFFIN_HOME
+
+#add db
+RUN mkdir /db
+ADD db /db
+
+#remove tomcat webapps
+ENV APACHE_HOME /apache/tomcat
+RUN rm -rf $APACHE_HOME/webapps/*
+
+#prepare data
+WORKDIR $GRIFFIN_HOME
+RUN ./hadoop-start.sh && ./pre-start.sh && ./hadoop-end.sh && rm hadoop-start.sh hadoop-end.sh pre-start.sh
+
+#download resources
+RUN ./download-resources.sh && rm download-resources.sh
+
+#entrypoint
+ENTRYPOINT ["./start-up.sh"]

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/SEQUENCES.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/SEQUENCES.bson b/docker/griffin/db/unitdb0/SEQUENCES.bson
new file mode 100644
index 0000000..5a5cc64
Binary files /dev/null and b/docker/griffin/db/unitdb0/SEQUENCES.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/SEQUENCES.metadata.json b/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
new file mode 100644
index 0000000..0395aac
--- /dev/null
+++ b/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.SEQUENCES"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/data_assets.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/data_assets.bson b/docker/griffin/db/unitdb0/data_assets.bson
new file mode 100644
index 0000000..5e2c65f
Binary files /dev/null and b/docker/griffin/db/unitdb0/data_assets.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/data_assets.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/data_assets.metadata.json b/docker/griffin/db/unitdb0/data_assets.metadata.json
new file mode 100644
index 0000000..9e102a1
--- /dev/null
+++ b/docker/griffin/db/unitdb0/data_assets.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.data_assets"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_job.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_job.bson b/docker/griffin/db/unitdb0/dq_job.bson
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_job.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_job.metadata.json b/docker/griffin/db/unitdb0/dq_job.metadata.json
new file mode 100644
index 0000000..fa3a369
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_job.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_job"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_metrics_values.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_metrics_values.bson b/docker/griffin/db/unitdb0/dq_metrics_values.bson
new file mode 100644
index 0000000..857df0c
Binary files /dev/null and b/docker/griffin/db/unitdb0/dq_metrics_values.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json b/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
new file mode 100644
index 0000000..c259dec
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_metrics_values"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson
new file mode 100644
index 0000000..8294c87
Binary files /dev/null and b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
new file mode 100644
index 0000000..05a5987
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_missed_file_path_lkp"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_model.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_model.bson b/docker/griffin/db/unitdb0/dq_model.bson
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_model.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_model.metadata.json b/docker/griffin/db/unitdb0/dq_model.metadata.json
new file mode 100644
index 0000000..2da029a
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_model.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_model"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_schedule.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_schedule.bson b/docker/griffin/db/unitdb0/dq_schedule.bson
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_schedule.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_schedule.metadata.json b/docker/griffin/db/unitdb0/dq_schedule.metadata.json
new file mode 100644
index 0000000..036d141
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_schedule.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_schedule"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/system.indexes.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/system.indexes.bson b/docker/griffin/db/unitdb0/system.indexes.bson
new file mode 100644
index 0000000..973ade6
Binary files /dev/null and b/docker/griffin/db/unitdb0/system.indexes.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/user_subscribe.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/user_subscribe.bson b/docker/griffin/db/unitdb0/user_subscribe.bson
new file mode 100644
index 0000000..c00ae2b
Binary files /dev/null and b/docker/griffin/db/unitdb0/user_subscribe.bson differ

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/user_subscribe.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/user_subscribe.metadata.json b/docker/griffin/db/unitdb0/user_subscribe.metadata.json
new file mode 100644
index 0000000..9cf671a
--- /dev/null
+++ b/docker/griffin/db/unitdb0/user_subscribe.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.user_subscribe"}]}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/download-resources.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/download-resources.sh b/docker/griffin/download-resources.sh
new file mode 100755
index 0000000..758d0da
--- /dev/null
+++ b/docker/griffin/download-resources.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+
+URI=https://oss.sonatype.org/content/repositories/snapshots/com/ebay/oss
+
+#get xml line
+getval()
+{
+        RESULT=`sed -n 's/.*>\(.*\)<\/'$2'>/\1/p' $1`
+        echo $RESULT | cut -d ' ' -f 1
+}
+
+#get model version jar
+MODEL_DIR=griffin-models
+MODEL_MAVEN_XML=maven-metadata.xml
+wget $URI/$MODEL_DIR/$MODEL_MAVEN_XML
+MODEL_ARTIFACT_ID=`getval $MODEL_MAVEN_XML artifactId`
+MODEL_VERSION=`getval $MODEL_MAVEN_XML version`
+rm $MODEL_MAVEN_XML
+
+MODEL_MAVEN_METADATA_XML=maven-metadata.xml
+MODEL_PATH=$URI/$MODEL_DIR/$MODEL_VERSION
+wget $MODEL_PATH/$MODEL_MAVEN_METADATA_XML
+MODEL_VALUE=`getval $MODEL_MAVEN_METADATA_XML value`
+rm $MODEL_MAVEN_METADATA_XML
+
+MODEL_JAR=${MODEL_ARTIFACT_ID}-${MODEL_VALUE}.jar
+
+
+#get core version war
+CORE_DIR=griffin-core
+CORE_MAVEN_XML=maven-metadata.xml
+wget $URI/$CORE_DIR/$CORE_MAVEN_XML
+CORE_ARTIFACT_ID=`getval $CORE_MAVEN_XML artifactId`
+CORE_VERSION=`getval $CORE_MAVEN_XML version`
+rm $CORE_MAVEN_XML
+
+CORE_MAVEN_METADATA_XML=maven-metadata.xml
+CORE_PATH=$URI/$CORE_DIR/$CORE_VERSION
+wget $CORE_PATH/$CORE_MAVEN_METADATA_XML
+CORE_VALUE=`getval $CORE_MAVEN_METADATA_XML value`
+rm $MODEL_MAVEN_METADATA_XML
+
+CORE_WAR=${CORE_ARTIFACT_ID}-${CORE_VALUE}.war
+
+#download and move
+MODEL_FULL_PATH=$MODEL_PATH/$MODEL_JAR
+CORE_FULL_PATH=$CORE_PATH/$CORE_WAR
+
+MODEL_JAR_NEW=griffin-models.jar
+CORE_WAR_NEW=ROOT.war
+
+MODEL_JAR_NEW_PATH=$GRIFFIN_HOME/$MODEL_JAR_NEW
+CORE_WAR_NEW_PATH=$APACHE_HOME/webapps/$CORE_WAR_NEW
+
+wget $MODEL_FULL_PATH
+wget $CORE_FULL_PATH
+
+mv $MODEL_JAR $MODEL_JAR_NEW_PATH
+mv $CORE_WAR $CORE_WAR_NEW_PATH
+
+chmod 755 $MODEL_JAR_NEW_PATH
+chmod 755 $CORE_WAR_NEW_PATH

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/hadoop-end.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/hadoop-end.sh b/docker/griffin/hadoop-end.sh
new file mode 100755
index 0000000..432bf8f
--- /dev/null
+++ b/docker/griffin/hadoop-end.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/sbin/stop-dfs.sh
+$HADOOP_PREFIX/sbin/stop-yarn.sh
+service sshd stop

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/hadoop-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/hadoop-start.sh b/docker/griffin/hadoop-start.sh
new file mode 100755
index 0000000..8e88406
--- /dev/null
+++ b/docker/griffin/hadoop-start.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+HOSTNAME=sandbox
+echo $(head -1 /etc/hosts | cut -f1) $HOSTNAME >> /etc/hosts
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/pre-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/pre-start.sh b/docker/griffin/pre-start.sh
new file mode 100755
index 0000000..37a51b0
--- /dev/null
+++ b/docker/griffin/pre-start.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+
+#sleep 10 seconds
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#restore mongodb data
+mongod -f /etc/mongod.conf
+mongorestore --db unitdb0 /db/unitdb0
+
+#create griffin env
+hadoop fs -mkdir /user/griffin
+hadoop fs -chmod g+w /user/griffin
+hadoop fs -mkdir /user/griffin/running
+hadoop fs -mkdir /user/griffin/history
+hadoop fs -mkdir /user/griffin/failure
+hadoop fs -chmod g+w /user/griffin/running
+hadoop fs -chmod g+w /user/griffin/history
+hadoop fs -chmod g+w /user/griffin/failure

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/env.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/env.sh b/docker/griffin/script/env.sh
new file mode 100755
index 0000000..84bdd55
--- /dev/null
+++ b/docker/griffin/script/env.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+HDFS_WORKDIR=/user/griffin/running

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/griffin_jobs.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/griffin_jobs.sh b/docker/griffin/script/griffin_jobs.sh
new file mode 100755
index 0000000..0200fec
--- /dev/null
+++ b/docker/griffin/script/griffin_jobs.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+
+ROOT_DIR=$(cd $(dirname $0); pwd)
+if [ -f $ROOT_DIR/env.sh ]; then
+  . $ROOT_DIR/env.sh
+fi
+
+GRIFFIN_HOME=${GRIFFIN_HOME:-$ROOT_DIR}
+
+HDFS_WORKDIR=${HDFS_WORKDIR:-/user/griffin/running}
+TEMP_DIR=${TEMP_DIR:-$ROOT_DIR/temp}
+LOG_DIR=${LOG_DIR:-$ROOT_DIR/log}
+
+mkdir -p $TEMP_DIR
+mkdir -p $LOG_DIR
+
+lv1tempfile=$TEMP_DIR/temp.txt
+lv2tempfile=$TEMP_DIR/temp2.txt
+
+set +e
+
+hadoop fs -ls $HDFS_WORKDIR > $lv1tempfile
+
+while read line
+do
+  lv1dir=${line##* }
+  jobid=${lv1dir##*/}
+  ts=`date +%Y%m%d%H%M%S`
+  echo $ts
+  logfile=$LOG_DIR/${jobid}_${ts}.dqjoblog
+  rm -rf $logfile
+  touch $logfile
+
+  hadoop fs -test -f $lv1dir/_START
+  rc=$?
+  echo "$rc $lv1dir/_START" >> $logfile
+  if [ $rc -ne 0 ] && [ "${lv1dir:0:1}" == "/" ]
+  then
+    lv2tempfile=${LOG_DIR}/${jobid}_watch
+    rm -rf $lv2tempfile
+    hadoop fs -get $lv1dir/_watchfile $lv2tempfile
+    cat $lv2tempfile >> $logfile
+
+    watchfiledone=1
+    hascontent=0
+    while read watchline
+    do
+      hascontent=1
+      hadoop fs -test -f $watchline/_SUCCESS
+      rcode1=$?
+      hadoop fs -test -f $watchline/*done
+      rcode2=$?
+      echo "$rcode1 $watchline/_SUCCESS  $rcode2 $watchline/*done" >> $logfile
+      if [ $rcode1 -ne 0 ] && [ $rcode2 -ne 0 ]
+      then
+        watchfiledone=0
+        break
+      fi
+    done < $lv2tempfile
+
+    if [ $watchfiledone -eq 1 ] && [ $hascontent -eq 1 ]
+    then
+      hadoop fs -touchz $lv1dir/_START
+      hadoop fs -test -f $lv1dir/_type_0.done
+      rc1=$?
+      echo "$rc1 $lv1dir/_type_0.done" >> $logfile
+      hadoop fs -test -f $lv1dir/_type_1.done
+      rc2=$?
+      echo "$rc2 $lv1dir/_type_1.done" >> $logfile
+      if [ $rc1 -eq 0 ]
+      then
+        echo "spark-submit --class org.apache.griffin.accuracy.Accu --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar  $lv1dir/cmd.txt $lv1dir/ " >> $logfile
+        spark-submit --class org.apache.griffin.accuracy.Accu --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar  $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
+      elif [ $rc2 -eq 0 ]
+      then
+        echo "spark-submit --class org.apache.griffin.validility.Vali --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar  $lv1dir/cmd.txt $lv1dir/ " >> $logfile
+        spark-submit --class org.apache.griffin.validility.Vali --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar  $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
+      fi
+
+      echo "done" >> $logfile
+      exit
+    fi
+  fi
+
+done < $lv1tempfile
+
+set -e

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/griffin_regular_run.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/griffin_regular_run.sh b/docker/griffin/script/griffin_regular_run.sh
new file mode 100755
index 0000000..35646c2
--- /dev/null
+++ b/docker/griffin/script/griffin_regular_run.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+ROOT_DIR=$(cd $(dirname $0); pwd)
+
+set +e
+while true
+do
+  echo "start"
+  $ROOT_DIR/griffin_jobs.sh 2>&1
+  rcode=$?
+  echo "end $rcode"
+  rm -rf $ROOT_DIR/nohup.out
+  sleep 60
+done
+set -e

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/start-up.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/start-up.sh b/docker/griffin/start-up.sh
new file mode 100755
index 0000000..164da7b
--- /dev/null
+++ b/docker/griffin/start-up.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+: ${TOMCAT_HOME:=/apache/tomcat}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
+
+#spark start
+$SPARK_HOME/sbin/start-all.sh
+
+#start mongodb
+mongod -f /etc/mongod.conf
+
+#mysql service
+service mysqld start
+
+#hive metastore service
+hive --service metastore &
+
+echo "hive metastore service"
+
+#start script
+nohup ./griffin_regular_run.sh &
+
+echo "bark regular run"
+
+
+CMD=${1:-"log"}
+if [[ "$CMD" == "bash" ]];
+then
+	#start tomcat
+	/etc/init.d/tomcat start
+	/bin/bash -c "$*"
+else
+	${TOMCAT_HOME}/bin/catalina.sh run
+fi

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/.gitignore
----------------------------------------------------------------------
diff --git a/griffin-core/.gitignore b/griffin-core/.gitignore
new file mode 100644
index 0000000..c13c943
--- /dev/null
+++ b/griffin-core/.gitignore
@@ -0,0 +1,27 @@
+dist
+build
+bin
+rebel.xml
+MANIFEST.MF
+/tests/
+/coverage/
+/cobertura.ser
+*/ebay.log
+*.class
+*.jar
+*.zip
+*/target/*
+*/temp_persist_config_*.xml
+*/src/main/resources/buildinfo.properties
+/.idea/
+**/*.iml
+build.xml
+ebay-build.xml
+4cc.checksum
+/.metadata
+/.project
+/target/**
+/.settings/**
+/target/
+/.externalToolBuilders/
+/.classpath

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/pom.xml
----------------------------------------------------------------------
diff --git a/griffin-core/pom.xml b/griffin-core/pom.xml
new file mode 100644
index 0000000..752a8fd
--- /dev/null
+++ b/griffin-core/pom.xml
@@ -0,0 +1,302 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache
+	License, Version 2.0 (the "License"); you may not use this file except in
+	compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+	Unless required by applicable law or agreed to in writing, software distributed
+	under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+	OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+	the specific language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+	<parent>
+		<groupId>com.ebay.oss</groupId>
+		<artifactId>griffin-parent</artifactId>
+		<version>0.1.0-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+
+
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>griffin-core</artifactId>
+	<name>griffin-core</name>
+	<packaging>war</packaging>
+
+
+	<dependencies>
+
+		<!-- <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId>
+			<version>2.4</version> </dependency> -->
+
+		<dependency>
+			<groupId>javax.servlet</groupId>
+			<artifactId>javax.servlet-api</artifactId>
+			<version>3.0.1</version>
+			<scope>provided</scope>
+		</dependency>
+
+
+
+		<dependency>
+			<groupId>com.google.code.gson</groupId>
+			<artifactId>gson</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.hibernate</groupId>
+			<artifactId>hibernate-validator</artifactId>
+			<version>5.2.4.Final</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+
+		<!-- Jersey -->
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-server</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-json</artifactId>
+
+		</dependency>
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-bundle</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>org.aspectj</groupId>
+			<artifactId>aspectjweaver</artifactId>
+			<version>1.6.10</version>
+		</dependency>
+
+		<!-- Spring 3 dependencies -->
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+			<version>3.2.8.RELEASE</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+			<version>3.2.8.RELEASE</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-webmvc</artifactId>
+			<version>3.2.8.RELEASE</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-common</artifactId>
+
+		</dependency>
+
+
+
+		<!-- <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> -->
+		<!-- </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId>
+			</dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> -->
+
+		<!-- </dependency> -->
+
+		<!-- Jersey + Spring -->
+		<dependency>
+			<groupId>com.sun.jersey.contribs</groupId>
+			<artifactId>jersey-spring</artifactId>
+			<version>${jersey.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-core</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-web</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-webmvc</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-beans</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-context</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>spring-aop</artifactId>
+				</exclusion>
+
+			</exclusions>
+		</dependency>
+
+		<dependency>
+			<groupId>org.mongodb</groupId>
+			<artifactId>mongo-java-driver</artifactId>
+			<version>${mongo.version}</version>
+		</dependency>
+
+		<!-- <dependency> <groupId>javax.ws.rs</groupId> <artifactId>javax.ws.rs-api</artifactId>
+			<version>2.0</version> </dependency> -->
+
+		<dependency>
+			<groupId>com.google.code.morphia</groupId>
+			<artifactId>morphia</artifactId>
+			<version>0.104</version>
+		</dependency>
+		<!-- <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId>
+			</dependency> <dependency> <groupId>commons-digester</groupId> <artifactId>commons-digester</artifactId>
+			<version>2.1</version> </dependency> <dependency> <groupId>org.aspectj</groupId>
+			<artifactId>aspectjrt</artifactId> <version>1.8.1</version> </dependency> -->
+
+		<!-- <dependency>
+			<groupId>org.apache.griffin</groupId>
+			<artifactId>griffin-scheduler</artifactId>
+			<version>${griffin.version}</version>
+			<scope>compile</scope>
+		</dependency> -->
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>griffin-ui</artifactId>
+			<version>${griffin.version}</version>
+			<classifier>resources</classifier>
+			<type>zip</type>
+			<!-- Make sure this isn't included on any classpath -->
+			<scope>provided</scope>
+		</dependency>
+		<!-- <dependency> <groupId>org.apache.griffin</groupId> <artifactId>griffin-core</artifactId>
+			<version>0.0.1-SNAPSHOT</version> <scope>compile</scope> </dependency> -->
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+        </dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-test</artifactId>
+			<version>3.2.8.RELEASE</version>
+			<scope>test</scope>
+		</dependency>
+
+		<!-- Guice =============================================================
+			<dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId>
+			<version>${guice.version}</version> <scope>test</scope> </dependency> <dependency>
+			<groupId>com.google.inject.extensions</groupId> <artifactId>guice-servlet</artifactId>
+			<version>${guice.version}</version> <scope>test</scope> </dependency> <dependency>
+			<groupId>com.sun.jersey.contribs</groupId> <artifactId>jersey-guice</artifactId>
+			<version>${jersey.version}</version> <scope>test</scope> </dependency> -->
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-client</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>com.sun.jersey</groupId>
+			<artifactId>jersey-grizzly2</artifactId>
+			<version>${jersey.version}</version>
+			<scope>test</scope>
+		</dependency>
+
+
+	</dependencies>
+
+	<build>
+		<finalName>ROOT</finalName>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<configuration>
+					<source>1.7</source>
+					<target>1.7</target>
+				</configuration>
+			</plugin>
+			<!-- <plugin>
+				<artifactId>maven-war-plugin</artifactId>
+				<version>2.1.1</version>
+				<configuration>
+					<attachClasses>true</attachClasses>
+					<classesClassifier>classes</classesClassifier>
+				</configuration>
+			</plugin> -->
+
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>cobertura-maven-plugin</artifactId>
+				<version>2.6</version>
+				<configuration>
+					<formats>
+						<format>html</format>
+						<format>xml</format>
+					</formats>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-war-plugin</artifactId>
+				<version>2.6</version>
+				<configuration>
+					<webResources>
+						<resource>
+							<directory>${project.build.directory}/generated-resources/griffin-ui-${griffin.version}</directory>
+						</resource>
+					</webResources>
+					<attachClasses>true</attachClasses>
+					<classesClassifier>classes</classesClassifier>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-dependency-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>unpack-shared-resources</id>
+						<goals>
+							<goal>unpack-dependencies</goal>
+						</goals>
+						<phase>generate-resources</phase>
+						<configuration>
+							<outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
+							<includeArtifacIds>griffin-ui</includeArtifacIds>
+							<includeGroupIds>${project.groupId}</includeGroupIds>
+							<overWriteReleases>true</overWriteReleases>
+							<overWriteSnapshots>true</overWriteSnapshots>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<configuration>
+					<skipTests>true</skipTests>
+				</configuration>
+			</plugin>
+
+		</plugins>
+
+
+	</build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
new file mode 100644
index 0000000..3b090ec
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
@@ -0,0 +1,33 @@
+package com.ebay.oss.griffin.common;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HDFSUtils {
+
+	static Logger logger = LoggerFactory.getLogger(HDFSUtils.class);
+
+	public static boolean checkHDFSFolder(String folderPath) {
+		
+		Process processMoveFolder;
+		int result;
+		try {
+			processMoveFolder = Runtime.getRuntime().exec("hadoop fs -ls " + folderPath);
+			
+			result = processMoveFolder.waitFor();
+			
+			if(result == 0) 
+			{
+				return true;
+			}
+			
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			return false;
+		}
+		return false;
+	}
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
new file mode 100644
index 0000000..ae051f1
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
@@ -0,0 +1,41 @@
+package com.ebay.oss.griffin.common;
+
+import org.springframework.util.StringUtils;
+
+public class NumberUtils {
+
+    public static int parseInt(Object o) {
+        if (o == null) {
+            return -1;
+        }
+        
+        String s = o.toString();
+        if(StringUtils.isEmpty(s) ) {
+            return -1;
+        }
+        
+        try {
+            return Integer.parseInt(s);
+        } catch (Exception e) {
+            return -1;
+        }
+    }
+
+    public static long parseLong(Object o) {
+        if (o == null) {
+            return -1;
+        }
+        
+        String s = o.toString();
+        if(StringUtils.isEmpty(s) ) {
+            return -1;
+        }
+        
+        try {
+            return Long.parseLong(s);
+        } catch (Exception e) {
+            return -1;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
new file mode 100644
index 0000000..f8e3386
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
@@ -0,0 +1,64 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+
+public class Pair {
+
+	public final String key;
+
+	public final Object value;
+
+	public Pair(String key, Object value) {
+		this.key = key;
+		this.value = value;
+	}
+
+	@Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((key == null) ? 0 : key.hashCode());
+        result = prime * result + ((value == null) ? 0 : value.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj)
+            return true;
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        Pair other = (Pair) obj;
+        if (key == null) {
+            if (other.key != null)
+                return false;
+        } else if (!key.equals(other.key))
+            return false;
+        if (value == null) {
+            if (other.value != null)
+                return false;
+        } else if (!value.equals(other.value))
+            return false;
+        return true;
+    }
+
+
+    @Override
+	public String toString() {
+		return "Pair [key=" + key + ", value=" + value + "]";
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
new file mode 100644
index 0000000..14899eb
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
@@ -0,0 +1,21 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+
+public class ScheduleModelSeperator {
+	public static final String SEPERATOR = "-=-";
+	public static final String SPLIT_SEPERATOR = "-=-";
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
new file mode 100644
index 0000000..924c530
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
@@ -0,0 +1,45 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+
+public class SimpleCORSFilter implements Filter {
+
+	@Override
+	public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException {
+		HttpServletResponse response = (HttpServletResponse) res;
+		response.setHeader("Access-Control-Allow-Origin", "*");
+		response.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE,PUT");
+		response.setHeader("Access-Control-Max-Age", "3600");
+		response.setHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept");
+		chain.doFilter(req, res);
+	}
+
+	@Override
+	public void init(FilterConfig filterConfig) {}
+
+	@Override
+	public void destroy() {}
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
new file mode 100644
index 0000000..4c2bcf9
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
@@ -0,0 +1,26 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+/** the enumerable type of Anomaly Detection */
+public class AnomalyType {
+
+	public static final int ANOMALY_DEFAULT = 0;
+	public static final int HISTORY_TREND = 1;
+	public static final int BOLLINGER_BANDS = 2;
+	public static final int MAD = 3;
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
new file mode 100644
index 0000000..3b92040
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
@@ -0,0 +1,199 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import com.google.code.morphia.annotations.Embedded;
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Property;
+import com.mongodb.DBObject;
+
+/**
+ * A DataAsset is an abstraction of a data source, from which a same type of data comes. A DataAsset
+ * could be a 'table' in RDB or a topic/feed from a stream.
+ */
+@Entity("data_assets")
+// unique constraint: asssetName + assetType + system
+public class DataAsset extends IdEntity{
+
+    // HDP the only one
+	@Property("platform")
+	private String platform;
+
+	// SystemTypeConstants
+	@Property("system")
+	private String system;
+
+	// e.g. viewitem
+	@Property("assetName")
+	private String assetName;
+
+	// HiveTable the only one
+	@Property("assetType")
+	private String assetType;
+
+	@Property("assetHDFSPath")
+	private String assetHDFSPath;
+
+	@Property("owner")
+	private String owner;
+
+	// createdDate
+	@Property("timestamp")
+	private Date timestamp;
+
+
+	// home-made schema, DataSchema is atually a simple schemaItem, a schema consists of a list of 
+	// schemaItem 
+	@Embedded
+	private List<DataSchema> schema = new ArrayList<>();
+
+	// hive table partition format
+	@Embedded
+	private List<PartitionFormat> partitions;
+
+	public DataAsset() { }
+
+	// FIXME should be somewhere else, ORM
+	@SuppressWarnings({"unchecked", "deprecation"})
+    public DataAsset(DBObject o) {
+		this.set_id(Long.parseLong(o.get("_id").toString()));
+		this.setAssetHDFSPath((String)o.get("assetHDFSPath"));
+		this.setAssetName((String)o.get("assetName"));
+		this.setAssetType((String)o.get("assetType"));
+
+		this.setOwner((String)o.get("owner"));
+		this.setPlatform((String)o.get("platform"));
+		this.setSystem((String)o.get("system"));
+
+//		this.setPartitions((List<PartitionFormat>) o.get("partitions")); // this doesn't work
+	    if(o.get("partitions")!=null) {
+	        List<PartitionFormat> partitionlist = new ArrayList<PartitionFormat>();
+	        List<DBObject> tlist = (List<DBObject>) o.get("partitions");
+	        for(DBObject temp : tlist) {
+	            partitionlist.add(new PartitionFormat(temp.get("name").toString(), temp.get("format").toString()));
+	        }
+	        this.setPartitions(partitionlist);
+	    }
+
+//		this.setSchema((List<DataSchema>) o.get("schema"));
+		if(o.get("schema")!=null) {
+		    List<DBObject> tlist = (List<DBObject>) o.get("schema");
+		    List<DataSchema> list = new ArrayList<DataSchema>();
+		    for(DBObject temp : tlist) {
+		        list.add(new DataSchema(temp.get("name").toString(), temp.get("type").toString(), temp.get("desc").toString(), temp.get("sample").toString()));
+		    }
+		    this.setSchema(list);
+		}
+
+        if (!o.containsField("timestamp")) {
+            this.setTimestamp(new Date());
+        } else {
+            this.setTimestamp(new Date(o.get("timestamp").toString()));
+        }
+	}
+	
+	{
+	    
+	    
+	}
+
+	public String getPlatform() {
+		return platform;
+	}
+
+	public void setPlatform(String platform) {
+		this.platform = platform;
+	}
+
+	public String getSystem() {
+		return system;
+	}
+
+	public void setSystem(String system) {
+		this.system = system;
+	}
+
+	public String getAssetName() {
+		return assetName;
+	}
+
+	public void setAssetName(String assetName) {
+		this.assetName = assetName;
+	}
+
+	public String getAssetType() {
+		return assetType;
+	}
+
+	public void setAssetType(String assetType) {
+		this.assetType = assetType;
+	}
+
+	public String getAssetHDFSPath() {
+		return assetHDFSPath;
+	}
+
+	public void setAssetHDFSPath(String assetHDFSPath) {
+		this.assetHDFSPath = assetHDFSPath;
+	}
+
+	public String getOwner() {
+		return owner;
+	}
+
+	public void setOwner(String owner) {
+		this.owner = owner;
+	}
+
+	public Date getTimestamp() {
+		return timestamp;
+	}
+
+	public void setTimestamp(Date timestamp) {
+		this.timestamp = timestamp;
+	}
+
+	public List<DataSchema> getSchema() {
+		return schema;
+	}
+
+	public void setSchema(List<DataSchema> schema) {
+		this.schema = schema;
+	}
+
+	public List<PartitionFormat> getPartitions() {
+		return partitions;
+	}
+
+	public void setPartitions(List<PartitionFormat> partitions) {
+		this.partitions = partitions;
+	}
+
+    public int getColId(String colName) {
+        if(schema == null || schema.isEmpty()) {
+            return -1;
+        }
+
+        for (int i = 0; i < schema.size(); i++) {
+            if (colName.equals(schema.get(i).getName()))
+                return i;
+        }
+        return -1;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
new file mode 100644
index 0000000..6f92a5d
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
@@ -0,0 +1,80 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Embedded;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * this is a simple representation of a schema system, which describe the data type of a table.
+ */
+@Embedded
+public class DataSchema{
+
+	@Property("name")
+	private String name;
+
+	@Property("type")
+	private String type;
+
+	@Property("desc")
+	private String desc;
+
+	@Property("sample")
+	private String sample;
+
+	public DataSchema() {
+	}
+
+	public DataSchema(String name, String type, String desc, String sample) {
+		this.name = name;
+		this.type = type;
+		this.desc = desc;
+		this.sample = sample;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public String getDesc() {
+		return desc;
+	}
+
+	public void setDesc(String desc) {
+		this.desc = desc;
+	}
+
+	public String getSample() {
+		return sample;
+	}
+
+	public void setSample(String sample) {
+		this.sample = sample;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
new file mode 100644
index 0000000..9a10e8c
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
@@ -0,0 +1,121 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Id;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * A DqJob is an instance of DqSchedule, the frequency/multiplicity is based on ScheduleType.
+ */
+@Entity("dq_job")
+public class DqJob{
+
+    /** {modelName} + "_" + "yyyy-mm-dd xx:00:00 000".getTime() */
+	@Id
+	private String _id;
+
+	// Inherits from DqSchedule.modelList
+	@Property("modelList")
+	private String modelList;
+
+	/** Inherits from DqSchedule#jobType.*/
+	@Property("jobType")
+	private int jobType;
+
+	/** @see JobStatus */
+	@Property("status")
+	private int status;
+
+	@Property("starttime")
+	private long starttime;
+
+	@Property("content")
+	private String content;
+
+	@Property("endtime")
+	private long endtime;
+
+	@Property("value")
+	private long value;
+
+	public String getId() {
+		return _id;
+	}
+
+	public void setId(String _id) {
+		this._id = _id;
+	}
+
+	public String getModelList() {
+		return modelList;
+	}
+
+	public void setModelList(String modelList) {
+		this.modelList = modelList;
+	}
+
+	public int getStatus() {
+		return status;
+	}
+
+	public void setStatus(int status) {
+		this.status = status;
+	}
+
+	public long getStarttime() {
+		return starttime;
+	}
+
+	public void setStarttime(long starttime) {
+		this.starttime = starttime;
+	}
+
+	public String getContent() {
+		return content;
+	}
+
+	public void setContent(String content) {
+		this.content = content;
+	}
+
+	public long getEndtime() {
+		return endtime;
+	}
+
+	public void setEndtime(long endtime) {
+		this.endtime = endtime;
+	}
+
+	public long getValue() {
+		return value;
+	}
+
+	public void setValue(long value) {
+		this.value = value;
+	}
+
+	public int getJobType() {
+		return jobType;
+	}
+
+	public void setJobType(int jobType) {
+		this.jobType = jobType;
+	}
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
new file mode 100644
index 0000000..5b9aac2
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
@@ -0,0 +1,111 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+
+import com.ebay.oss.griffin.vo.BaseObj;
+import com.google.code.morphia.annotations.Id;
+
+
+/**
+ * Metrics is the value of a DQ perspective (DqModel) execution/calculation.
+ * 
+ * TODO: should introduce UnitOfMeasure in, so that a Metrics could be render independently.
+ */
+// uniq{modelName, timestamp}
+public class DqMetricsValue extends BaseObj implements Comparable<DqMetricsValue> {
+
+    @Id
+	private Long _id;
+
+	// model.name
+	@NotNull
+	@Pattern(regexp="\\A([0-9a-zA-Z\\_\\-\\.])+$")
+	private String metricName;
+
+	private String assetId;
+
+	@Min(0)
+	private long timestamp;
+
+	@Min(0)
+	private float value;
+
+	public DqMetricsValue() { }
+
+	public DqMetricsValue(String name, long timestamp, float value) {
+		this.metricName = name;
+		this.timestamp = timestamp;
+		this.value = value;
+	}
+
+    public Long get_id() {
+		return _id;
+	}
+
+	public void set_id(Long _id) {
+		this._id = _id;
+	}
+
+	public String getMetricName() {
+		return metricName;
+	}
+
+	public void setMetricName(String metricName) {
+		this.metricName = metricName;
+	}
+
+	public String getAssetId() {
+		return assetId;
+	}
+
+	public void setAssetId(String assetId) {
+		this.assetId = assetId;
+	}
+
+	public long getTimestamp() {
+		return timestamp;
+	}
+
+	public void setTimestamp(long timestamp) {
+		this.timestamp = timestamp;
+	}
+
+	public float getValue() {
+		return value;
+	}
+
+	public void setValue(float value) {
+		this.value = value;
+	}
+
+	@Override
+	public int compareTo(DqMetricsValue o) {
+	    return (int) Math.signum(o.getTimestamp() - this.getTimestamp());
+//		return o.getTimestamp() == this.getTimestamp() ? 0 :
+//			(o.getTimestamp() > this.getTimestamp() ? 1 : -1);
+	}
+
+    @Override
+    public String toString() {
+        return "DqMetricsValue [_id=" + _id + ", metricName=" + metricName + ", timestamp="
+                        + timestamp + ", value=" + value + "]";
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
new file mode 100644
index 0000000..5347af7
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
@@ -0,0 +1,231 @@
+/*
+	Copyright (c) 2016 eBay Software Foundation.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	    http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * It is a specification of Data Quality perspective, which defines: 
+ * <ol>
+ * <li> what data against, DataAsset(s) </li>
+ * <li> which type of DQ, ModelType </li>
+ * <li> related meta data based on the concrete modelType, modelContent </li>
+ * </ul>
+ */
+@Entity("dq_model")
+public class DqModel extends IdEntity{
+
+    // same as modelName
+	@Property("modelId")
+	private String modelId;
+
+	@Property("modelName")
+	private String modelName;
+
+	@Property("modelType")
+	private int modelType;
+
+	@Property("modelDesc")
+	private String modelDesc;
+
+	@Property("assetId")
+	private long assetId;
+
+	@Property("assetName")
+	private String assetName;
+
+	@Property("threshold")
+	private float threshold;
+
+	@Property("notificationEmail")
+	private String notificationEmail;
+
+	@Property("owner")
+	private String owner;
+
+	@Property("status")
+	private int status;
+
+	// @see ScheduleType
+	@Property("schedule")
+	private int schedule;
+
+	/**
+	 * <ul>
+	 * <li>ModelType.ACCURACY</li>
+	 * <pre>
+	 * srcDb | srcDataSet | dstDB | dstDataSet | [MappingItemInput.src, MII.dst, MII.isPk, MII.matchedMethod];
+	 * </pre>
+	 * <li>ModelType.VALIDITY</li>
+	 * <pre> srcDb | srcDataSet | validityType | column </pre>
+	 * <li>ModelType.Anomaly</li>  
+	 * <pre> srcDb | srcDataSet | anomalType </pre>
+	 * <li>ModelType.Publish</li>
+	 * <pre> publishUrl </pre>
+	 */
+	// FIXME should model this explicitly, ASAP. 
+	@Property("modelContent")
+	private String modelContent;
+
+	// @See SystemType
+	@Property("system")
+	private int system;
+
+	/**
+	 * for a AnormalyDetection model need a reference model, say a count_model. So a count mound will
+	 * hold a reference to Anomaly model.
+	 */
+	@Property("referenceModel")
+	private String referenceModel;
+
+	@Property("timestamp")
+	private long timestamp;
+
+	@Property("starttime")
+	private long starttime;
+
+	public String getModelId() {
+		return modelId;
+	}
+
+	public void setModelId(String modelId) {
+		this.modelId = modelId;
+	}
+
+	public String getModelName() {
+		return modelName;
+	}
+
+	public void setModelName(String modelName) {
+		this.modelName = modelName;
+	}
+
+	public int getModelType() {
+		return modelType;
+	}
+
+	public void setModelType(int modelType) {
+		this.modelType = modelType;
+	}
+
+	public String getModelDesc() {
+		return modelDesc;
+	}
+
+	public void setModelDesc(String modelDesc) {
+		this.modelDesc = modelDesc;
+	}
+
+	public long getAssetId() {
+		return assetId;
+	}
+
+	public void setAssetId(long assetId) {
+		this.assetId = assetId;
+	}
+
+	public float getThreshold() {
+		return threshold;
+	}
+
+	public void setThreshold(float threshold) {
+		this.threshold = threshold;
+	}
+
+	public String getNotificationEmail() {
+		return notificationEmail;
+	}
+
+	public void setNotificationEmail(String notificationEmail) {
+		this.notificationEmail = notificationEmail;
+	}
+
+	public String getOwner() {
+		return owner;
+	}
+
+	public void setOwner(String owner) {
+		this.owner = owner;
+	}
+
+	public int getStatus() {
+		return status;
+	}
+
+	public void setStatus(int status) {
+		this.status = status;
+	}
+
+	public String getModelContent() {
+		return modelContent;
+	}
+
+	public void setModelContent(String modelContent) {
+		this.modelContent = modelContent;
+	}
+
+	public long  getTimestamp() {
+		return timestamp;
+	}
+
+	public void setTimestamp(long  timestamp) {
+		this.timestamp = timestamp;
+	}
+
+	public int getSchedule() {
+		return schedule;
+	}
+
+	public void setSchedule(int schedule) {
+		this.schedule = schedule;
+	}
+
+	public int getSystem() {
+		return system;
+	}
+
+	public void setSystem(int system) {
+		this.system = system;
+	}
+
+	public String getAssetName() {
+		return assetName;
+	}
+
+	public void setAssetName(String assetName) {
+		this.assetName = assetName;
+	}
+
+	public String getReferenceModel() {
+		return referenceModel;
+	}
+
+	public void setReferenceModel(String referenceModel) {
+		this.referenceModel = referenceModel;
+	}
+
+	public long getStarttime() {
+		return starttime;
+	}
+
+	public void setStarttime(long starttime) {
+		this.starttime = starttime;
+	}
+
+
+
+
+}