You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/03/03 12:16:55 UTC
[49/51] [partial] incubator-griffin git commit: Init move
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mongodb-org-3.2.repo
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mongodb-org-3.2.repo b/docker/griffin-base/config/mongodb-org-3.2.repo
new file mode 100755
index 0000000..da73c93
--- /dev/null
+++ b/docker/griffin-base/config/mongodb-org-3.2.repo
@@ -0,0 +1,5 @@
+[mongodb-org-3.2]
+name=MongoDB Repository
+baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.2/x86_64/
+gpgcheck=0
+enabled=1
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_init.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_init.sh b/docker/griffin-base/config/mysql_init.sh
new file mode 100755
index 0000000..de68ffa
--- /dev/null
+++ b/docker/griffin-base/config/mysql_init.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+sed -i.bak s/^.*"hive-txn-schema-0.13.0.mysql.sql".*/"SOURCE \/apache\/hive\/scripts\/metastore\/upgrade\/mysql\/hive-txn-schema-0.13.0.mysql.sql;"/ /apache/hive/scripts/metastore/upgrade/mysql/hive-schema-1.2.0.mysql.sql
+
+mysql -u root -p123456 < $GRIFFIN_HOME/mysql_init_metastore.sql
+
+rm $GRIFFIN_HOME/mysql_init_metastore.sql
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_init_metastore.sql
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_init_metastore.sql b/docker/griffin-base/config/mysql_init_metastore.sql
new file mode 100755
index 0000000..3477a06
--- /dev/null
+++ b/docker/griffin-base/config/mysql_init_metastore.sql
@@ -0,0 +1,9 @@
+CREATE DATABASE metastore;
+USE metastore;
+SOURCE /apache/hive/scripts/metastore/upgrade/mysql/hive-schema-1.2.0.mysql.sql;
+
+
+CREATE USER 'hive'@'localhost' IDENTIFIED BY '123456';
+REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost';
+GRANT SELECT,INSERT,UPDATE,DELETE,LOCK TABLES,EXECUTE ON metastore.* TO 'hive'@'localhost';
+FLUSH PRIVILEGES;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/mysql_secure.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/mysql_secure.sh b/docker/griffin-base/config/mysql_secure.sh
new file mode 100755
index 0000000..aebcd75
--- /dev/null
+++ b/docker/griffin-base/config/mysql_secure.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+
+#
+# Automate mysql secure installation for debian-baed systems
+#
+# - You can set a password for root accounts.
+# - You can remove root accounts that are accessible from outside the local host.
+# - You can remove anonymous-user accounts.
+# - You can remove the test database (which by default can be accessed by all users, even anonymous users),
+# and privileges that permit anyone to access databases with names that start with test_.
+# For details see documentation: http://dev.mysql.com/doc/refman/5.7/en/mysql-secure-installation.html
+#
+# @version 13.08.2014 00:39 +03:00
+# Tested on Debian 7.6 (wheezy)
+#
+# Usage:
+# Setup mysql root password: ./mysql_secure.sh 'your_new_root_password'
+# Change mysql root password: ./mysql_secure.sh 'your_old_root_password' 'your_new_root_password'"
+#
+
+# Delete package expect when script is done
+# 0 - No;
+# 1 - Yes.
+PURGE_EXPECT_WHEN_DONE=0
+
+#
+# Check the bash shell script is being run by root
+#
+if [[ $EUID -ne 0 ]]; then
+ echo "This script must be run as root" 1>&2
+ exit 1
+fi
+
+
+#
+# Check input params
+#
+if [ -n "${1}" -a -z "${2}" ]; then
+ # Setup root password
+ CURRENT_MYSQL_PASSWORD=''
+ NEW_MYSQL_PASSWORD="${1}"
+elif [ -n "${1}" -a -n "${2}" ]; then
+ # Change existens root password
+ CURRENT_MYSQL_PASSWORD="${1}"
+ NEW_MYSQL_PASSWORD="${2}"
+else
+ echo "Usage:"
+ echo " Setup mysql root password: ${0} 'your_new_root_password'"
+ echo " Change mysql root password: ${0} 'your_old_root_password' 'your_new_root_password'"
+ exit 1
+fi
+
+#
+# Check is expect package installed
+#
+if [ $(dpkg-query -W -f='${Status}' expect 2>/dev/null | grep -c "ok installed") -eq 0 ]; then
+ echo "Can't find expect. Trying install it..."
+ yum -y install expect
+
+fi
+
+SECURE_MYSQL=$(expect -c "
+set timeout 3
+spawn mysql_secure_installation
+expect \"Enter current password for root (enter for none):\"
+send \"$CURRENT_MYSQL_PASSWORD\r\"
+expect \"root password?\"
+send \"y\r\"
+expect \"New password:\"
+send \"$NEW_MYSQL_PASSWORD\r\"
+expect \"Re-enter new password:\"
+send \"$NEW_MYSQL_PASSWORD\r\"
+expect \"Remove anonymous users?\"
+send \"y\r\"
+expect \"Disallow root login remotely?\"
+send \"n\r\"
+expect \"Remove test database and access to it?\"
+send \"y\r\"
+expect \"Reload privilege tables now?\"
+send \"y\r\"
+expect eof
+")
+
+#
+# Execution mysql_secure_installation
+#
+echo "${SECURE_MYSQL}"
+
+if [ "${PURGE_EXPECT_WHEN_DONE}" -eq 1 ]; then
+ # Uninstalling expect package
+ yum -y purge expect
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/spark-conf-mod.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/spark-conf-mod.sh b/docker/griffin-base/config/spark-conf-mod.sh
new file mode 100755
index 0000000..f309f5e
--- /dev/null
+++ b/docker/griffin-base/config/spark-conf-mod.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+
+cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
+echo "SPARK_MASTER_WEBUI_PORT=8082" >> $SPARK_HOME/conf/spark-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/config/tomcat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/config/tomcat b/docker/griffin-base/config/tomcat
new file mode 100755
index 0000000..e195bd9
--- /dev/null
+++ b/docker/griffin-base/config/tomcat
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# description: Tomcat Start Stop Restart
+# processname: tomcat
+# chkconfig: 234 20 80
+JAVA_HOME=/usr/java/latest
+export JAVA_HOME
+PATH=$JAVA_HOME/bin:$PATH
+export PATH
+JAVA_OPTS="-Dfile.encoding=UTF-8 -Xms128m -Xmx1024m -XX:PermSize=128m -XX:MaxPermSize=512m"
+export JAVA_OPTS
+echo $JAVA_OPTS
+CATALINA_HOME=/apache/tomcat
+RETVAL=$?
+case "$1" in
+start)
+$CATALINA_HOME/bin/startup.sh
+;;
+restart)
+$CATALINA_HOME/bin/shutdown.sh
+$CATALINA_HOME/bin/startup.sh
+;;
+stop)
+$CATALINA_HOME/bin/shutdown.sh
+esac
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/dataFile/users_info_src.dat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/dataFile/users_info_src.dat b/docker/griffin-base/griffin/dataFile/users_info_src.dat
new file mode 100755
index 0000000..ce49443
--- /dev/null
+++ b/docker/griffin-base/griffin/dataFile/users_info_src.dat
@@ -0,0 +1,50 @@
+10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000001|94022
+10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000002|94022
+10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
+10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
+10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
+10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
+10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
+10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
+10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
+10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
+10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
+10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
+10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
+10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
+10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
+10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
+10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
+10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
+10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
+10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
+10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
+10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
+10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
+10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
+10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
+10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
+10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
+10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
+10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
+10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
+10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
+10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
+10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
+10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
+10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
+10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
+10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
+10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
+10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
+10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
+10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
+10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
+10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
+10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
+10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
+10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
+10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
+10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
+10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
+10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/dataFile/users_info_target.dat
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/dataFile/users_info_target.dat b/docker/griffin-base/griffin/dataFile/users_info_target.dat
new file mode 100755
index 0000000..07a6b40
--- /dev/null
+++ b/docker/griffin-base/griffin/dataFile/users_info_target.dat
@@ -0,0 +1,50 @@
+10001|Tom001|Jerrya|201 DisneyCity|tomajerrya001@dc.org|10000101|94022
+10002|Tom002|Jerrya|202 DisneyCity|tomajerrya002@dc.org|10000102|94022
+10003|Tom003|Jerrya|203 DisneyCity|tomajerrya003@dc.org|10000003|94022
+10004|Tom004|Jerrya|204 DisneyCity|tomajerrya004@dc.org|10000004|94022
+10005|Tom005|Jerrya|205 DisneyCity|tomajerrya005@dc.org|10000005|94022
+10006|Tom006|Jerrya|206 DisneyCity|tomajerrya006@dc.org|10000006|94022
+10007|Tom007|Jerrya|207 DisneyCity|tomajerrya007@dc.org|10000007|94022
+10008|Tom008|Jerrya|208 DisneyCity|tomajerrya008@dc.org|10000008|94022
+10009|Tom009|Jerrya|209 DisneyCity|tomajerrya009@dc.org|10000009|94022
+10010|Tom010|Jerrya|210 DisneyCity|tomajerrya010@dc.org|10000010|94022
+10011|Tom011|Jerrya|211 DisneyCity|tomajerrya011@dc.org|10000011|94022
+10012|Tom012|Jerrya|212 DisneyCity|tomajerrya012@dc.org|10000012|94022
+10013|Tom013|Jerrya|213 DisneyCity|tomajerrya013@dc.org|10000013|94022
+10014|Tom014|Jerrya|214 DisneyCity|tomajerrya014@dc.org|10000014|94022
+10015|Tom015|Jerrya|215 DisneyCity|tomajerrya015@dc.org|10000015|94022
+10016|Tom016|Jerrya|216 DisneyCity|tomajerrya016@dc.org|10000016|94022
+10017|Tom017|Jerrya|217 DisneyCity|tomajerrya017@dc.org|10000017|94022
+10018|Tom018|Jerrya|218 DisneyCity|tomajerrya018@dc.org|10000018|94022
+10019|Tom019|Jerrya|219 DisneyCity|tomajerrya019@dc.org|10000019|94022
+10020|Tom020|Jerrya|220 DisneyCity|tomajerrya020@dc.org|10000020|94022
+10021|Tom021|Jerrya|221 DisneyCity|tomajerrya021@dc.org|10000021|94022
+10022|Tom022|Jerrya|222 DisneyCity|tomajerrya022@dc.org|10000022|94022
+10023|Tom023|Jerrya|223 DisneyCity|tomajerrya023@dc.org|10000023|94022
+10024|Tom024|Jerrya|224 DisneyCity|tomajerrya024@dc.org|10000024|94022
+10025|Tom025|Jerrya|225 DisneyCity|tomajerrya025@dc.org|10000025|94022
+10026|Tom026|Jerrya|226 DisneyCity|tomajerrya026@dc.org|10000026|94022
+10027|Tom027|Jerrya|227 DisneyCity|tomajerrya027@dc.org|10000027|94022
+10028|Tom028|Jerrya|228 DisneyCity|tomajerrya028@dc.org|10000028|94022
+10029|Tom029|Jerrya|229 DisneyCity|tomajerrya029@dc.org|10000029|94022
+10030|Tom030|Jerrya|230 DisneyCity|tomajerrya030@dc.org|10000030|94022
+10031|Tom031|Jerrya|231 DisneyCity|tomajerrya031@dc.org|10000031|94022
+10032|Tom032|Jerrya|232 DisneyCity|tomajerrya032@dc.org|10000032|94022
+10033|Tom033|Jerrya|233 DisneyCity|tomajerrya033@dc.org|10000033|94022
+10034|Tom034|Jerrya|234 DisneyCity|tomajerrya034@dc.org|10000034|94022
+10035|Tom035|Jerrya|235 DisneyCity|tomajerrya035@dc.org|10000035|94022
+10036|Tom036|Jerrya|236 DisneyCity|tomajerrya036@dc.org|10000036|94022
+10037|Tom037|Jerrya|237 DisneyCity|tomajerrya037@dc.org|10000037|94022
+10038|Tom038|Jerrya|238 DisneyCity|tomajerrya038@dc.org|10000038|94022
+10039|Tom039|Jerrya|239 DisneyCity|tomajerrya039@dc.org|10000039|94022
+10040|Tom040|Jerrya|240 DisneyCity|tomajerrya040@dc.org|10000040|94022
+10041|Tom041|Jerrya|241 DisneyCity|tomajerrya041@dc.org|10000041|94022
+10042|Tom042|Jerrya|242 DisneyCity|tomajerrya042@dc.org|10000042|94022
+10043|Tom043|Jerrya|243 DisneyCity|tomajerrya043@dc.org|10000043|94022
+10044|Tom044|Jerrya|244 DisneyCity|tomajerrya044@dc.org|10000044|94022
+10045|Tom045|Jerrya|245 DisneyCity|tomajerrya045@dc.org|10000045|94022
+10046|Tom046|Jerrya|246 DisneyCity|tomajerrya046@dc.org|10000046|94022
+10047|Tom047|Jerrya|247 DisneyCity|tomajerrya047@dc.org|10000047|94022
+10048|Tom048|Jerrya|248 DisneyCity|tomajerrya048@dc.org|10000048|94022
+10049|Tom049|Jerrya|249 DisneyCity|tomajerrya049@dc.org|10000049|94022
+10050|Tom050|Jerrya|250 DisneyCity|tomajerrya050@dc.org|10000050|94022
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hadoop-end.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hadoop-end.sh b/docker/griffin-base/griffin/hadoop-end.sh
new file mode 100755
index 0000000..432bf8f
--- /dev/null
+++ b/docker/griffin-base/griffin/hadoop-end.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/sbin/stop-dfs.sh
+$HADOOP_PREFIX/sbin/stop-yarn.sh
+service sshd stop
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hadoop-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hadoop-start.sh b/docker/griffin-base/griffin/hadoop-start.sh
new file mode 100755
index 0000000..8e88406
--- /dev/null
+++ b/docker/griffin-base/griffin/hadoop-start.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+HOSTNAME=sandbox
+echo $(head -1 /etc/hosts | cut -f1) $HOSTNAME >> /etc/hosts
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-after-hive.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-after-hive.sh b/docker/griffin-base/griffin/hd-after-hive.sh
new file mode 100755
index 0000000..75e8ccd
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-after-hive.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+
+hadoop fs -touchz /user/hive/warehouse/users_info_src/_SUCCESS
+hadoop fs -touchz /user/hive/warehouse/users_info_target/_SUCCESS
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-before-hive.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-before-hive.sh b/docker/griffin-base/griffin/hd-before-hive.sh
new file mode 100755
index 0000000..c932d7a
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-before-hive.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+hadoop fs -mkdir /user/hive/warehouse/users_info_src
+hadoop fs -put $GRIFFIN_HOME/dataFile/users_info_src.dat /user/hive/warehouse/users_info_src/
+hadoop fs -mkdir /user/hive/warehouse/users_info_target
+hadoop fs -put $GRIFFIN_HOME/dataFile/users_info_target.dat /user/hive/warehouse/users_info_target/
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hd-test-json.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hd-test-json.sh b/docker/griffin-base/griffin/hd-test-json.sh
new file mode 100755
index 0000000..8fa0aec
--- /dev/null
+++ b/docker/griffin-base/griffin/hd-test-json.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+hadoop fs -mkdir /user/test
+hadoop fs -put $GRIFFIN_HOME/jsonFile/accu_config.json /user/test/
+hadoop fs -put $GRIFFIN_HOME/jsonFile/vali_config.json /user/test/
+hadoop fs -mkdir /user/test/output
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hive-init.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hive-init.sh b/docker/griffin-base/griffin/hive-init.sh
new file mode 100755
index 0000000..70fab77
--- /dev/null
+++ b/docker/griffin-base/griffin/hive-init.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#mysql service
+service mysqld start
+
+$GRIFFIN_HOME/mysql_secure.sh 123456 && rm $GRIFFIN_HOME/mysql_secure.sh
+$GRIFFIN_HOME/mysql_init.sh && rm $GRIFFIN_HOME/mysql_init.sh
+
+#hive metastore service
+hive --service metastore &
+
+#insert hive table
+hive -f $GRIFFIN_HOME/hive-input.hql
+
+rm $GRIFFIN_HOME/hive-input.hql
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/hive-input.hql
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/hive-input.hql b/docker/griffin-base/griffin/hive-input.hql
new file mode 100755
index 0000000..9f33879
--- /dev/null
+++ b/docker/griffin-base/griffin/hive-input.hql
@@ -0,0 +1,25 @@
+CREATE TABLE users_info_src (
+ user_id bigint,
+ first_name string,
+ last_name string,
+ address string,
+ email string,
+ phone string,
+ post_code string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;
+
+
+
+CREATE TABLE users_info_target (
+ user_id bigint,
+ first_name string,
+ last_name string,
+ address string,
+ email string,
+ phone string,
+ post_code string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|'
+STORED AS TEXTFILE;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/jsonFile/accu_config.json
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/jsonFile/accu_config.json b/docker/griffin-base/griffin/jsonFile/accu_config.json
new file mode 100755
index 0000000..0b24d8b
--- /dev/null
+++ b/docker/griffin-base/griffin/jsonFile/accu_config.json
@@ -0,0 +1,63 @@
+{
+ "source": "users_info_src",
+ "target": "users_info_target",
+ "accuracyMapping": [
+ {
+ "sourceColId": 0,
+ "sourceColName": "user_id",
+ "targetColId": 0,
+ "targetColName": "user_id",
+ "matchFunction": "true",
+ "isPK": true
+ },
+ {
+ "sourceColId": 1,
+ "sourceColName": "first_name",
+ "targetColId": 1,
+ "targetColName": "first_name",
+ "matchFunction": "false",
+ "isPK": false
+ },
+ {
+ "sourceColId": 2,
+ "sourceColName": "last_name",
+ "targetColId": 2,
+ "targetColName": "last_name",
+ "matchFunction": "false",
+ "isPK": false
+ },
+ {
+ "sourceColId": 3,
+ "sourceColName": "address",
+ "targetColId": 3,
+ "targetColName": "address",
+ "matchFunction": "false",
+ "isPK": false
+ },
+ {
+ "sourceColId": 4,
+ "sourceColName": "email",
+ "targetColId": 4,
+ "targetColName": "email",
+ "matchFunction": "false",
+ "isPK": false
+ },
+ {
+ "sourceColId": 5,
+ "sourceColName": "phone",
+ "targetColId": 5,
+ "targetColName": "phone",
+ "matchFunction": "false",
+ "isPK": false
+ },
+ {
+ "sourceColId": 6,
+ "sourceColName": "post_code",
+ "targetColId": 6,
+ "targetColName": "post_code",
+ "matchFunction": "false",
+ "isPK": false
+ }
+ ]
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/jsonFile/vali_config.json
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/jsonFile/vali_config.json b/docker/griffin-base/griffin/jsonFile/vali_config.json
new file mode 100755
index 0000000..80dcf41
--- /dev/null
+++ b/docker/griffin-base/griffin/jsonFile/vali_config.json
@@ -0,0 +1,135 @@
+{
+ "dataSet": "users_info_src",
+ "validityReq": [
+ {
+ "colId": 0,
+ "colName": "user_id",
+ "metrics": [
+ {
+ "name": 5
+ },
+ {
+ "name": 6
+ },
+ {
+ "name": 7
+ },
+ {
+ "name": 8
+ },
+ {
+ "name": 1
+ }
+ ]
+ },
+ {
+ "colId": 1,
+ "colName": "first_name",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ },
+ {
+ "colId": 2,
+ "colName": "last_name",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ },
+ {
+ "colId": 3,
+ "colName": "address",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ },
+ {
+ "colId": 4,
+ "colName": "email",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ },
+ {
+ "colId": 5,
+ "colName": "phone",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ },
+ {
+ "colId": 6,
+ "colName": "post_code",
+ "metrics": [
+ {
+ "name": 1
+ },
+ {
+ "name": 2
+ },
+ {
+ "name": 3
+ },
+ {
+ "name": 4
+ }
+ ]
+ }
+ ]
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/griffin/pre-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/griffin/pre-start.sh b/docker/griffin-base/griffin/pre-start.sh
new file mode 100755
index 0000000..7ea925d
--- /dev/null
+++ b/docker/griffin-base/griffin/pre-start.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+#sleep 10 seconds
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#create directories
+hadoop fs -mkdir /tmp
+hadoop fs -mkdir /user/hive
+hadoop fs -mkdir /user/hive/warehouse
+hadoop fs -chmod g+w /tmp
+hadoop fs -chmod g+w /user/hive/warehouse
+
+#prepare mongod configure
+mongod -f /etc/mongod.conf
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/start-up.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/start-up.sh b/docker/griffin-base/start-up.sh
new file mode 100755
index 0000000..0aed01a
--- /dev/null
+++ b/docker/griffin-base/start-up.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+: ${TOMCAT_HOME:=/apache/tomcat}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
+
+#spark start
+$SPARK_HOME/sbin/start-all.sh
+
+#start mongodb
+mongod -f /etc/mongod.conf
+
+#mysql service
+service mysqld start
+
+#hive metastore service
+hive --service metastore &
+
+
+/bin/bash -c "bash"
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/adduser.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/adduser.sh b/docker/griffin-base/user/adduser.sh
new file mode 100755
index 0000000..3d6f362
--- /dev/null
+++ b/docker/griffin-base/user/adduser.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ROOT_UID=0
+SUCCESS=0
+E_USEREXISTS=70
+
+# Run as root, of course. (this might not be necessary, because we have to run the script somehow with root anyway)
+if [ "$UID" -ne "$ROOT_UID" ]
+then
+ echo "Must be root to run this script."
+ exit $E_NOTROOT
+fi
+
+#test, if both argument are there
+if [ $# -eq 2 ]; then
+username=$1
+pass=$2
+
+ # Check if user already exists.
+ grep -q "$username" /etc/passwd
+ if [ $? -eq $SUCCESS ]
+ then
+ echo "User $username does already exist."
+ echo "please chose another username."
+ exit $E_USEREXISTS
+ fi
+
+
+ useradd -d /home/"$username" -m -g users -s /bin/bash "$username"
+
+ echo "the account is setup"
+
+ echo "$username:$pass" | chpasswd
+
+else
+ echo " this programm needs 2 arguments you have given $# "
+ echo " you have to call the script $0 username and the pass "
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/env.txt
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/env.txt b/docker/griffin-base/user/env.txt
new file mode 100644
index 0000000..e6ee39e
--- /dev/null
+++ b/docker/griffin-base/user/env.txt
@@ -0,0 +1,13 @@
+#user define enviroment
+export GRIFFIN_HOME=/home/griffin
+
+export JAVA_HOME=/usr/java/latest
+export HADOOP_HOME=/usr/local/hadoop
+export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+export SPARK_HOME=/usr/local/spark
+export HIVE_HOME=/apache/hive
+export TOMCAT_HOME=/apache/tomcat
+
+export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$HIVE_HOME/bin:$TOMCAT_HOME/bin:$PATH
+
+export HADOOP_USER_CLASSPATH_FIRST=true
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin-base/user/sudouser.sh
----------------------------------------------------------------------
diff --git a/docker/griffin-base/user/sudouser.sh b/docker/griffin-base/user/sudouser.sh
new file mode 100755
index 0000000..3678da9
--- /dev/null
+++ b/docker/griffin-base/user/sudouser.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+if [ $# -eq 1 ]; then
+username=$1
+echo "$username ALL=(ALL:ALL) ALL" >> /etc/sudoers
+fi
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/griffin/Dockerfile b/docker/griffin/Dockerfile
new file mode 100755
index 0000000..647c138
--- /dev/null
+++ b/docker/griffin/Dockerfile
@@ -0,0 +1,27 @@
+FROM griffin-base-env:latest
+MAINTAINER com.ebay.oss
+
+
+#input files
+ADD script/*.sh $GRIFFIN_HOME/
+ADD ./*.sh $GRIFFIN_HOME/
+RUN mkdir $GRIFFIN_HOME/tmp $GRIFFIN_HOME/temp $GRIFFIN_HOME/log
+RUN chmod -R 755 $GRIFFIN_HOME
+
+#add db
+RUN mkdir /db
+ADD db /db
+
+#remove tomcat webapps
+ENV APACHE_HOME /apache/tomcat
+RUN rm -rf $APACHE_HOME/webapps/*
+
+#prepare data
+WORKDIR $GRIFFIN_HOME
+RUN ./hadoop-start.sh && ./pre-start.sh && ./hadoop-end.sh && rm hadoop-start.sh hadoop-end.sh pre-start.sh
+
+#download resources
+RUN ./download-resources.sh && rm download-resources.sh
+
+#entrypoint
+ENTRYPOINT ["./start-up.sh"]
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/SEQUENCES.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/SEQUENCES.bson b/docker/griffin/db/unitdb0/SEQUENCES.bson
new file mode 100644
index 0000000..5a5cc64
Binary files /dev/null and b/docker/griffin/db/unitdb0/SEQUENCES.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/SEQUENCES.metadata.json b/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
new file mode 100644
index 0000000..0395aac
--- /dev/null
+++ b/docker/griffin/db/unitdb0/SEQUENCES.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.SEQUENCES"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/data_assets.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/data_assets.bson b/docker/griffin/db/unitdb0/data_assets.bson
new file mode 100644
index 0000000..5e2c65f
Binary files /dev/null and b/docker/griffin/db/unitdb0/data_assets.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/data_assets.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/data_assets.metadata.json b/docker/griffin/db/unitdb0/data_assets.metadata.json
new file mode 100644
index 0000000..9e102a1
--- /dev/null
+++ b/docker/griffin/db/unitdb0/data_assets.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.data_assets"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_job.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_job.bson b/docker/griffin/db/unitdb0/dq_job.bson
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_job.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_job.metadata.json b/docker/griffin/db/unitdb0/dq_job.metadata.json
new file mode 100644
index 0000000..fa3a369
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_job.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_job"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_metrics_values.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_metrics_values.bson b/docker/griffin/db/unitdb0/dq_metrics_values.bson
new file mode 100644
index 0000000..857df0c
Binary files /dev/null and b/docker/griffin/db/unitdb0/dq_metrics_values.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json b/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
new file mode 100644
index 0000000..c259dec
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_metrics_values.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_metrics_values"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson
new file mode 100644
index 0000000..8294c87
Binary files /dev/null and b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
new file mode 100644
index 0000000..05a5987
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_missed_file_path_lkp.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_missed_file_path_lkp"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_model.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_model.bson b/docker/griffin/db/unitdb0/dq_model.bson
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_model.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_model.metadata.json b/docker/griffin/db/unitdb0/dq_model.metadata.json
new file mode 100644
index 0000000..2da029a
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_model.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_model"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_schedule.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_schedule.bson b/docker/griffin/db/unitdb0/dq_schedule.bson
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/dq_schedule.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/dq_schedule.metadata.json b/docker/griffin/db/unitdb0/dq_schedule.metadata.json
new file mode 100644
index 0000000..036d141
--- /dev/null
+++ b/docker/griffin/db/unitdb0/dq_schedule.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.dq_schedule"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/system.indexes.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/system.indexes.bson b/docker/griffin/db/unitdb0/system.indexes.bson
new file mode 100644
index 0000000..973ade6
Binary files /dev/null and b/docker/griffin/db/unitdb0/system.indexes.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/user_subscribe.bson
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/user_subscribe.bson b/docker/griffin/db/unitdb0/user_subscribe.bson
new file mode 100644
index 0000000..c00ae2b
Binary files /dev/null and b/docker/griffin/db/unitdb0/user_subscribe.bson differ
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/db/unitdb0/user_subscribe.metadata.json
----------------------------------------------------------------------
diff --git a/docker/griffin/db/unitdb0/user_subscribe.metadata.json b/docker/griffin/db/unitdb0/user_subscribe.metadata.json
new file mode 100644
index 0000000..9cf671a
--- /dev/null
+++ b/docker/griffin/db/unitdb0/user_subscribe.metadata.json
@@ -0,0 +1 @@
+{"options":{},"indexes":[{"v":1,"key":{"_id":1},"name":"_id_","ns":"unitdb0.user_subscribe"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/download-resources.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/download-resources.sh b/docker/griffin/download-resources.sh
new file mode 100755
index 0000000..758d0da
--- /dev/null
+++ b/docker/griffin/download-resources.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+
+URI=https://oss.sonatype.org/content/repositories/snapshots/com/ebay/oss
+
+#get xml line
+getval()
+{
+ RESULT=`sed -n 's/.*>\(.*\)<\/'$2'>/\1/p' $1`
+ echo $RESULT | cut -d ' ' -f 1
+}
+
+#get model version jar
+MODEL_DIR=griffin-models
+MODEL_MAVEN_XML=maven-metadata.xml
+wget $URI/$MODEL_DIR/$MODEL_MAVEN_XML
+MODEL_ARTIFACT_ID=`getval $MODEL_MAVEN_XML artifactId`
+MODEL_VERSION=`getval $MODEL_MAVEN_XML version`
+rm $MODEL_MAVEN_XML
+
+MODEL_MAVEN_METADATA_XML=maven-metadata.xml
+MODEL_PATH=$URI/$MODEL_DIR/$MODEL_VERSION
+wget $MODEL_PATH/$MODEL_MAVEN_METADATA_XML
+MODEL_VALUE=`getval $MODEL_MAVEN_METADATA_XML value`
+rm $MODEL_MAVEN_METADATA_XML
+
+MODEL_JAR=${MODEL_ARTIFACT_ID}-${MODEL_VALUE}.jar
+
+
+#get core version war
+CORE_DIR=griffin-core
+CORE_MAVEN_XML=maven-metadata.xml
+wget $URI/$CORE_DIR/$CORE_MAVEN_XML
+CORE_ARTIFACT_ID=`getval $CORE_MAVEN_XML artifactId`
+CORE_VERSION=`getval $CORE_MAVEN_XML version`
+rm $CORE_MAVEN_XML
+
+CORE_MAVEN_METADATA_XML=maven-metadata.xml
+CORE_PATH=$URI/$CORE_DIR/$CORE_VERSION
+wget $CORE_PATH/$CORE_MAVEN_METADATA_XML
+CORE_VALUE=`getval $CORE_MAVEN_METADATA_XML value`
+rm $MODEL_MAVEN_METADATA_XML
+
+CORE_WAR=${CORE_ARTIFACT_ID}-${CORE_VALUE}.war
+
+#download and move
+MODEL_FULL_PATH=$MODEL_PATH/$MODEL_JAR
+CORE_FULL_PATH=$CORE_PATH/$CORE_WAR
+
+MODEL_JAR_NEW=griffin-models.jar
+CORE_WAR_NEW=ROOT.war
+
+MODEL_JAR_NEW_PATH=$GRIFFIN_HOME/$MODEL_JAR_NEW
+CORE_WAR_NEW_PATH=$APACHE_HOME/webapps/$CORE_WAR_NEW
+
+wget $MODEL_FULL_PATH
+wget $CORE_FULL_PATH
+
+mv $MODEL_JAR $MODEL_JAR_NEW_PATH
+mv $CORE_WAR $CORE_WAR_NEW_PATH
+
+chmod 755 $MODEL_JAR_NEW_PATH
+chmod 755 $CORE_WAR_NEW_PATH
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/hadoop-end.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/hadoop-end.sh b/docker/griffin/hadoop-end.sh
new file mode 100755
index 0000000..432bf8f
--- /dev/null
+++ b/docker/griffin/hadoop-end.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/sbin/stop-dfs.sh
+$HADOOP_PREFIX/sbin/stop-yarn.sh
+service sshd stop
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/hadoop-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/hadoop-start.sh b/docker/griffin/hadoop-start.sh
new file mode 100755
index 0000000..8e88406
--- /dev/null
+++ b/docker/griffin/hadoop-start.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+HOSTNAME=sandbox
+echo $(head -1 /etc/hosts | cut -f1) $HOSTNAME >> /etc/hosts
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/pre-start.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/pre-start.sh b/docker/griffin/pre-start.sh
new file mode 100755
index 0000000..37a51b0
--- /dev/null
+++ b/docker/griffin/pre-start.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+
+#sleep 10 seconds
+sleep 5
+hdfs dfsadmin -safemode leave
+
+#restore mongodb data
+mongod -f /etc/mongod.conf
+mongorestore --db unitdb0 /db/unitdb0
+
+#create griffin env
+hadoop fs -mkdir /user/griffin
+hadoop fs -chmod g+w /user/griffin
+hadoop fs -mkdir /user/griffin/running
+hadoop fs -mkdir /user/griffin/history
+hadoop fs -mkdir /user/griffin/failure
+hadoop fs -chmod g+w /user/griffin/running
+hadoop fs -chmod g+w /user/griffin/history
+hadoop fs -chmod g+w /user/griffin/failure
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/env.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/env.sh b/docker/griffin/script/env.sh
new file mode 100755
index 0000000..84bdd55
--- /dev/null
+++ b/docker/griffin/script/env.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+HDFS_WORKDIR=/user/griffin/running
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/griffin_jobs.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/griffin_jobs.sh b/docker/griffin/script/griffin_jobs.sh
new file mode 100755
index 0000000..0200fec
--- /dev/null
+++ b/docker/griffin/script/griffin_jobs.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+
+ROOT_DIR=$(cd $(dirname $0); pwd)
+if [ -f $ROOT_DIR/env.sh ]; then
+ . $ROOT_DIR/env.sh
+fi
+
+GRIFFIN_HOME=${GRIFFIN_HOME:-$ROOT_DIR}
+
+HDFS_WORKDIR=${HDFS_WORKDIR:-/user/griffin/running}
+TEMP_DIR=${TEMP_DIR:-$ROOT_DIR/temp}
+LOG_DIR=${LOG_DIR:-$ROOT_DIR/log}
+
+mkdir -p $TEMP_DIR
+mkdir -p $LOG_DIR
+
+lv1tempfile=$TEMP_DIR/temp.txt
+lv2tempfile=$TEMP_DIR/temp2.txt
+
+set +e
+
+hadoop fs -ls $HDFS_WORKDIR > $lv1tempfile
+
+while read line
+do
+ lv1dir=${line##* }
+ jobid=${lv1dir##*/}
+ ts=`date +%Y%m%d%H%M%S`
+ echo $ts
+ logfile=$LOG_DIR/${jobid}_${ts}.dqjoblog
+ rm -rf $logfile
+ touch $logfile
+
+ hadoop fs -test -f $lv1dir/_START
+ rc=$?
+ echo "$rc $lv1dir/_START" >> $logfile
+ if [ $rc -ne 0 ] && [ "${lv1dir:0:1}" == "/" ]
+ then
+ lv2tempfile=${LOG_DIR}/${jobid}_watch
+ rm -rf $lv2tempfile
+ hadoop fs -get $lv1dir/_watchfile $lv2tempfile
+ cat $lv2tempfile >> $logfile
+
+ watchfiledone=1
+ hascontent=0
+ while read watchline
+ do
+ hascontent=1
+ hadoop fs -test -f $watchline/_SUCCESS
+ rcode1=$?
+ hadoop fs -test -f $watchline/*done
+ rcode2=$?
+ echo "$rcode1 $watchline/_SUCCESS $rcode2 $watchline/*done" >> $logfile
+ if [ $rcode1 -ne 0 ] && [ $rcode2 -ne 0 ]
+ then
+ watchfiledone=0
+ break
+ fi
+ done < $lv2tempfile
+
+ if [ $watchfiledone -eq 1 ] && [ $hascontent -eq 1 ]
+ then
+ hadoop fs -touchz $lv1dir/_START
+ hadoop fs -test -f $lv1dir/_type_0.done
+ rc1=$?
+ echo "$rc1 $lv1dir/_type_0.done" >> $logfile
+ hadoop fs -test -f $lv1dir/_type_1.done
+ rc2=$?
+ echo "$rc2 $lv1dir/_type_1.done" >> $logfile
+ if [ $rc1 -eq 0 ]
+ then
+ echo "spark-submit --class org.apache.griffin.accuracy.Accu --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ " >> $logfile
+ spark-submit --class org.apache.griffin.accuracy.Accu --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
+ elif [ $rc2 -eq 0 ]
+ then
+ echo "spark-submit --class org.apache.griffin.validility.Vali --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ " >> $logfile
+ spark-submit --class org.apache.griffin.validility.Vali --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
+ fi
+
+ echo "done" >> $logfile
+ exit
+ fi
+ fi
+
+done < $lv1tempfile
+
+set -e
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/script/griffin_regular_run.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/script/griffin_regular_run.sh b/docker/griffin/script/griffin_regular_run.sh
new file mode 100755
index 0000000..35646c2
--- /dev/null
+++ b/docker/griffin/script/griffin_regular_run.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+ROOT_DIR=$(cd $(dirname $0); pwd)
+
+set +e
+while true
+do
+ echo "start"
+ $ROOT_DIR/griffin_jobs.sh 2>&1
+ rcode=$?
+ echo "end $rcode"
+ rm -rf $ROOT_DIR/nohup.out
+ sleep 60
+done
+set -e
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/docker/griffin/start-up.sh
----------------------------------------------------------------------
diff --git a/docker/griffin/start-up.sh b/docker/griffin/start-up.sh
new file mode 100755
index 0000000..164da7b
--- /dev/null
+++ b/docker/griffin/start-up.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+: ${TOMCAT_HOME:=/apache/tomcat}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+# setting spark defaults
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+
+service sshd start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
+
+#spark start
+$SPARK_HOME/sbin/start-all.sh
+
+#start mongodb
+mongod -f /etc/mongod.conf
+
+#mysql service
+service mysqld start
+
+#hive metastore service
+hive --service metastore &
+
+echo "hive metastore service"
+
+#start script
+nohup ./griffin_regular_run.sh &
+
+echo "bark regular run"
+
+
+CMD=${1:-"log"}
+if [[ "$CMD" == "bash" ]];
+then
+ #start tomcat
+ /etc/init.d/tomcat start
+ /bin/bash -c "$*"
+else
+ ${TOMCAT_HOME}/bin/catalina.sh run
+fi
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/.gitignore
----------------------------------------------------------------------
diff --git a/griffin-core/.gitignore b/griffin-core/.gitignore
new file mode 100644
index 0000000..c13c943
--- /dev/null
+++ b/griffin-core/.gitignore
@@ -0,0 +1,27 @@
+dist
+build
+bin
+rebel.xml
+MANIFEST.MF
+/tests/
+/coverage/
+/cobertura.ser
+*/ebay.log
+*.class
+*.jar
+*.zip
+*/target/*
+*/temp_persist_config_*.xml
+*/src/main/resources/buildinfo.properties
+/.idea/
+**/*.iml
+build.xml
+ebay-build.xml
+4cc.checksum
+/.metadata
+/.project
+/target/**
+/.settings/**
+/target/
+/.externalToolBuilders/
+/.classpath
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/pom.xml
----------------------------------------------------------------------
diff --git a/griffin-core/pom.xml b/griffin-core/pom.xml
new file mode 100644
index 0000000..752a8fd
--- /dev/null
+++ b/griffin-core/pom.xml
@@ -0,0 +1,302 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache
+ License, Version 2.0 (the "License"); you may not use this file except in
+ compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <groupId>com.ebay.oss</groupId>
+ <artifactId>griffin-parent</artifactId>
+ <version>0.1.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>griffin-core</artifactId>
+ <name>griffin-core</name>
+ <packaging>war</packaging>
+
+
+ <dependencies>
+
+ <!-- <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId>
+ <version>2.4</version> </dependency> -->
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>3.0.1</version>
+ <scope>provided</scope>
+ </dependency>
+
+
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hibernate</groupId>
+ <artifactId>hibernate-validator</artifactId>
+ <version>5.2.4.Final</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- Jersey -->
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjweaver</artifactId>
+ <version>1.6.10</version>
+ </dependency>
+
+ <!-- Spring 3 dependencies -->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>3.2.8.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>3.2.8.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-webmvc</artifactId>
+ <version>3.2.8.RELEASE</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+
+ </dependency>
+
+
+
+ <!-- <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> -->
+ <!-- </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId>
+ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> -->
+
+ <!-- </dependency> -->
+
+ <!-- Jersey + Spring -->
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-spring</artifactId>
+ <version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-webmvc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ </exclusion>
+
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>${mongo.version}</version>
+ </dependency>
+
+ <!-- <dependency> <groupId>javax.ws.rs</groupId> <artifactId>javax.ws.rs-api</artifactId>
+ <version>2.0</version> </dependency> -->
+
+ <dependency>
+ <groupId>com.google.code.morphia</groupId>
+ <artifactId>morphia</artifactId>
+ <version>0.104</version>
+ </dependency>
+ <!-- <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId>
+ </dependency> <dependency> <groupId>commons-digester</groupId> <artifactId>commons-digester</artifactId>
+ <version>2.1</version> </dependency> <dependency> <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId> <version>1.8.1</version> </dependency> -->
+
+ <!-- <dependency>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>griffin-scheduler</artifactId>
+ <version>${griffin.version}</version>
+ <scope>compile</scope>
+ </dependency> -->
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>griffin-ui</artifactId>
+ <version>${griffin.version}</version>
+ <classifier>resources</classifier>
+ <type>zip</type>
+ <!-- Make sure this isn't included on any classpath -->
+ <scope>provided</scope>
+ </dependency>
+ <!-- <dependency> <groupId>org.apache.griffin</groupId> <artifactId>griffin-core</artifactId>
+ <version>0.0.1-SNAPSHOT</version> <scope>compile</scope> </dependency> -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <version>3.2.8.RELEASE</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Guice =============================================================
+ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId>
+ <version>${guice.version}</version> <scope>test</scope> </dependency> <dependency>
+ <groupId>com.google.inject.extensions</groupId> <artifactId>guice-servlet</artifactId>
+ <version>${guice.version}</version> <scope>test</scope> </dependency> <dependency>
+ <groupId>com.sun.jersey.contribs</groupId> <artifactId>jersey-guice</artifactId>
+ <version>${jersey.version}</version> <scope>test</scope> </dependency> -->
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-grizzly2</artifactId>
+ <version>${jersey.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+
+ </dependencies>
+
+ <build>
+ <finalName>ROOT</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ <!-- <plugin>
+ <artifactId>maven-war-plugin</artifactId>
+ <version>2.1.1</version>
+ <configuration>
+ <attachClasses>true</attachClasses>
+ <classesClassifier>classes</classesClassifier>
+ </configuration>
+ </plugin> -->
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <formats>
+ <format>html</format>
+ <format>xml</format>
+ </formats>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <webResources>
+ <resource>
+ <directory>${project.build.directory}/generated-resources/griffin-ui-${griffin.version}</directory>
+ </resource>
+ </webResources>
+ <attachClasses>true</attachClasses>
+ <classesClassifier>classes</classesClassifier>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-shared-resources</id>
+ <goals>
+ <goal>unpack-dependencies</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/generated-resources</outputDirectory>
+ <includeArtifacIds>griffin-ui</includeArtifacIds>
+ <includeGroupIds>${project.groupId}</includeGroupIds>
+ <overWriteReleases>true</overWriteReleases>
+ <overWriteSnapshots>true</overWriteSnapshots>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
new file mode 100644
index 0000000..3b090ec
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/HDFSUtils.java
@@ -0,0 +1,33 @@
+package com.ebay.oss.griffin.common;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HDFSUtils {
+
+ static Logger logger = LoggerFactory.getLogger(HDFSUtils.class);
+
+ public static boolean checkHDFSFolder(String folderPath) {
+
+ Process processMoveFolder;
+ int result;
+ try {
+ processMoveFolder = Runtime.getRuntime().exec("hadoop fs -ls " + folderPath);
+
+ result = processMoveFolder.waitFor();
+
+ if(result == 0)
+ {
+ return true;
+ }
+
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ return false;
+ }
+ return false;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
new file mode 100644
index 0000000..ae051f1
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/NumberUtils.java
@@ -0,0 +1,41 @@
+package com.ebay.oss.griffin.common;
+
+import org.springframework.util.StringUtils;
+
+public class NumberUtils {
+
+ public static int parseInt(Object o) {
+ if (o == null) {
+ return -1;
+ }
+
+ String s = o.toString();
+ if(StringUtils.isEmpty(s) ) {
+ return -1;
+ }
+
+ try {
+ return Integer.parseInt(s);
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ public static long parseLong(Object o) {
+ if (o == null) {
+ return -1;
+ }
+
+ String s = o.toString();
+ if(StringUtils.isEmpty(s) ) {
+ return -1;
+ }
+
+ try {
+ return Long.parseLong(s);
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
new file mode 100644
index 0000000..f8e3386
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/Pair.java
@@ -0,0 +1,64 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+
+public class Pair {
+
+ public final String key;
+
+ public final Object value;
+
+ public Pair(String key, Object value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ result = prime * result + ((value == null) ? 0 : value.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Pair other = (Pair) obj;
+ if (key == null) {
+ if (other.key != null)
+ return false;
+ } else if (!key.equals(other.key))
+ return false;
+ if (value == null) {
+ if (other.value != null)
+ return false;
+ } else if (!value.equals(other.value))
+ return false;
+ return true;
+ }
+
+
+ @Override
+ public String toString() {
+ return "Pair [key=" + key + ", value=" + value + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
new file mode 100644
index 0000000..14899eb
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/ScheduleModelSeperator.java
@@ -0,0 +1,21 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+
+public class ScheduleModelSeperator {
+ public static final String SEPERATOR = "-=-";
+ public static final String SPLIT_SEPERATOR = "-=-";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
new file mode 100644
index 0000000..924c530
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/common/SimpleCORSFilter.java
@@ -0,0 +1,45 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.common;
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+
+public class SimpleCORSFilter implements Filter {
+
+ @Override
+ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException {
+ HttpServletResponse response = (HttpServletResponse) res;
+ response.setHeader("Access-Control-Allow-Origin", "*");
+ response.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE,PUT");
+ response.setHeader("Access-Control-Max-Age", "3600");
+ response.setHeader("Access-Control-Allow-Headers", "X-PINGOTHER, Origin, X-Requested-With, Content-Type, Accept");
+ chain.doFilter(req, res);
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) {}
+
+ @Override
+ public void destroy() {}
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
new file mode 100644
index 0000000..4c2bcf9
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/AnomalyType.java
@@ -0,0 +1,26 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+/** the enumerable type of Anomaly Detection */
+public class AnomalyType {
+
+ public static final int ANOMALY_DEFAULT = 0;
+ public static final int HISTORY_TREND = 1;
+ public static final int BOLLINGER_BANDS = 2;
+ public static final int MAD = 3;
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
new file mode 100644
index 0000000..3b92040
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataAsset.java
@@ -0,0 +1,199 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import com.google.code.morphia.annotations.Embedded;
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Property;
+import com.mongodb.DBObject;
+
+/**
+ * A DataAsset is an abstraction of a data source, from which a same type of data comes. A DataAsset
+ * could be a 'table' in RDB or a topic/feed from a stream.
+ */
+@Entity("data_assets")
+// unique constraint: asssetName + assetType + system
+public class DataAsset extends IdEntity{
+
+ // HDP the only one
+ @Property("platform")
+ private String platform;
+
+ // SystemTypeConstants
+ @Property("system")
+ private String system;
+
+ // e.g. viewitem
+ @Property("assetName")
+ private String assetName;
+
+ // HiveTable the only one
+ @Property("assetType")
+ private String assetType;
+
+ @Property("assetHDFSPath")
+ private String assetHDFSPath;
+
+ @Property("owner")
+ private String owner;
+
+ // createdDate
+ @Property("timestamp")
+ private Date timestamp;
+
+
+ // home-made schema, DataSchema is atually a simple schemaItem, a schema consists of a list of
+ // schemaItem
+ @Embedded
+ private List<DataSchema> schema = new ArrayList<>();
+
+ // hive table partition format
+ @Embedded
+ private List<PartitionFormat> partitions;
+
+ public DataAsset() { }
+
+ // FIXME should be somewhere else, ORM
+ @SuppressWarnings({"unchecked", "deprecation"})
+ public DataAsset(DBObject o) {
+ this.set_id(Long.parseLong(o.get("_id").toString()));
+ this.setAssetHDFSPath((String)o.get("assetHDFSPath"));
+ this.setAssetName((String)o.get("assetName"));
+ this.setAssetType((String)o.get("assetType"));
+
+ this.setOwner((String)o.get("owner"));
+ this.setPlatform((String)o.get("platform"));
+ this.setSystem((String)o.get("system"));
+
+// this.setPartitions((List<PartitionFormat>) o.get("partitions")); // this doesn't work
+ if(o.get("partitions")!=null) {
+ List<PartitionFormat> partitionlist = new ArrayList<PartitionFormat>();
+ List<DBObject> tlist = (List<DBObject>) o.get("partitions");
+ for(DBObject temp : tlist) {
+ partitionlist.add(new PartitionFormat(temp.get("name").toString(), temp.get("format").toString()));
+ }
+ this.setPartitions(partitionlist);
+ }
+
+// this.setSchema((List<DataSchema>) o.get("schema"));
+ if(o.get("schema")!=null) {
+ List<DBObject> tlist = (List<DBObject>) o.get("schema");
+ List<DataSchema> list = new ArrayList<DataSchema>();
+ for(DBObject temp : tlist) {
+ list.add(new DataSchema(temp.get("name").toString(), temp.get("type").toString(), temp.get("desc").toString(), temp.get("sample").toString()));
+ }
+ this.setSchema(list);
+ }
+
+ if (!o.containsField("timestamp")) {
+ this.setTimestamp(new Date());
+ } else {
+ this.setTimestamp(new Date(o.get("timestamp").toString()));
+ }
+ }
+
+ {
+
+
+ }
+
+ public String getPlatform() {
+ return platform;
+ }
+
+ public void setPlatform(String platform) {
+ this.platform = platform;
+ }
+
+ public String getSystem() {
+ return system;
+ }
+
+ public void setSystem(String system) {
+ this.system = system;
+ }
+
+ public String getAssetName() {
+ return assetName;
+ }
+
+ public void setAssetName(String assetName) {
+ this.assetName = assetName;
+ }
+
+ public String getAssetType() {
+ return assetType;
+ }
+
+ public void setAssetType(String assetType) {
+ this.assetType = assetType;
+ }
+
+ public String getAssetHDFSPath() {
+ return assetHDFSPath;
+ }
+
+ public void setAssetHDFSPath(String assetHDFSPath) {
+ this.assetHDFSPath = assetHDFSPath;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(Date timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public List<DataSchema> getSchema() {
+ return schema;
+ }
+
+ public void setSchema(List<DataSchema> schema) {
+ this.schema = schema;
+ }
+
+ public List<PartitionFormat> getPartitions() {
+ return partitions;
+ }
+
+ public void setPartitions(List<PartitionFormat> partitions) {
+ this.partitions = partitions;
+ }
+
+ public int getColId(String colName) {
+ if(schema == null || schema.isEmpty()) {
+ return -1;
+ }
+
+ for (int i = 0; i < schema.size(); i++) {
+ if (colName.equals(schema.get(i).getName()))
+ return i;
+ }
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
new file mode 100644
index 0000000..6f92a5d
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DataSchema.java
@@ -0,0 +1,80 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Embedded;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * this is a simple representation of a schema system, which describe the data type of a table.
+ */
+@Embedded
+public class DataSchema{
+
+ @Property("name")
+ private String name;
+
+ @Property("type")
+ private String type;
+
+ @Property("desc")
+ private String desc;
+
+ @Property("sample")
+ private String sample;
+
+ public DataSchema() {
+ }
+
+ public DataSchema(String name, String type, String desc, String sample) {
+ this.name = name;
+ this.type = type;
+ this.desc = desc;
+ this.sample = sample;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getDesc() {
+ return desc;
+ }
+
+ public void setDesc(String desc) {
+ this.desc = desc;
+ }
+
+ public String getSample() {
+ return sample;
+ }
+
+ public void setSample(String sample) {
+ this.sample = sample;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
new file mode 100644
index 0000000..9a10e8c
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqJob.java
@@ -0,0 +1,121 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Id;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * A DqJob is an instance of DqSchedule, the frequency/multiplicity is based on ScheduleType.
+ */
+@Entity("dq_job")
+public class DqJob{
+
+ /** {modelName} + "_" + "yyyy-mm-dd xx:00:00 000".getTime() */
+ @Id
+ private String _id;
+
+ // Inherits from DqSchedule.modelList
+ @Property("modelList")
+ private String modelList;
+
+ /** Inherits from DqSchedule#jobType.*/
+ @Property("jobType")
+ private int jobType;
+
+ /** @see JobStatus */
+ @Property("status")
+ private int status;
+
+ @Property("starttime")
+ private long starttime;
+
+ @Property("content")
+ private String content;
+
+ @Property("endtime")
+ private long endtime;
+
+ @Property("value")
+ private long value;
+
+ public String getId() {
+ return _id;
+ }
+
+ public void setId(String _id) {
+ this._id = _id;
+ }
+
+ public String getModelList() {
+ return modelList;
+ }
+
+ public void setModelList(String modelList) {
+ this.modelList = modelList;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public long getStarttime() {
+ return starttime;
+ }
+
+ public void setStarttime(long starttime) {
+ this.starttime = starttime;
+ }
+
+ public String getContent() {
+ return content;
+ }
+
+ public void setContent(String content) {
+ this.content = content;
+ }
+
+ public long getEndtime() {
+ return endtime;
+ }
+
+ public void setEndtime(long endtime) {
+ this.endtime = endtime;
+ }
+
+ public long getValue() {
+ return value;
+ }
+
+ public void setValue(long value) {
+ this.value = value;
+ }
+
+ public int getJobType() {
+ return jobType;
+ }
+
+ public void setJobType(int jobType) {
+ this.jobType = jobType;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
new file mode 100644
index 0000000..5b9aac2
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqMetricsValue.java
@@ -0,0 +1,111 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+
+import javax.validation.constraints.Min;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Pattern;
+
+import com.ebay.oss.griffin.vo.BaseObj;
+import com.google.code.morphia.annotations.Id;
+
+
+/**
+ * Metrics is the value of a DQ perspective (DqModel) execution/calculation.
+ *
+ * TODO: should introduce UnitOfMeasure in, so that a Metrics could be render independently.
+ */
+// uniq{modelName, timestamp}
+public class DqMetricsValue extends BaseObj implements Comparable<DqMetricsValue> {
+
+ @Id
+ private Long _id;
+
+ // model.name
+ @NotNull
+ @Pattern(regexp="\\A([0-9a-zA-Z\\_\\-\\.])+$")
+ private String metricName;
+
+ private String assetId;
+
+ @Min(0)
+ private long timestamp;
+
+ @Min(0)
+ private float value;
+
+ public DqMetricsValue() { }
+
+ public DqMetricsValue(String name, long timestamp, float value) {
+ this.metricName = name;
+ this.timestamp = timestamp;
+ this.value = value;
+ }
+
+ public Long get_id() {
+ return _id;
+ }
+
+ public void set_id(Long _id) {
+ this._id = _id;
+ }
+
+ public String getMetricName() {
+ return metricName;
+ }
+
+ public void setMetricName(String metricName) {
+ this.metricName = metricName;
+ }
+
+ public String getAssetId() {
+ return assetId;
+ }
+
+ public void setAssetId(String assetId) {
+ this.assetId = assetId;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public float getValue() {
+ return value;
+ }
+
+ public void setValue(float value) {
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(DqMetricsValue o) {
+ return (int) Math.signum(o.getTimestamp() - this.getTimestamp());
+// return o.getTimestamp() == this.getTimestamp() ? 0 :
+// (o.getTimestamp() > this.getTimestamp() ? 1 : -1);
+ }
+
+ @Override
+ public String toString() {
+ return "DqMetricsValue [_id=" + _id + ", metricName=" + metricName + ", timestamp="
+ + timestamp + ", value=" + value + "]";
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
new file mode 100644
index 0000000..5347af7
--- /dev/null
+++ b/griffin-core/src/main/java/com/ebay/oss/griffin/domain/DqModel.java
@@ -0,0 +1,231 @@
+/*
+ Copyright (c) 2016 eBay Software Foundation.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package com.ebay.oss.griffin.domain;
+
+import com.google.code.morphia.annotations.Entity;
+import com.google.code.morphia.annotations.Property;
+
+/**
+ * It is a specification of Data Quality perspective, which defines:
+ * <ol>
+ * <li> what data against, DataAsset(s) </li>
+ * <li> which type of DQ, ModelType </li>
+ * <li> related meta data based on the concrete modelType, modelContent </li>
+ * </ul>
+ */
+@Entity("dq_model")
+public class DqModel extends IdEntity{
+
+ // same as modelName
+ @Property("modelId")
+ private String modelId;
+
+ @Property("modelName")
+ private String modelName;
+
+ @Property("modelType")
+ private int modelType;
+
+ @Property("modelDesc")
+ private String modelDesc;
+
+ @Property("assetId")
+ private long assetId;
+
+ @Property("assetName")
+ private String assetName;
+
+ @Property("threshold")
+ private float threshold;
+
+ @Property("notificationEmail")
+ private String notificationEmail;
+
+ @Property("owner")
+ private String owner;
+
+ @Property("status")
+ private int status;
+
+ // @see ScheduleType
+ @Property("schedule")
+ private int schedule;
+
+ /**
+ * <ul>
+ * <li>ModelType.ACCURACY</li>
+ * <pre>
+ * srcDb | srcDataSet | dstDB | dstDataSet | [MappingItemInput.src, MII.dst, MII.isPk, MII.matchedMethod];
+ * </pre>
+ * <li>ModelType.VALIDITY</li>
+ * <pre> srcDb | srcDataSet | validityType | column </pre>
+ * <li>ModelType.Anomaly</li>
+ * <pre> srcDb | srcDataSet | anomalType </pre>
+ * <li>ModelType.Publish</li>
+ * <pre> publishUrl </pre>
+ */
+ // FIXME should model this explicitly, ASAP.
+ @Property("modelContent")
+ private String modelContent;
+
+ // @See SystemType
+ @Property("system")
+ private int system;
+
+ /**
+ * for a AnormalyDetection model need a reference model, say a count_model. So a count mound will
+ * hold a reference to Anomaly model.
+ */
+ @Property("referenceModel")
+ private String referenceModel;
+
+ @Property("timestamp")
+ private long timestamp;
+
+ @Property("starttime")
+ private long starttime;
+
+ public String getModelId() {
+ return modelId;
+ }
+
+ public void setModelId(String modelId) {
+ this.modelId = modelId;
+ }
+
+ public String getModelName() {
+ return modelName;
+ }
+
+ public void setModelName(String modelName) {
+ this.modelName = modelName;
+ }
+
+ public int getModelType() {
+ return modelType;
+ }
+
+ public void setModelType(int modelType) {
+ this.modelType = modelType;
+ }
+
+ public String getModelDesc() {
+ return modelDesc;
+ }
+
+ public void setModelDesc(String modelDesc) {
+ this.modelDesc = modelDesc;
+ }
+
+ public long getAssetId() {
+ return assetId;
+ }
+
+ public void setAssetId(long assetId) {
+ this.assetId = assetId;
+ }
+
+ public float getThreshold() {
+ return threshold;
+ }
+
+ public void setThreshold(float threshold) {
+ this.threshold = threshold;
+ }
+
+ public String getNotificationEmail() {
+ return notificationEmail;
+ }
+
+ public void setNotificationEmail(String notificationEmail) {
+ this.notificationEmail = notificationEmail;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getModelContent() {
+ return modelContent;
+ }
+
+ public void setModelContent(String modelContent) {
+ this.modelContent = modelContent;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public int getSchedule() {
+ return schedule;
+ }
+
+ public void setSchedule(int schedule) {
+ this.schedule = schedule;
+ }
+
+ public int getSystem() {
+ return system;
+ }
+
+ public void setSystem(int system) {
+ this.system = system;
+ }
+
+ public String getAssetName() {
+ return assetName;
+ }
+
+ public void setAssetName(String assetName) {
+ this.assetName = assetName;
+ }
+
+ public String getReferenceModel() {
+ return referenceModel;
+ }
+
+ public void setReferenceModel(String referenceModel) {
+ this.referenceModel = referenceModel;
+ }
+
+ public long getStarttime() {
+ return starttime;
+ }
+
+ public void setStarttime(long starttime) {
+ this.starttime = starttime;
+ }
+
+
+
+
+}