You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/05/26 09:17:39 UTC
[9/9] incubator-griffin git commit: [GRIFFIN-19] update document of
docker
[GRIFFIN-19] update document of docker
- unit test of measure
- update the document of docker and ui
- update the docker directory to new version
Author: Liu <ll...@ebay.com>
Author: Liu <ll...@lm-shc-16501428.corp.ebay.com>
Author: Lionel Liu <bh...@163.com>
Author: Liu <ll...@lm-shc-16501428.dhcp>
Author: bhlx3lyx7 <bh...@163.com>
Closes #39 from bhlx3lyx7/master.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/2de6a549
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/2de6a549
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/2de6a549
Branch: refs/heads/master
Commit: 2de6a5497201d2288edfa8ad994ecd3b2e919e5f
Parents: ae2b136
Author: Liu <ll...@ebay.com>
Authored: Fri May 26 17:17:29 2017 +0800
Committer: William Guo <gu...@icloud.com>
Committed: Fri May 26 17:17:29 2017 +0800
----------------------------------------------------------------------
README.md | 237 +-
docker/.gitignore | 33 -
docker/griffin-base/Dockerfile | 81 -
docker/griffin-base/config/hive-site.xml | 3916 -
docker/griffin-base/config/mongodb-org-3.2.repo | 5 -
docker/griffin-base/config/mysql_init.sh | 8 -
.../config/mysql_init_metastore.sql | 9 -
docker/griffin-base/config/mysql_secure.sh | 94 -
docker/griffin-base/config/spark-conf-mod.sh | 5 -
docker/griffin-base/config/tomcat | 26 -
.../griffin/dataFile/users_info_src.dat | 50 -
.../griffin/dataFile/users_info_target.dat | 50 -
docker/griffin-base/griffin/hadoop-end.sh | 8 -
docker/griffin-base/griffin/hadoop-start.sh | 23 -
docker/griffin-base/griffin/hd-after-hive.sh | 5 -
docker/griffin-base/griffin/hd-before-hive.sh | 7 -
docker/griffin-base/griffin/hd-test-json.sh | 7 -
docker/griffin-base/griffin/hive-init.sh | 19 -
docker/griffin-base/griffin/hive-input.hql | 25 -
.../griffin/jsonFile/accu_config.json | 63 -
.../griffin/jsonFile/vali_config.json | 135 -
docker/griffin-base/griffin/pre-start.sh | 16 -
docker/griffin-base/start-up.sh | 35 -
docker/griffin-base/user/adduser.sh | 40 -
docker/griffin-base/user/env.txt | 13 -
docker/griffin-base/user/sudouser.sh | 7 -
docker/griffin/Dockerfile | 27 -
docker/griffin/db/unitdb0/SEQUENCES.bson | Bin 326 -> 0 bytes
.../griffin/db/unitdb0/SEQUENCES.metadata.json | 1 -
docker/griffin/db/unitdb0/data_assets.bson | Bin 2404 -> 0 bytes
.../db/unitdb0/data_assets.metadata.json | 1 -
docker/griffin/db/unitdb0/dq_job.bson | 0
docker/griffin/db/unitdb0/dq_job.metadata.json | 1 -
.../griffin/db/unitdb0/dq_metrics_values.bson | Bin 502940 -> 0 bytes
.../db/unitdb0/dq_metrics_values.metadata.json | 1 -
.../db/unitdb0/dq_missed_file_path_lkp.bson | Bin 4550 -> 0 bytes
.../dq_missed_file_path_lkp.metadata.json | 1 -
docker/griffin/db/unitdb0/dq_model.bson | 0
.../griffin/db/unitdb0/dq_model.metadata.json | 1 -
docker/griffin/db/unitdb0/dq_schedule.bson | 0
.../db/unitdb0/dq_schedule.metadata.json | 1 -
docker/griffin/db/unitdb0/system.indexes.bson | Bin 603 -> 0 bytes
docker/griffin/db/unitdb0/user_subscribe.bson | Bin 255 -> 0 bytes
.../db/unitdb0/user_subscribe.metadata.json | 1 -
docker/griffin/download-resources.sh | 63 -
docker/griffin/hadoop-end.sh | 8 -
docker/griffin/hadoop-start.sh | 23 -
docker/griffin/pre-start.sh | 20 -
docker/griffin/script/env.sh | 4 -
docker/griffin/script/griffin_jobs.sh | 88 -
docker/griffin/script/griffin_regular_run.sh | 16 -
docker/griffin/start-up.sh | 51 -
docker/griffin_demo/Dockerfile | 15 +
docker/griffin_demo/bootstrap-all.sh | 52 +
docker/griffin_demo/prep/data/create-table.hql | 24 +
docker/griffin_demo/prep/data/demo_src | 125000 ++++++++++++++++
docker/griffin_demo/prep/data/demo_tgt | 125000 ++++++++++++++++
docker/griffin_demo/prep/data/gen-es-data.sh | 64 +
docker/griffin_demo/prep/data/gen-hive-data.sh | 30 +
.../prep/data/insert-data.hql.template | 2 +
.../jar/griffin-measure-batch.jar.placeholder | 0
docker/griffin_demo/prep/job/config.json | 27 +
docker/griffin_demo/prep/job/env.json.template | 27 +
docker/griffin_demo/prep/prepare.sh | 20 +
.../config/application.properties.template | 22 +
.../prep/service/service.jar.placeholder | 0
docker/griffin_env/Dockerfile | 71 +
docker/griffin_env/bootstrap-all.sh | 39 +
docker/griffin_env/bootstrap.sh | 25 +
.../conf/elasticsearch/elasticsearch | 207 +
.../conf/elasticsearch/elasticsearch.yml | 11 +
.../conf/hadoop/core-site.xml.template | 29 +
docker/griffin_env/conf/hadoop/hadoop-env.sh | 99 +
docker/griffin_env/conf/hadoop/hdfs-site.xml | 44 +
.../conf/hadoop/mapred-site.xml.template | 32 +
docker/griffin_env/conf/hadoop/slaves | 1 +
.../conf/hadoop/yarn-site.xml.template | 65 +
.../conf/hive/hive-site.xml.template | 3911 +
docker/griffin_env/conf/livy/livy.conf | 3 +
docker/griffin_env/conf/mysql/bind_0.cnf | 2 +
.../conf/mysql/hive-metastore-init.sql | 7 +
docker/griffin_env/conf/mysql/mysql-init.sh | 10 +
.../conf/mysql/quartz-metastore-init.sql | 6 +
.../conf/mysql/quartz-table-init.sql | 180 +
docker/griffin_env/conf/spark/slaves | 19 +
.../griffin_env/conf/spark/spark-defaults.conf | 32 +
docker/griffin_env/conf/spark/spark-env.sh | 75 +
docker/griffin_env/prep/dir.sh | 17 +
docker/griffin_env/prep/hdfs_file.sh | 29 +
docker/griffin_env/prep/software-config.sh | 35 +
docker/griffin_env/prep/software-install.sh | 38 +
docker/griffin_env/ssh_config | 5 +
griffin-doc/dockerUIguide.md | 78 +-
griffin-doc/userDataCaseGuide.md | 58 -
.../griffin/measure/batch/Application.scala | 13 +-
.../measure/batch/algo/BatchAccuracyAlgo.scala | 33 +-
.../measure/batch/algo/core/AccuracyCore.scala | 17 +-
.../config/validator/AllParamValidator.scala | 4 +-
.../batch/connector/AvroDataConnector.scala | 22 +-
.../measure/batch/connector/CacheDataUtil.scala | 63 -
.../batch/connector/DataConnectorFactory.scala | 14 +-
.../batch/connector/HiveDataConnector.scala | 22 +-
.../measure/batch/persist/HdfsPersist.scala | 6 +-
.../measure/batch/persist/HttpPersist.scala | 1 +
.../measure/batch/persist/MultiPersists.scala | 1 +
.../measure/batch/persist/PersistFactory.scala | 1 +
.../measure/batch/result/AccuracyResult.scala | 2 +-
.../measure/batch/rule/CalculationUtil.scala | 272 +
.../measure/batch/rule/ExprValueUtil.scala | 75 +
.../measure/batch/rule/RuleAnalyzer.scala | 62 +-
.../griffin/measure/batch/rule/RuleParser.scala | 85 +-
.../batch/rule/expr/AnalyzableExpr.scala | 2 +-
.../measure/batch/rule/expr/LiteralExpr.scala | 7 +-
.../measure/batch/rule/expr/LogicalExpr.scala | 10 +-
.../measure/batch/rule/expr/MathExpr.scala | 2 +-
.../measure/batch/rule/expr/SelectExpr.scala | 6 +-
.../measure/batch/rule/expr/StatementExpr.scala | 4 +-
.../measure/batch/utils/CalculationUtil.scala | 265 -
.../measure/batch/utils/StringParseUtil.scala | 10 -
.../batch/algo/BatchAccuracyAlgoTest.scala | 381 +-
.../batch/algo/core/AccuracyCoreTest.scala | 71 +
.../config/reader/ParamFileReaderTest.scala | 34 -
.../reader/ParamRawStringReaderTest.scala | 20 +
.../validator/AllParamValidatorTest.scala | 22 +
.../measure/batch/persist/HdfsPersistTest.scala | 30 +
.../measure/batch/persist/HttpPersistTest.scala | 36 +-
.../batch/result/AccuracyResultTest.scala | 39 +
.../measure/batch/rule/RuleAnalyzerTest.scala | 46 +
.../measure/batch/rule/RuleFactoryTest.scala | 26 +
.../measure/batch/rule/RuleParserTest.scala | 203 +-
.../measure/batch/utils/JsonUtilTest.scala | 42 +
measure/pom.xml | 23 +-
.../core/measure/MeasureServiceImpl.java | 2 +
.../griffin/core/metric/MetricServiceImpl.java | 2 +
134 files changed, 256501 insertions(+), 6111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/2de6a549/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 152086d..0eee2b7 100644
--- a/README.md
+++ b/README.md
@@ -11,204 +11,89 @@ Apache Griffin is a model driven Data Quality solution for distributed data syst
### Repository
-Snapshot:
+Snapshot:
-Release:
-
-### How to build docker
-1. git clone the repository of https://github.com/apache/incubator-griffin
-2. run "mvn install"
+Release:
### How to run in docker
1. Install [docker](https://www.docker.com/).
-2. Pull our built docker image, and tag it griffin-env.
+2. Pull our built docker image.
```
- docker pull bhlx3lyx7/griffin-env
- ```
+ docker pull bhlx3lyx7/griffin_demo:0.1.0
```
- docker tag bhlx3lyx7/griffin-env griffin-env
+3. Run this docker image, then griffin is ready.
```
-3. Run docker image griffin-env, then the backend is ready.
+ docker run -it -h sandbox --name griffin_demo -m 8G --memory-swap -1 \
+ -p 32122:2122 -p 37077:7077 -p 36066:6066 -p 38088:8088 -p 38040:8040 \
+ -p 33306:3306 -p 39000:9000 -p 38042:8042 -p 38080:8080 -p 37017:27017 \
+ -p 39083:9083 -p 38998:8998 -p 39200:9200 bhlx3lyx7/griffin_demo:0.1.0
```
- docker run -it -h sandbox --name griffin -m 8G --memory-swap -1 \
- -p 2122:2122 -p 47077:7077 -p 46066:6066 -p 48088:8088 -p 48040:8040 \
- -p 48042:8042 -p 48080:8080 -p 47017:27017 griffin-env bash
+4. Now you can visit UI through your browser, login with account "test" and password "test" if required.
```
- You can also drop the tail "bash" of the command above, then you will get tomcat service log printing in docker only.
-
-4. Now you can visit UI through your browser, and follow the next steps on web UI [here](https://github.com/eBay/griffin/tree/master/griffin-doc/dockerUIguide.md#webui-test-case-guide). You can login with account "test" and password "test" if required.
+ http://<your local IP address>:38080/
```
- http://<your local IP address>:48080/
+ You can also follow the steps using UI [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/dockerUIguide.md#webui-test-case-guide).
+
+### How to deploy and run at local
+1. Install jdk (1.8 or later versions).
+2. Install mysql.
+2. Install [Hadoop](http://apache.claz.org/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz) (2.6.0 or later), you can get some help [here](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/SingleCluster.html).
+3. Install [Spark](http://spark.apache.org/downloads.html) (version 1.6.x, griffin does not support 2.0.x at current), if you want to install Pseudo Distributed/Single Node Cluster, you can get some help [here](http://why-not-learn-something.blogspot.com/2015/06/spark-installation-pseudo.html).
+4. Install [Hive](http://apache.claz.org/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz) (version 1.2.1 or later), you can get some help [here](https://cwiki.apache.org/confluence/display/Hive/GettingStarted#GettingStarted-RunningHive).
+ You need to make sure that your spark cluster could access your HiveContext.
+5. Install [Livy](http://archive.cloudera.com/beta/livy/livy-server-0.3.0.zip), you can get some help [here](http://livy.io/quickstart.html).
+ Griffin need to schedule spark jobs by server, we use livy to submit our jobs.
+ For some issues of Livy for HiveContext, we need to download 3 files, and put them into Hdfs.
```
- And you can also ssh to the docker container using account "griffin" with password "griffin".
+ datanucleus-api-jdo-3.2.6.jar
+ datanucleus-core-3.2.10.jar
+ datanucleus-rdbms-3.2.9.jar
```
- ssh griffin@<your local IP address> -p 2122
+6. Install [ElasticSearch]().
+ ElasticSearch works as a metrics collector, Griffin produces metrics to it, and our default UI get metrics from it, you can use your own way as well.
+7. Modify configuration for your environment.
+ You need to modify the configuration part of code, to make Griffin works well in you environment.
+ service/src/main/resources/application.properties
```
+ spring.datasource.url = jdbc:mysql://<your IP>:3306/quartz?autoReconnect=true&useSSL=false
+ spring.datasource.username = <user name>
+ spring.datasource.password = <password>
-### How to deploy and run at local
-1. Install jdk (1.7 or later versions)
-2. Install Tomcat (7.0 or later versions)
-3. Install MongoDB and import the collections
- ```
- mongorestore /db:unitdb0 /dir:<dir of griffin-doc>/db/unitdb0
- ```
-
-4. Install [Hadoop](http://apache.claz.org/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz) (2.6.0 or later), you can get some help [here](https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/SingleCluster.html).
- Make sure you have the permission to use command "hadoop".
- Create an empty directory in hdfs as your hdfs path, and then create running and history directory in it
- ```
- hadoop fs -mkdir <your hdfs path>
- hadoop fs -mkdir <your hdfs path>/running
- hadoop fs -mkdir <your hdfs path>/history
- ```
-5. Install [Spark](http://spark.apache.org/downloads.html) (version 1.6.x, griffin does not support 2.0.x at current), if you want to install Pseudo Distributed/Single Node Cluster, you can get some help [here](http://why-not-learn-something.blogspot.com/2015/06/spark-installation-pseudo.html).
- Make sure you have the permission to use command "spark-shell".
-6. Install [Hive](http://apache.claz.org/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz) (version 1.2.1 or later), you can get some help [here](https://cwiki.apache.org/confluence/display/Hive/GettingStarted#GettingStarted-RunningHive).
- Make sure you have the permission to use command "hive".
-7. Create a working directory, and it will be **your local path** now.
-8. In your local path, put your data into Hive.
- First, you need to create some directories in hdfs.
- ```
- hadoop fs -mkdir /tmp
- hadoop fs -mkdir /user/hive/warehouse
- hadoop fs -chmod g+w /tmp
- hadoop fs -chmod g+w /user/hive/warehouse
- ```
- Then, run the following command in **your local path**
+ hive.metastore.uris = thrift://<your IP>:9083
+ hive.metastore.dbname = <hive database name> # default is "default"
```
- schematool -dbType derby -initSchema
+ service/src/main/resources/sparkJob.properties
```
- Now you can put your data into Hive by running "hive" here. You can get sample data [here](https://github.com/eBay/griffin/tree/master/griffin-doc/hive), then put into hive as following commands
-
+ sparkJob.file = hdfs://<griffin measure path>/griffin-measure-batch.jar
+ sparkJob.args_1 = hdfs://<griffin env path>/env.json
+ sparkJob.jars_1 = hdfs://<datanucleus path>/datanucleus-api-jdo-3.2.6.jar
+ sparkJob.jars_2 = hdfs://<datanucleus path>/datanucleus-core-3.2.10.jar
+ sparkJob.jars_3 = hdfs://<datanucleus path>/datanucleus-rdbms-3.2.9.jar
+ sparkJob.uri = http://<your IP>:8998/batches
```
- CREATE TABLE users_info_src (
- user_id bigint,
- first_name string,
- last_name string,
- address string,
- email string,
- phone string,
- post_code string)
- ROW FORMAT DELIMITED
- FIELDS TERMINATED BY '|'
- STORED AS TEXTFILE;
-
- LOAD DATA LOCAL INPATH '<your data path>/users_info_src.dat' OVERWRITE INTO TABLE users_info_src;
-
- CREATE TABLE users_info_target (
- user_id bigint,
- first_name string,
- last_name string,
- address string,
- email string,
- phone string,
- post_code string)
- ROW FORMAT DELIMITED
- FIELDS TERMINATED BY '|'
- STORED AS TEXTFILE;
-
- LOAD DATA LOCAL INPATH '<your data path>/users_info_target.dat' OVERWRITE INTO TABLE users_info_target;
- ```
-
- If you use hive command mode to input data, remember to create _SUCCESS file in hdfs table path as following
-
- ```
- hadoop fs -touchz /user/hive/warehouse/users_info_src/_SUCCESS
- hadoop fs -touchz /user/hive/warehouse/users_info_target/_SUCCESS
- ```
-9. You can create your own model by modifying code.
- (If you want to use our default models, please skip this step)
-10. Currently we need to run the jobs automatically by script files, you need to set your own parameters in the script files and run it.
- You can edit the [demo script files](https://github.com/eBay/griffin/tree/master/griffin-doc/hive/script/) as following.
-
- [env.sh](https://github.com/eBay/griffin/blob/master/docker/griffin/script/env.sh)
- ```
- HDFS_WORKDIR=<your hdfs path>/running
- ```
-
- [griffin_jobs.sh](https://github.com/eBay/griffin/blob/master/docker/griffin/script/griffin_jobs.sh)
- ```
- spark-submit --class org.apache.griffin.accuracy.Accu --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
- spark-submit --class org.apache.griffin.validility.Vali --master yarn-client --queue default --executor-memory 1g --num-executors 4 $GRIFFIN_HOME/griffin-models.jar $lv1dir/cmd.txt $lv1dir/ >> $logfile 2>&1
- ```
-
- These commands submit the jobs to spark, if you want to try your own model or modify some parameters, please edit it.
- If you want to use your own model, change "$GRIFFIN_HOME/griffin-models.jar" to "your path/your model.jar", and change the class name.
-
- Put these script files in **your local path**.
-
-11. Open [application.properties](https://github.com/eBay/griffin/tree/master/griffin-core/src/main/resources/application.properties) file, read the comments and specify the properties correctly. Or you can edit it as following.
- ```
- env=prod
- job.local.folder=<your local path>/tmp
- job.hdfs.folder=<your hdfs path>
- job.hdfs.runningfoldername=running
- job.hdfs.historyfoldername=history
- ```
- If you set the properties as above, you need to make sure the directory "tmp" exists in your local path
-12. Build the whole project and deploy.
- ```
- mvn install -DskipTests
- ```
-
- Find the griffin-models-0.1.0-SNAPSHOT.jar in path griffin-models/target, rename it to griffin-models.jar (or your model.jar), and put in **your local path**.
- Till now, please make sure that, in **your local path**, there exists griffin-modes.jar, env.sh, griffin_jobs.sh and griffin_regular_run.sh.
- Run griffin_regular_run.sh as following.
- ```
- nohup ./griffin_regular_run.sh &
- ```
-
- Deploy griffin-core/target/ROOT.war to tomcat, start tomcat server, then you can follow the web UI steps [here](https://github.com/eBay/griffin/blob/master/griffin-doc/dockerUIguide.md#webui-test-case-guide).
-13. You can also review the RESTful APIs through http://localhost:8080/api/v1/application.wadl
-
-### How to develop
-In dev environment, you can run backend REST service and frontend UI seperately. The majority of the backend code logics are in the [service](https://github.com/apache/incubator-griffin/tree/master/service) project. So, to start backend, please import maven project Griffin into eclipse, ***GriffinWebApplication as Spring Boot App***
-
-To start frontend, please follow up the below steps.
-
-1. Open **ui/js/services/services.js** file
-
-2. Specify **BACKEND_SERVER** to your real backend server address, below is an example
-
+ ui/js/services/services.js
```
- var BACKEND_SERVER = 'http://localhost:8080'; //dev env
+ ES_SERVER = "http://<your IP>:9200"
```
-3. Specify some variables like mysql, hive and kafka connectors in your properies file under service/src/main/resources/application.properties
-
+ Configure measure/measure-batch/src/main/resources/env.json for your environment, and put it into Hdfs <griffin env path>/
+8. Build the whole project and deploy.
```
- spring.datasource.url= jdbc:mysql://localhost:3306/metastore
- spring.datasource.username =griffin
- spring.datasource.password =123456
-
- spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-
- ## Hibernate ddl auto (validate,create, create-drop, update)
-
- spring.jpa.hibernate.ddl-auto = create-drop
- spring.jpa.show-sql=true
- spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5Dialect
- #
- #
- ## Naming strategy
- spring.jpa.hibernate.naming-strategy = org.hibernate.cfg.ImprovedNamingStrategy
-
- # hive metastore
- hive.metastore.uris = thrift://localhost:9083
- hive.metastore.dbname = default
-
- # kafka schema registry
- kafka.schema.registry.url = http://localhost:8081
+ mvn install
```
+ Create a directory in Hdfs, and put our measure package into it.
+ ```
+ hdfs dfs -put /measure/measure-batch/target/griffin-measure-batch.jar <griffin measure path>/
+ ```
+ After all our environment services startup, we can start our server.
+ ```
+ java -jar service/target/service.jar
+ ```
+ After a few seconds, we can visit our default UI of Griffin (by default the port of spring boot is 8080).
+ ```
+ http://<your IP>:8080
+ ```
+9. Follow the steps using UI [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/dockerUIguide.md#webui-test-case-guide).
-4. Open a command line, run the below commands in root directory
-
- - mvn clean install
-
-5. Find the GriffinWebApplication,
-
- - run as spring boot application
-
-6. In your browser, open http://localhost:8080 ,enjoy your journey!
**Note**: The front-end UI is still under development, you can only access some basic features currently.
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/2de6a549/docker/.gitignore
----------------------------------------------------------------------
diff --git a/docker/.gitignore b/docker/.gitignore
deleted file mode 100644
index c59e858..0000000
--- a/docker/.gitignore
+++ /dev/null
@@ -1,33 +0,0 @@
-# use glob syntax.
-syntax: glob
-*.ser
-*.class
-*~
-*.bak
-#*.off
-*.old
-
-# eclipse conf file
-.settings
-.classpath
-.project
-.manager
-.scala_dependencies
-.cache-main
-.cache-tests
-
-# idea
-.idea
-*.iml
-
-# other scm
-.svn
-.CVS
-.hg*
-
-# switch to regexp syntax.
-# syntax: regexp
-# ^\.pc/
-
-#SHITTY output not in target directory
-build.log
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/2de6a549/docker/griffin-base/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/griffin-base/Dockerfile b/docker/griffin-base/Dockerfile
deleted file mode 100755
index 03ac0f6..0000000
--- a/docker/griffin-base/Dockerfile
+++ /dev/null
@@ -1,81 +0,0 @@
-FROM sequenceiq/spark:1.6.0
-MAINTAINER com.ebay.oss
-
-#add user
-RUN rpm -e cracklib-dicts --nodeps && yum install -y cracklib-dicts
-ADD user/*.sh /root/
-WORKDIR /root
-RUN ./adduser.sh griffin griffin && ./sudouser.sh griffin && rm *.sh
-ENV GRIFFIN_HOME /home/griffin
-
-#set java environment variables
-ENV JAVA_HOME /usr/java/latest
-ENV PATH $JAVA_HOME/bin:$PATH
-
-#install wget
-RUN yum install -y wget
-
-#enter /apache
-RUN mkdir /apache
-
-#install hive 1.2.1 and set environment variables
-RUN cd /apache && wget https://www.apache.org/dist/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz && tar -xvf apache-hive-1.2.1-bin.tar.gz && ln -s apache-hive-1.2.1-bin hive
-ENV HIVE_HOME /apache/hive
-ENV PATH $HIVE_HOME/bin:$PATH
-
-#running HiveServer2 and Beeline
-ENV HADOOP_USER_CLASSPATH_FIRST true
-RUN rm /usr/local/hadoop-2.6.0/share/hadoop/yarn/lib/jline-0.9.94.jar
-
-#mkdir
-ADD griffin $GRIFFIN_HOME
-RUN chmod -R 755 $GRIFFIN_HOME
-
-#install tomcat 7
-RUN cd /apache && wget https://www.apache.org/dist/tomcat/tomcat-7/v7.0.73/bin/apache-tomcat-7.0.73.tar.gz && tar -xvf apache-tomcat-7.0.73.tar.gz && ln -s apache-tomcat-7.0.73 tomcat
-ADD config/tomcat /etc/init.d/
-RUN chmod 755 /etc/init.d/tomcat
-ENV TOMCAT_HOME /apache/tomcat
-ENV PATH $TOMCAT_HOME/bin:$PATH
-
-#install mongodb
-ADD config/mongodb-org-3.2.repo /etc/yum.repos.d/
-RUN yum install -y mongodb-org mongodb-org-server mongodb-org-shell mongodb-org-mongos mongodb-org-tools && yum clean all
-
-#expose ports
-EXPOSE 8080 27017 6066 2122 9083 3306
-
-#env
-ENV HADOOP_HOME /usr/local/hadoop
-ENV PATH $PATH:$HADOOP_HOME/bin
-
-#input hadoop data
-WORKDIR $GRIFFIN_HOME
-RUN ./hadoop-start.sh && ./pre-start.sh && ./hd-before-hive.sh && ./hd-after-hive.sh && ./hd-test-json.sh && ./hadoop-end.sh
-
-#install mysql
-ADD config/mysql_* $GRIFFIN_HOME/
-RUN chmod 755 $GRIFFIN_HOME/mysql_*
-RUN yum install -y mysql-server && yum install -y mysql-connector-java && ln -s /usr/share/java/mysql-connector-java.jar $HIVE_HOME/lib/mysql-connector-java.jar
-
-#configure hive metastore as remote mode
-ADD config/hive-site.xml $HIVE_HOME/conf/
-ADD config/hive-site.xml $SPARK_HOME/conf/
-RUN chmod 664 $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf/hive-site.xml
-
-#prepare env data
-RUN ./hadoop-start.sh && ./hive-init.sh && ./hadoop-end.sh && rm hadoop-start.sh pre-start.sh hd-before-hive.sh hd-after-hive.sh hd-test-json.sh hadoop-end.sh hive-init.sh
-
-#modify spark webui port
-ADD config/spark-conf-mod.sh $GRIFFIN_HOME/
-RUN ./spark-conf-mod.sh && rm spark-conf-mod.sh
-
-#edit profile
-ADD user/env.txt $GRIFFIN_HOME/
-RUN cat ./env.txt >> /etc/profile
-
-#input start-up.sh
-ADD start-up.sh $GRIFFIN_HOME/
-
-#entry point
-ENTRYPOINT ["./start-up.sh"]