You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2022/09/30 08:08:40 UTC
[griffin] 01/01: init griffin new data quality workflow
This is an automated email from the ASF dual-hosted git repository.
guoyp pushed a commit to branch griffin-1.0.0-dev
in repository https://gitbox.apache.org/repos/asf/griffin.git
commit 5978a103297fc180c859f6c21d5b8a62947a2213
Author: William <gu...@apache.org>
AuthorDate: Fri Sep 30 16:07:31 2022 +0800
init griffin new data quality workflow
---
anomalydetection/pom.xml | 409 +++++++++++++++++++++
.../src/main/resources/assembly/assembly.xml | 84 +++++
.../src/test/resources/application.properties | 77 ++++
core/pom.xml | 409 +++++++++++++++++++++
.../apache/griffin/core/GriffinWebApplication.java | 40 ++
core/src/main/resources/assembly/assembly.xml | 84 +++++
core/src/test/resources/application.properties | 77 ++++
dispatcher/pom.xml | 409 +++++++++++++++++++++
.../src/main/resources/assembly/assembly.xml | 84 +++++
.../src/test/resources/application.properties | 77 ++++
pom.xml | 9 +-
scheduler/pom.xml | 409 +++++++++++++++++++++
scheduler/src/main/resources/assembly/assembly.xml | 84 +++++
.../src/test/resources/application.properties | 77 ++++
14 files changed, 2325 insertions(+), 4 deletions(-)
diff --git a/anomalydetection/pom.xml b/anomalydetection/pom.xml
new file mode 100644
index 00000000..2a501a48
--- /dev/null
+++ b/anomalydetection/pom.xml
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>griffin</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>anomalydetection</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Griffin :: Anomaly Detection</name>
+
+ <properties>
+ <hadoop.version>2.7.1</hadoop.version>
+ <hive.version>2.2.0</hive.version>
+ <spring.boot.version>2.1.7.RELEASE</spring.boot.version>
+ <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version>
+ <confluent.version>3.2.0</confluent.version>
+ <quartz.version>2.2.2</quartz.version>
+ <start-class>org.apache.griffin.core.GriffinWebApplication</start-class>
+ <powermock.version>2.0.2</powermock.version>
+ <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version>
+ <derby.version>10.14.1.0</derby.version>
+ <eclipselink.version>2.6.0</eclipselink.version>
+ <mysql.java.version>5.1.47</mysql.java.version>
+ <postgresql.version>42.4.1</postgresql.version>
+ <livy.core.version>0.3.0</livy.core.version>
+ <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version>
+ <jackson-databind.version>2.9.9.3</jackson-databind.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>http://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-log4j2</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-properties-migrator</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hibernate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aspects</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.kerberos</groupId>
+ <artifactId>spring-security-kerberos-client</artifactId>
+ <version>${spring.security.kerberos.version}</version>
+ </dependency>
+ <!--eclipse link-->
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>mysql</groupId>-->
+ <!--<artifactId>mysql-connector-java</artifactId>-->
+ <!--<version>${mysql.java.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.retry</groupId>
+ <artifactId>spring-retry</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
+ </dependency>
+
+ <!-- to access metastore from hive-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <!--<scope>provided</scope>-->
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access Hive using JDBC -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access confluent schema registry -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>${confluent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!--schedule-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <!--livy-core-->
+ <dependency>
+ <groupId>com.cloudera.livy</groupId>
+ <artifactId>livy-core_2.10</artifactId>
+ <version>${livy.core.version}</version>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch-rest-client.version}</version>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <!--if you need mysql, please uncomment mysql-connector-java -->
+ <!--<profile>-->
+ <!--<id>mysql</id>-->
+ <!--<activation>-->
+ <!--<property>-->
+ <!--<name>mysql</name>-->
+ <!--</property>-->
+ <!--</activation>-->
+ <!--</profile>-->
+ <profile>
+ <id>dev</id>
+ <activation>
+ <property>
+ <name>dev</name>
+ </property>
+ </activation>
+ </profile>
+ <profile>
+ <id>postgresql</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>prod</name>
+ </property>
+ </activation>
+ </profile>
+ </profiles>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.ethlo.persistence.tools</groupId>
+ <artifactId>eclipselink-maven-plugin</artifactId>
+ <version>2.7.0</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>weave</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>lib</classifier>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring-boot-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>build-info</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>false</executable>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <skipAssembly>false</skipAssembly>
+ <outputDirectory>../target</outputDirectory>
+ <descriptors>
+ <descriptor>src/main/resources/assembly/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/anomalydetection/src/main/resources/assembly/assembly.xml b/anomalydetection/src/main/resources/assembly/assembly.xml
new file mode 100644
index 00000000..70af6651
--- /dev/null
+++ b/anomalydetection/src/main/resources/assembly/assembly.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 ">
+ <id>${project.version}</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+
+ <includeBaseDirectory>true</includeBaseDirectory>
+
+ <!-- Adds dependencies to zip package under lib directory -->
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>README*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>config/</outputDirectory>
+ <includes>
+ <include>env/*</include>
+ <include>*.xml</include>
+ <include>*.sql</include>
+ <include>*.json</include>
+ <include>*.properties</include>
+ </includes>
+ <excludes>
+ <exclude>*.txt</exclude>
+ <exclude>logback-test.xml</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources/shell</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>*.*</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <outputDirectory>./lib</outputDirectory>
+ <includes>
+ <include>*${version}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
diff --git a/anomalydetection/src/test/resources/application.properties b/anomalydetection/src/test/resources/application.properties
new file mode 100644
index 00000000..81da26e3
--- /dev/null
+++ b/anomalydetection/src/test/resources/application.properties
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+spring.jpa.generate-ddl=true
+spring.datasource.driver-class-name=org.h2.Driver
+spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1
+spring.datasource.username=test
+spring.datasource.password=test
+spring.datasource.schema=classpath:Init_quartz_h2_test.sql
+spring.jpa.show-sql=true
+# Hive metastore
+hive.metastore.uris=thrift://localhost:39083
+hive.metastore.dbname=default
+hive.hmshandler.retry.attempts=15
+hive.hmshandler.retry.interval=2000ms
+# Hive cache time
+cache.evict.hive.fixedRate.in.milliseconds=900000
+# Kafka schema registry
+kafka.schema.registry.url=http://localhost:8081
+# Update job instance state at regular intervals
+jobInstance.fixedDelay.in.milliseconds=60000
+# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds
+jobInstance.expired.milliseconds=604800000
+# schedule predicate job every 5 minutes and repeat 12 times at most
+#interval time unit m:minute h:hour d:day,only support these three units
+predicate.job.interval=5m
+predicate.job.repeat.count=12
+# external properties directory location
+external.config.location=
+# external BATCH or STREAMING env
+external.env.location=
+# login strategy ("default" or "ldap")
+login.strategy=default
+# ldap
+ldap.url=ldap://hostname:port
+ldap.email=@example.com
+ldap.searchBase=DC=org,DC=example
+ldap.searchPattern=(sAMAccountName={0})
+# elasticsearch
+elasticsearch.host=localhost
+elasticsearch.port=9200
+elasticsearch.scheme=http
+# elasticsearch.user = user
+# elasticsearch.password = password
+# livy
+livy.uri=http://localhost:38998/batches
+livy.need.queue=false
+livy.task.max.concurrent.count=20
+livy.task.submit.interval.second=3
+livy.task.appId.retry.count=3
+livy.need.kerberos=false
+livy.server.auth.kerberos.principal=livy/kerberos.principal
+livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file
+# yarn url
+yarn.uri=http://localhost:38088
+
+# griffin event listener
+internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook
+
+logging.file=logs/griffin-service.log
+
+spring.main.allow-bean-definition-overriding=true
\ No newline at end of file
diff --git a/core/pom.xml b/core/pom.xml
new file mode 100644
index 00000000..57aadedf
--- /dev/null
+++ b/core/pom.xml
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>griffin</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>core</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Griffin :: Core</name>
+
+ <properties>
+ <hadoop.version>2.7.1</hadoop.version>
+ <hive.version>2.2.0</hive.version>
+ <spring.boot.version>2.1.7.RELEASE</spring.boot.version>
+ <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version>
+ <confluent.version>3.2.0</confluent.version>
+ <quartz.version>2.2.2</quartz.version>
+ <start-class>org.apache.griffin.core.GriffinWebApplication</start-class>
+ <powermock.version>2.0.2</powermock.version>
+ <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version>
+ <derby.version>10.14.1.0</derby.version>
+ <eclipselink.version>2.6.0</eclipselink.version>
+ <mysql.java.version>5.1.47</mysql.java.version>
+ <postgresql.version>42.4.1</postgresql.version>
+ <livy.core.version>0.3.0</livy.core.version>
+ <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version>
+ <jackson-databind.version>2.9.9.3</jackson-databind.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>http://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-log4j2</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-properties-migrator</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hibernate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aspects</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.kerberos</groupId>
+ <artifactId>spring-security-kerberos-client</artifactId>
+ <version>${spring.security.kerberos.version}</version>
+ </dependency>
+ <!--eclipse link-->
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>mysql</groupId>-->
+ <!--<artifactId>mysql-connector-java</artifactId>-->
+ <!--<version>${mysql.java.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.retry</groupId>
+ <artifactId>spring-retry</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
+ </dependency>
+
+ <!-- to access metastore from hive-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <!--<scope>provided</scope>-->
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access Hive using JDBC -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access confluent schema registry -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>${confluent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!--schedule-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <!--livy-core-->
+ <dependency>
+ <groupId>com.cloudera.livy</groupId>
+ <artifactId>livy-core_2.10</artifactId>
+ <version>${livy.core.version}</version>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch-rest-client.version}</version>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <!--if you need mysql, please uncomment mysql-connector-java -->
+ <!--<profile>-->
+ <!--<id>mysql</id>-->
+ <!--<activation>-->
+ <!--<property>-->
+ <!--<name>mysql</name>-->
+ <!--</property>-->
+ <!--</activation>-->
+ <!--</profile>-->
+ <profile>
+ <id>dev</id>
+ <activation>
+ <property>
+ <name>dev</name>
+ </property>
+ </activation>
+ </profile>
+ <profile>
+ <id>postgresql</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>prod</name>
+ </property>
+ </activation>
+ </profile>
+ </profiles>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.ethlo.persistence.tools</groupId>
+ <artifactId>eclipselink-maven-plugin</artifactId>
+ <version>2.7.0</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>weave</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>lib</classifier>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring-boot-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>build-info</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>false</executable>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <skipAssembly>false</skipAssembly>
+ <outputDirectory>../target</outputDirectory>
+ <descriptors>
+ <descriptor>src/main/resources/assembly/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
new file mode 100644
index 00000000..7c7ddb73
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+package org.apache.griffin.core;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+@SpringBootApplication
+@EnableScheduling
+public class GriffinWebApplication {
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(GriffinWebApplication.class);
+
+ public static void main(String[] args) {
+ SpringApplication.run(GriffinWebApplication.class, args);
+ LOGGER.info("application started");
+ }
+
+
+}
diff --git a/core/src/main/resources/assembly/assembly.xml b/core/src/main/resources/assembly/assembly.xml
new file mode 100644
index 00000000..70af6651
--- /dev/null
+++ b/core/src/main/resources/assembly/assembly.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 ">
+ <id>${project.version}</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+
+ <includeBaseDirectory>true</includeBaseDirectory>
+
+ <!-- Adds dependencies to zip package under lib directory -->
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>README*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>config/</outputDirectory>
+ <includes>
+ <include>env/*</include>
+ <include>*.xml</include>
+ <include>*.sql</include>
+ <include>*.json</include>
+ <include>*.properties</include>
+ </includes>
+ <excludes>
+ <exclude>*.txt</exclude>
+ <exclude>logback-test.xml</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources/shell</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>*.*</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <outputDirectory>./lib</outputDirectory>
+ <includes>
+ <include>*${version}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
diff --git a/core/src/test/resources/application.properties b/core/src/test/resources/application.properties
new file mode 100644
index 00000000..81da26e3
--- /dev/null
+++ b/core/src/test/resources/application.properties
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+spring.jpa.generate-ddl=true
+spring.datasource.driver-class-name=org.h2.Driver
+spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1
+spring.datasource.username=test
+spring.datasource.password=test
+spring.datasource.schema=classpath:Init_quartz_h2_test.sql
+spring.jpa.show-sql=true
+# Hive metastore
+hive.metastore.uris=thrift://localhost:39083
+hive.metastore.dbname=default
+hive.hmshandler.retry.attempts=15
+hive.hmshandler.retry.interval=2000ms
+# Hive cache time
+cache.evict.hive.fixedRate.in.milliseconds=900000
+# Kafka schema registry
+kafka.schema.registry.url=http://localhost:8081
+# Update job instance state at regular intervals
+jobInstance.fixedDelay.in.milliseconds=60000
+# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds
+jobInstance.expired.milliseconds=604800000
+# schedule predicate job every 5 minutes and repeat 12 times at most
+#interval time unit m:minute h:hour d:day,only support these three units
+predicate.job.interval=5m
+predicate.job.repeat.count=12
+# external properties directory location
+external.config.location=
+# external BATCH or STREAMING env
+external.env.location=
+# login strategy ("default" or "ldap")
+login.strategy=default
+# ldap
+ldap.url=ldap://hostname:port
+ldap.email=@example.com
+ldap.searchBase=DC=org,DC=example
+ldap.searchPattern=(sAMAccountName={0})
+# elasticsearch
+elasticsearch.host=localhost
+elasticsearch.port=9200
+elasticsearch.scheme=http
+# elasticsearch.user = user
+# elasticsearch.password = password
+# livy
+livy.uri=http://localhost:38998/batches
+livy.need.queue=false
+livy.task.max.concurrent.count=20
+livy.task.submit.interval.second=3
+livy.task.appId.retry.count=3
+livy.need.kerberos=false
+livy.server.auth.kerberos.principal=livy/kerberos.principal
+livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file
+# yarn url
+yarn.uri=http://localhost:38088
+
+# griffin event listener
+internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook
+
+logging.file=logs/griffin-service.log
+
+spring.main.allow-bean-definition-overriding=true
\ No newline at end of file
diff --git a/dispatcher/pom.xml b/dispatcher/pom.xml
new file mode 100644
index 00000000..fce4ed45
--- /dev/null
+++ b/dispatcher/pom.xml
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>griffin</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>dispatcher</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Griffin :: Dispatcher</name>
+
+ <properties>
+ <hadoop.version>2.7.1</hadoop.version>
+ <hive.version>2.2.0</hive.version>
+ <spring.boot.version>2.1.7.RELEASE</spring.boot.version>
+ <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version>
+ <confluent.version>3.2.0</confluent.version>
+ <quartz.version>2.2.2</quartz.version>
+ <start-class>org.apache.griffin.core.GriffinWebApplication</start-class>
+ <powermock.version>2.0.2</powermock.version>
+ <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version>
+ <derby.version>10.14.1.0</derby.version>
+ <eclipselink.version>2.6.0</eclipselink.version>
+ <mysql.java.version>5.1.47</mysql.java.version>
+ <postgresql.version>42.4.1</postgresql.version>
+ <livy.core.version>0.3.0</livy.core.version>
+ <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version>
+ <jackson-databind.version>2.9.9.3</jackson-databind.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>http://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-log4j2</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-properties-migrator</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hibernate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aspects</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.kerberos</groupId>
+ <artifactId>spring-security-kerberos-client</artifactId>
+ <version>${spring.security.kerberos.version}</version>
+ </dependency>
+ <!--eclipse link-->
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>mysql</groupId>-->
+ <!--<artifactId>mysql-connector-java</artifactId>-->
+ <!--<version>${mysql.java.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.retry</groupId>
+ <artifactId>spring-retry</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
+ </dependency>
+
+ <!-- to access metastore from hive-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <!--<scope>provided</scope>-->
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access Hive using JDBC -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access confluent schema registry -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>${confluent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!--schedule-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <!--livy-core-->
+ <dependency>
+ <groupId>com.cloudera.livy</groupId>
+ <artifactId>livy-core_2.10</artifactId>
+ <version>${livy.core.version}</version>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch-rest-client.version}</version>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <!--if you need mysql, please uncomment mysql-connector-java -->
+ <!--<profile>-->
+ <!--<id>mysql</id>-->
+ <!--<activation>-->
+ <!--<property>-->
+ <!--<name>mysql</name>-->
+ <!--</property>-->
+ <!--</activation>-->
+ <!--</profile>-->
+ <profile>
+ <id>dev</id>
+ <activation>
+ <property>
+ <name>dev</name>
+ </property>
+ </activation>
+ </profile>
+ <profile>
+ <id>postgresql</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>prod</name>
+ </property>
+ </activation>
+ </profile>
+ </profiles>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.ethlo.persistence.tools</groupId>
+ <artifactId>eclipselink-maven-plugin</artifactId>
+ <version>2.7.0</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>weave</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>lib</classifier>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring-boot-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>build-info</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>false</executable>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <skipAssembly>false</skipAssembly>
+ <outputDirectory>../target</outputDirectory>
+ <descriptors>
+ <descriptor>src/main/resources/assembly/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/dispatcher/src/main/resources/assembly/assembly.xml b/dispatcher/src/main/resources/assembly/assembly.xml
new file mode 100644
index 00000000..70af6651
--- /dev/null
+++ b/dispatcher/src/main/resources/assembly/assembly.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 ">
+ <id>${project.version}</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+
+ <includeBaseDirectory>true</includeBaseDirectory>
+
+ <!-- Adds dependencies to zip package under lib directory -->
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>README*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>config/</outputDirectory>
+ <includes>
+ <include>env/*</include>
+ <include>*.xml</include>
+ <include>*.sql</include>
+ <include>*.json</include>
+ <include>*.properties</include>
+ </includes>
+ <excludes>
+ <exclude>*.txt</exclude>
+ <exclude>logback-test.xml</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources/shell</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>*.*</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <outputDirectory>./lib</outputDirectory>
+ <includes>
+ <include>*${version}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
diff --git a/dispatcher/src/test/resources/application.properties b/dispatcher/src/test/resources/application.properties
new file mode 100644
index 00000000..81da26e3
--- /dev/null
+++ b/dispatcher/src/test/resources/application.properties
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+spring.jpa.generate-ddl=true
+spring.datasource.driver-class-name=org.h2.Driver
+spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1
+spring.datasource.username=test
+spring.datasource.password=test
+spring.datasource.schema=classpath:Init_quartz_h2_test.sql
+spring.jpa.show-sql=true
+# Hive metastore
+hive.metastore.uris=thrift://localhost:39083
+hive.metastore.dbname=default
+hive.hmshandler.retry.attempts=15
+hive.hmshandler.retry.interval=2000ms
+# Hive cache time
+cache.evict.hive.fixedRate.in.milliseconds=900000
+# Kafka schema registry
+kafka.schema.registry.url=http://localhost:8081
+# Update job instance state at regular intervals
+jobInstance.fixedDelay.in.milliseconds=60000
+# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds
+jobInstance.expired.milliseconds=604800000
+# schedule predicate job every 5 minutes and repeat 12 times at most
+#interval time unit m:minute h:hour d:day,only support these three units
+predicate.job.interval=5m
+predicate.job.repeat.count=12
+# external properties directory location
+external.config.location=
+# external BATCH or STREAMING env
+external.env.location=
+# login strategy ("default" or "ldap")
+login.strategy=default
+# ldap
+ldap.url=ldap://hostname:port
+ldap.email=@example.com
+ldap.searchBase=DC=org,DC=example
+ldap.searchPattern=(sAMAccountName={0})
+# elasticsearch
+elasticsearch.host=localhost
+elasticsearch.port=9200
+elasticsearch.scheme=http
+# elasticsearch.user = user
+# elasticsearch.password = password
+# livy
+livy.uri=http://localhost:38998/batches
+livy.need.queue=false
+livy.task.max.concurrent.count=20
+livy.task.submit.interval.second=3
+livy.task.appId.retry.count=3
+livy.need.kerberos=false
+livy.server.auth.kerberos.principal=livy/kerberos.principal
+livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file
+# yarn url
+yarn.uri=http://localhost:38088
+
+# griffin event listener
+internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook
+
+logging.file=logs/griffin-service.log
+
+spring.main.allow-bean-definition-overriding=true
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index d2d4ffee..119d0af4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@ under the License.
<groupId>org.apache.griffin</groupId>
<artifactId>griffin</artifactId>
- <version>0.7.0-SNAPSHOT</version>
+ <version>1.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Griffin ${project.version}</name>
<url>http://griffin.apache.org</url>
@@ -97,9 +97,10 @@ under the License.
<modules>
- <module>ui</module>
- <module>service</module>
- <module>measure</module>
+ <module>core</module>
+ <module>dispatcher</module>
+ <module>scheduler</module>
+ <module>anomalydetection</module>
</modules>
<dependencyManagement>
diff --git a/scheduler/pom.xml b/scheduler/pom.xml
new file mode 100644
index 00000000..03a88ff6
--- /dev/null
+++ b/scheduler/pom.xml
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>griffin</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>scheduler</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Griffin :: Scheduler</name>
+
+ <properties>
+ <hadoop.version>2.7.1</hadoop.version>
+ <hive.version>2.2.0</hive.version>
+ <spring.boot.version>2.1.7.RELEASE</spring.boot.version>
+ <spring.security.kerberos.version>1.0.1.RELEASE</spring.security.kerberos.version>
+ <confluent.version>3.2.0</confluent.version>
+ <quartz.version>2.2.2</quartz.version>
+ <start-class>org.apache.griffin.core.GriffinWebApplication</start-class>
+ <powermock.version>2.0.2</powermock.version>
+ <spring-boot-maven-plugin.version>2.1.7.RELEASE</spring-boot-maven-plugin.version>
+ <derby.version>10.14.1.0</derby.version>
+ <eclipselink.version>2.6.0</eclipselink.version>
+ <mysql.java.version>5.1.47</mysql.java.version>
+ <postgresql.version>42.4.1</postgresql.version>
+ <livy.core.version>0.3.0</livy.core.version>
+ <elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version>
+ <jackson-databind.version>2.9.9.3</jackson-databind.version>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>confluent</id>
+ <url>http://packages.confluent.io/maven/</url>
+ </repository>
+ </repositories>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-dependencies</artifactId>
+ <version>${spring.boot.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-log4j2</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-properties-migrator</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-json</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-data-jpa</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hibernate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aspects</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.kerberos</groupId>
+ <artifactId>spring-security-kerberos-client</artifactId>
+ <version>${spring.security.kerberos.version}</version>
+ </dependency>
+ <!--eclipse link-->
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ </dependency>
+ <!--<dependency>-->
+ <!--<groupId>mysql</groupId>-->
+ <!--<artifactId>mysql-connector-java</artifactId>-->
+ <!--<version>${mysql.java.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.retry</groupId>
+ <artifactId>spring-retry</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
+ </dependency>
+
+ <!-- to access metastore from hive-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <!--<scope>provided</scope>-->
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>de.ruedigermoeller</groupId>
+ <artifactId>fst</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access Hive using JDBC -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- to access confluent schema registry -->
+ <dependency>
+ <groupId>io.confluent</groupId>
+ <artifactId>kafka-schema-registry-client</artifactId>
+ <version>${confluent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!--schedule-->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.quartz-scheduler</groupId>
+ <artifactId>quartz-jobs</artifactId>
+ <version>${quartz.version}</version>
+ </dependency>
+
+ <!--livy-core-->
+ <dependency>
+ <groupId>com.cloudera.livy</groupId>
+ <artifactId>livy-core_2.10</artifactId>
+ <version>${livy.core.version}</version>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.vaadin.external.google</groupId>
+ <artifactId>android-json</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch-rest-client.version}</version>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <!--if you need mysql, please uncomment mysql-connector-java -->
+ <!--<profile>-->
+ <!--<id>mysql</id>-->
+ <!--<activation>-->
+ <!--<property>-->
+ <!--<name>mysql</name>-->
+ <!--</property>-->
+ <!--</activation>-->
+ <!--</profile>-->
+ <profile>
+ <id>dev</id>
+ <activation>
+ <property>
+ <name>dev</name>
+ </property>
+ </activation>
+ </profile>
+ <profile>
+ <id>postgresql</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>prod</name>
+ </property>
+ </activation>
+ </profile>
+ </profiles>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.ethlo.persistence.tools</groupId>
+ <artifactId>eclipselink-maven-plugin</artifactId>
+ <version>2.7.0</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>weave</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.persistence</groupId>
+ <artifactId>org.eclipse.persistence.jpa</artifactId>
+ <version>${eclipselink.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>lib</classifier>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <version>${spring-boot-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>build-info</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>false</executable>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <skipAssembly>false</skipAssembly>
+ <outputDirectory>../target</outputDirectory>
+ <descriptors>
+ <descriptor>src/main/resources/assembly/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/scheduler/src/main/resources/assembly/assembly.xml b/scheduler/src/main/resources/assembly/assembly.xml
new file mode 100644
index 00000000..70af6651
--- /dev/null
+++ b/scheduler/src/main/resources/assembly/assembly.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 ">
+ <id>${project.version}</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+
+ <includeBaseDirectory>true</includeBaseDirectory>
+
+ <!-- Adds dependencies to zip package under lib directory -->
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>./lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>README*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
+ </includes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>config/</outputDirectory>
+ <includes>
+ <include>env/*</include>
+ <include>*.xml</include>
+ <include>*.sql</include>
+ <include>*.json</include>
+ <include>*.properties</include>
+ </includes>
+ <excludes>
+ <exclude>*.txt</exclude>
+ <exclude>logback-test.xml</exclude>
+ </excludes>
+ </fileSet>
+
+ <fileSet>
+ <directory>src/main/resources/shell</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>*.*</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <outputDirectory>./lib</outputDirectory>
+ <includes>
+ <include>*${version}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
diff --git a/scheduler/src/test/resources/application.properties b/scheduler/src/test/resources/application.properties
new file mode 100644
index 00000000..81da26e3
--- /dev/null
+++ b/scheduler/src/test/resources/application.properties
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+spring.jpa.generate-ddl=true
+spring.datasource.driver-class-name=org.h2.Driver
+spring.datasource.url=jdbc:h2:mem:db;DB_CLOSE_DELAY=-1
+spring.datasource.username=test
+spring.datasource.password=test
+spring.datasource.schema=classpath:Init_quartz_h2_test.sql
+spring.jpa.show-sql=true
+# Hive metastore
+hive.metastore.uris=thrift://localhost:39083
+hive.metastore.dbname=default
+hive.hmshandler.retry.attempts=15
+hive.hmshandler.retry.interval=2000ms
+# Hive cache time
+cache.evict.hive.fixedRate.in.milliseconds=900000
+# Kafka schema registry
+kafka.schema.registry.url=http://localhost:8081
+# Update job instance state at regular intervals
+jobInstance.fixedDelay.in.milliseconds=60000
+# Expired time of job instance which is 7 days that is 604800000 milliseconds.Time unit only supports milliseconds
+jobInstance.expired.milliseconds=604800000
+# schedule predicate job every 5 minutes and repeat 12 times at most
+#interval time unit m:minute h:hour d:day,only support these three units
+predicate.job.interval=5m
+predicate.job.repeat.count=12
+# external properties directory location
+external.config.location=
+# external BATCH or STREAMING env
+external.env.location=
+# login strategy ("default" or "ldap")
+login.strategy=default
+# ldap
+ldap.url=ldap://hostname:port
+ldap.email=@example.com
+ldap.searchBase=DC=org,DC=example
+ldap.searchPattern=(sAMAccountName={0})
+# elasticsearch
+elasticsearch.host=localhost
+elasticsearch.port=9200
+elasticsearch.scheme=http
+# elasticsearch.user = user
+# elasticsearch.password = password
+# livy
+livy.uri=http://localhost:38998/batches
+livy.need.queue=false
+livy.task.max.concurrent.count=20
+livy.task.submit.interval.second=3
+livy.task.appId.retry.count=3
+livy.need.kerberos=false
+livy.server.auth.kerberos.principal=livy/kerberos.principal
+livy.server.auth.kerberos.keytab=/path/to/livy/keytab/file
+# yarn url
+yarn.uri=http://localhost:38088
+
+# griffin event listener
+internal.event.listeners=GriffinJobEventHook,GriffinTestJobEventHook
+
+logging.file=logs/griffin-service.log
+
+spring.main.allow-bean-definition-overriding=true
\ No newline at end of file