You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2023/02/16 14:43:59 UTC
[griffin] branch griffin-1.0.0-dev updated: grpc (#626)
This is an automated email from the ASF dual-hosted git repository.
guoyp pushed a commit to branch griffin-1.0.0-dev
in repository https://gitbox.apache.org/repos/asf/griffin.git
The following commit(s) were added to refs/heads/griffin-1.0.0-dev by this push:
new 32b062a7 grpc (#626)
32b062a7 is described below
commit 32b062a76dd39a282a17bf64a20d2b64831c59d8
Author: dabuliud <wa...@gmail.com>
AuthorDate: Thu Feb 16 22:43:51 2023 +0800
grpc (#626)
* grpc
* the proto file of tm and en
* prepare Server GRPC
* Schedule Server
* TaskManager Submit Process
---------
Co-authored-by: Warden <wa...@gmail.com>
---
anomalydetection/pom.xml | 134 +++++++--------
api/pom.xml | 84 +++++++++
.../org/apache/griffin/api/common/GRPCCode.java | 18 ++
.../griffin/api/context/DQCApplicationContext.java | 8 +
.../griffin}/api/transmit/AbstractProtocol.java | 2 +-
.../api/transmit/GriffinProtocolEvent.java | 2 +-
.../griffin}/api/transmit/ProtocolEventEnums.java | 2 +-
.../api/transmit/TmEnControlProtocolEvent.java | 2 +-
.../api/transmit/dto/TaskAssignRequestDTO.java | 12 ++
.../api/transmit/dto/TaskAssignResponseDTO.java | 11 ++
api/src/main/proto/ExecuteNodeProtocols.proto | 50 ++++++
api/src/main/proto/TaskManagerProtocols.proto | 36 ++++
api/src/main/proto/common.proto | 16 ++
core/pom.xml | 188 ++++++++++++---------
.../apache/griffin/core/GriffinWebApplication.java | 1 -
.../core/api/context/DQCApplicationContext.java | 17 --
.../apache/griffin/core/api/dao/DQContentDao.java | 7 -
.../api/transmit/dto/TaskAssignRequestDTO.java | 13 --
.../api/transmit/dto/TaskAssignResponseDTO.java | 12 --
.../{api => common}/dao/DQBusinessRuleDao.java | 4 +-
.../griffin/core/common/dao/DQContentDao.java | 8 +
.../dao/DQContentInstanceMapDao.java | 4 +-
.../{api => common}/entity/DQResoueceEnums.java | 2 +-
.../entity/GriffinDQBusinessRule.java | 2 +-
.../{api => common}/entity/GriffinDQContent.java | 2 +-
.../entity/GriffinDQContentInstanceMap.java | 2 +-
.../{api => common}/entity/GriffinDQTable.java | 2 +-
.../core/{api => common}/utils/SpringUtils.java | 2 +-
.../utils}/context/WorkerContext.java | 2 +-
.../core/master/bootstrap/GriffinMasterMain.java | 9 +-
.../name/ExecuteNodeNameResloverProvider.java | 28 +++
.../griffin/core/master/server/TaskManager.java | 45 +++++
.../core/master/service/TaskAssignService.java | 8 +-
.../master/service/TaskManagerServiceImpl.java | 53 ++++++
.../master/strategy/AbstractAssignStrategy.java | 11 +-
.../core/master/strategy/LooperAssignStrategy.java | 11 +-
.../core/master/transport/DQCConnection.java | 42 +++--
.../master/transport/DQCConnectionManager.java | 92 ++++++++++
.../core/worker/bootstrap/GriffinWorkerMain.java | 2 +-
.../core/worker/client/TaskManagerClient.java | 17 ++
.../griffin/core/worker/entity/bo/DQInstance.java | 7 +-
.../core/worker/entity/bo/task/DQBaseTask.java | 4 +-
.../core/worker/factory/DQInstanceFactory.java | 8 +-
.../core/worker/factory/DQStageFactory.java | 2 +-
.../griffin/core/worker/factory/DQTaskFactory.java | 4 +-
.../worker/schedule/TaskDispatcherScheduler.java | 2 +-
.../core/worker/service/DQInstanceService.java | 4 +-
.../worker/service/ExecuteNodeServiceImpl.java | 22 +++
.../core/worker/service/WorkCoreService.java | 2 +-
dispatcher/pom.xml | 134 +++++++--------
pom.xml | 1 +
scheduler/pom.xml | 134 +++++++--------
service/hibernate_mysql_pom.xml | 34 ++--
service/pom.xml | 134 +++++++--------
54 files changed, 978 insertions(+), 477 deletions(-)
diff --git a/anomalydetection/pom.xml b/anomalydetection/pom.xml
index 804de868..ad1f3a5e 100644
--- a/anomalydetection/pom.xml
+++ b/anomalydetection/pom.xml
@@ -156,74 +156,74 @@ under the License.
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>de.ruedigermoeller</groupId>
- <artifactId>fst</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>jetty-all</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>de.ruedigermoeller</groupId>-->
+<!-- <artifactId>fst</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
- <!-- to access Hive using JDBC -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <!– to access Hive using JDBC –>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-jdbc</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>*</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.mortbay.jetty</groupId>-->
+<!-- <artifactId>servlet-api-2.5</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<!-- to access confluent schema registry -->
<!-- <dependency>-->
diff --git a/api/pom.xml b/api/pom.xml
new file mode 100644
index 00000000..81f9a24d
--- /dev/null
+++ b/api/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>griffin</artifactId>
+ <groupId>org.apache.griffin</groupId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>api</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Griffin :: Api</name>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <!-- grpc -->
+<!-- <protobuf.version>3.7.1</protobuf.version>-->
+ <protobuf.version>3.17.2</protobuf.version>
+ <protobuf-plugin.version>0.6.1</protobuf-plugin.version>
+ <grpc.version>1.42.1</grpc.version>
+ </properties>
+
+ <dependencies>
+ <!-- new dependencies -->
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.20</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- grpc dependency-->
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <!-- Java 9+ compatibility - Do NOT update to 2.0.0 -->
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ <version>1.3.5</version>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
+ <build>
+ <extensions>
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>1.7.0</version>
+ </extension>
+ </extensions>
+
+ <plugins>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>0.6.1</version>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
diff --git a/api/src/main/java/org/apache/griffin/api/common/GRPCCode.java b/api/src/main/java/org/apache/griffin/api/common/GRPCCode.java
new file mode 100644
index 00000000..525ba4d8
--- /dev/null
+++ b/api/src/main/java/org/apache/griffin/api/common/GRPCCode.java
@@ -0,0 +1,18 @@
+package org.apache.griffin.api.common;
+
+public enum GRPCCode {
+ SUCCESS(200),
+ CLIENT_ERROR(500),
+ SERVER_ERROR(500),
+ EXTERNAL_ERROR(400);
+
+ private final int code;
+
+ GRPCCode(int code) {
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+}
diff --git a/api/src/main/java/org/apache/griffin/api/context/DQCApplicationContext.java b/api/src/main/java/org/apache/griffin/api/context/DQCApplicationContext.java
new file mode 100644
index 00000000..5c3286ac
--- /dev/null
+++ b/api/src/main/java/org/apache/griffin/api/context/DQCApplicationContext.java
@@ -0,0 +1,8 @@
+package org.apache.griffin.api.context;
+
+/**
+ * Master Runtime Env
+ * Scope: Singleton
+ */
+public interface DQCApplicationContext {
+}
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/AbstractProtocol.java b/api/src/main/java/org/apache/griffin/api/transmit/AbstractProtocol.java
similarity index 81%
rename from core/src/main/java/org/apache/griffin/core/api/transmit/AbstractProtocol.java
rename to api/src/main/java/org/apache/griffin/api/transmit/AbstractProtocol.java
index 1b310a83..f9ee0c95 100644
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/AbstractProtocol.java
+++ b/api/src/main/java/org/apache/griffin/api/transmit/AbstractProtocol.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.transmit;
+package org.apache.griffin.api.transmit;
public abstract class AbstractProtocol implements GriffinProtocolEvent{
private transient String ip;
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/GriffinProtocolEvent.java b/api/src/main/java/org/apache/griffin/api/transmit/GriffinProtocolEvent.java
similarity index 65%
rename from core/src/main/java/org/apache/griffin/core/api/transmit/GriffinProtocolEvent.java
rename to api/src/main/java/org/apache/griffin/api/transmit/GriffinProtocolEvent.java
index 56893d75..59aabd42 100644
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/GriffinProtocolEvent.java
+++ b/api/src/main/java/org/apache/griffin/api/transmit/GriffinProtocolEvent.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.transmit;
+package org.apache.griffin.api.transmit;
public interface GriffinProtocolEvent {
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/ProtocolEventEnums.java b/api/src/main/java/org/apache/griffin/api/transmit/ProtocolEventEnums.java
similarity index 87%
rename from core/src/main/java/org/apache/griffin/core/api/transmit/ProtocolEventEnums.java
rename to api/src/main/java/org/apache/griffin/api/transmit/ProtocolEventEnums.java
index c7e3f3b8..7b69388c 100644
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/ProtocolEventEnums.java
+++ b/api/src/main/java/org/apache/griffin/api/transmit/ProtocolEventEnums.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.transmit;
+package org.apache.griffin.api.transmit;
public enum ProtocolEventEnums {
TASK_ASSIGN_REQUEST(1, "the request for tm to assign task to en"),
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/TmEnControlProtocolEvent.java b/api/src/main/java/org/apache/griffin/api/transmit/TmEnControlProtocolEvent.java
similarity index 70%
rename from core/src/main/java/org/apache/griffin/core/api/transmit/TmEnControlProtocolEvent.java
rename to api/src/main/java/org/apache/griffin/api/transmit/TmEnControlProtocolEvent.java
index 4a075af7..00da7507 100644
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/TmEnControlProtocolEvent.java
+++ b/api/src/main/java/org/apache/griffin/api/transmit/TmEnControlProtocolEvent.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.transmit;
+package org.apache.griffin.api.transmit;
/**
* the Protocol for TaskManager to control ExecuteNode
diff --git a/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignRequestDTO.java b/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignRequestDTO.java
new file mode 100644
index 00000000..4c761ad5
--- /dev/null
+++ b/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignRequestDTO.java
@@ -0,0 +1,12 @@
+package org.apache.griffin.api.transmit.dto;
+
+import org.apache.griffin.api.transmit.AbstractProtocol;
+import org.apache.griffin.api.transmit.ProtocolEventEnums;
+
+public class TaskAssignRequestDTO extends AbstractProtocol {
+
+ @Override
+ public ProtocolEventEnums getProtocolEvent() {
+ return ProtocolEventEnums.TASK_ASSIGN_REQUEST;
+ }
+}
diff --git a/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignResponseDTO.java b/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignResponseDTO.java
new file mode 100644
index 00000000..3bae13b7
--- /dev/null
+++ b/api/src/main/java/org/apache/griffin/api/transmit/dto/TaskAssignResponseDTO.java
@@ -0,0 +1,11 @@
+package org.apache.griffin.api.transmit.dto;
+
+import org.apache.griffin.api.transmit.AbstractProtocol;
+import org.apache.griffin.api.transmit.ProtocolEventEnums;
+
+public class TaskAssignResponseDTO extends AbstractProtocol {
+ @Override
+ public ProtocolEventEnums getProtocolEvent() {
+ return ProtocolEventEnums.TASK_ASSIGN_RESPONSE;
+ }
+}
diff --git a/api/src/main/proto/ExecuteNodeProtocols.proto b/api/src/main/proto/ExecuteNodeProtocols.proto
new file mode 100644
index 00000000..871dab7b
--- /dev/null
+++ b/api/src/main/proto/ExecuteNodeProtocols.proto
@@ -0,0 +1,50 @@
+syntax = "proto3";
+
+import "common.proto";
+
+package org.apache.griffin.api.proto.protocol;
+
+option java_multiple_files = true;
+option java_package = "org.apache.griffin.api.proto.protocol";
+option java_outer_classname = "ExecuteNodeProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+service ExecuteNodeService {
+ // Submit Task to ExecuteNode
+ rpc submitDQTask(SubmitDQTaskRequest) returns (SubmitDQTaskResponse) {}
+ // Ask ExecuteNode to stop one task
+ rpc stopDQTask(StopDQTaskRequest) returns (StopDQTaskResponse) {}
+ // Ask ExecuteNode to stop one task
+ rpc querySingleDQTask(SQuerySingleDQTaskRequest) returns (QuerySingleDQTaskResponse) {}
+ rpc sayHello(SayHelloRequest) returns (SayHelloResponse) {}
+}
+
+message SubmitDQTaskRequest {
+ int64 instanceId = 1;
+}
+
+message SubmitDQTaskResponse {
+ int32 code = 1;
+ optional int32 status = 2;
+}
+
+message StopDQTaskRequest {
+ int64 instanceId = 1;
+}
+
+message StopDQTaskResponse {
+ int32 code = 1;
+ optional int32 status = 2;
+}
+
+message SQuerySingleDQTaskRequest {
+ int64 instanceId = 1;
+}
+
+message QuerySingleDQTaskResponse {
+ int32 code = 1;
+ optional int32 status = 2;
+}
+
+
diff --git a/api/src/main/proto/TaskManagerProtocols.proto b/api/src/main/proto/TaskManagerProtocols.proto
new file mode 100644
index 00000000..e0ddcf9d
--- /dev/null
+++ b/api/src/main/proto/TaskManagerProtocols.proto
@@ -0,0 +1,36 @@
+syntax = "proto3";
+
+import "common.proto";
+
+package org.apache.griffin.api.proto.protocol;
+
+option java_multiple_files = true;
+option java_package = "org.apache.griffin.api.proto.protocol";
+option java_outer_classname = "TaskManagerProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+service TaskManagerService {
+ rpc registDQWorkerNode(RegistDQWorkerNodeRequest) returns (RegistDQWorkerNodeResponse) {}
+ rpc reportDQWorkNodeStatus(ReportDQWorkNodeStatusRequest) returns (ReportDQWorkNodeStatusResponse) {}
+ rpc sayHello(SayHelloRequest) returns (SayHelloResponse) {}
+}
+
+message RegistDQWorkerNodeRequest {
+ string hostName = 1;
+ int32 port = 2;
+}
+
+message RegistDQWorkerNodeResponse {
+ int32 code = 1;
+ optional int32 status = 2;
+}
+
+message ReportDQWorkNodeStatusRequest {
+ int64 instanceId = 1;
+}
+
+message ReportDQWorkNodeStatusResponse {
+ int32 code = 1;
+ optional int32 status = 2;
+}
\ No newline at end of file
diff --git a/api/src/main/proto/common.proto b/api/src/main/proto/common.proto
new file mode 100644
index 00000000..a1de0953
--- /dev/null
+++ b/api/src/main/proto/common.proto
@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package org.apache.griffin.api.proto.protocol;
+
+option java_multiple_files = true;
+option java_package = "org.apache.griffin.api.proto.protocol";
+option java_outer_classname = "CommonProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+message SayHelloRequest {
+}
+
+message SayHelloResponse {
+ int32 code = 1;
+}
\ No newline at end of file
diff --git a/core/pom.xml b/core/pom.xml
index 4c0b7c00..d07312a8 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -48,6 +48,10 @@ under the License.
<livy.core.version>0.3.0</livy.core.version>
<elasticsearch-rest-client.version>6.2.4</elasticsearch-rest-client.version>
<jackson-databind.version>2.9.9.3</jackson-databind.version>
+ <!-- grpc -->
+<!-- <protobuf.version>3.19.1</protobuf.version>-->
+<!-- <protobuf-plugin.version>0.6.1</protobuf-plugin.version>-->
+<!-- <grpc.version>1.42.1</grpc.version>-->
</properties>
<repositories>
@@ -156,88 +160,74 @@ under the License.
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>de.ruedigermoeller</groupId>
- <artifactId>fst</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>jetty-all</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>de.ruedigermoeller</groupId>-->
+<!-- <artifactId>fst</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<!-- to access Hive using JDBC -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- to access confluent schema registry -->
<!-- <dependency>-->
-<!-- <groupId>io.confluent</groupId>-->
-<!-- <artifactId>kafka-schema-registry-client</artifactId>-->
-<!-- <version>${confluent.version}</version>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-jdbc</artifactId>-->
+<!-- <version>${hive.version}</version>-->
<!-- <exclusions>-->
<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>*</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.mortbay.jetty</groupId>-->
+<!-- <artifactId>servlet-api-2.5</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-log4j12</artifactId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<!-- </dependency>-->
-
<!--schedule-->
<dependency>
<groupId>org.springframework</groupId>
@@ -298,24 +288,58 @@ under the License.
<version>${elasticsearch-rest-client.version}</version>
</dependency>
- <!-- 新增依赖 -->
+ <!-- new dependencies -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
<scope>provided</scope>
</dependency>
+ <!-- grpc server dependency-->
+<!-- <dependency>-->
+<!-- <groupId>com.google.protobuf</groupId>-->
+<!-- <artifactId>protobuf-java</artifactId>-->
+<!-- <version>3.19.1</version>-->
+<!--<!– <version>3.7.1</version>–>-->
+<!-- </dependency>-->
+ <dependency>
+ <groupId>net.devh</groupId>
+ <artifactId>grpc-server-spring-boot-starter</artifactId>
+ <version>2.14.0.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.griffin</groupId>
+ <artifactId>api</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.devh</groupId>
+ <artifactId>grpc-client-spring-boot-starter</artifactId>
+ <version>2.14.0.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.12.0</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>3.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20220924</version>
+ </dependency>
</dependencies>
<profiles>
- <!--if you need mysql, please uncomment mysql-connector-java -->
- <!--<profile>-->
- <!--<id>mysql</id>-->
- <!--<activation>-->
- <!--<property>-->
- <!--<name>mysql</name>-->
- <!--</property>-->
- <!--</activation>-->
- <!--</profile>-->
<profile>
<id>dev</id>
<activation>
diff --git a/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
index 7c7ddb73..dfcfefa3 100644
--- a/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
+++ b/core/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
@@ -22,7 +22,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
diff --git a/core/src/main/java/org/apache/griffin/core/api/context/DQCApplicationContext.java b/core/src/main/java/org/apache/griffin/core/api/context/DQCApplicationContext.java
deleted file mode 100644
index d2a2fea3..00000000
--- a/core/src/main/java/org/apache/griffin/core/api/context/DQCApplicationContext.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.griffin.core.api.context;
-
-import lombok.Data;
-import org.springframework.stereotype.Component;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Master Runtime Env
- * Scope: Singleton
- */
-@Component
-@Data
-public class DQCApplicationContext {
- private Map<String, WorkerContext> context = new ConcurrentHashMap<>();
-}
diff --git a/core/src/main/java/org/apache/griffin/core/api/dao/DQContentDao.java b/core/src/main/java/org/apache/griffin/core/api/dao/DQContentDao.java
deleted file mode 100644
index 73849142..00000000
--- a/core/src/main/java/org/apache/griffin/core/api/dao/DQContentDao.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.griffin.core.api.dao;
-
-import org.apache.griffin.core.api.entity.GriffinDQContent;
-
-public interface DQContentDao {
- GriffinDQContent getById(Long id);
-}
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignRequestDTO.java b/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignRequestDTO.java
deleted file mode 100644
index 2c1e56df..00000000
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignRequestDTO.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.griffin.core.api.transmit.dto;
-
-import org.apache.griffin.core.api.transmit.AbstractProtocol;
-import org.apache.griffin.core.api.transmit.GriffinProtocolEvent;
-import org.apache.griffin.core.api.transmit.ProtocolEventEnums;
-
-public class TaskAssignRequestDTO extends AbstractProtocol {
-
- @Override
- public ProtocolEventEnums getProtocolEvent() {
- return ProtocolEventEnums.TASK_ASSIGN_REQUEST;
- }
-}
diff --git a/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignResponseDTO.java b/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignResponseDTO.java
deleted file mode 100644
index dee86532..00000000
--- a/core/src/main/java/org/apache/griffin/core/api/transmit/dto/TaskAssignResponseDTO.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.core.api.transmit.dto;
-
-import org.apache.griffin.core.api.transmit.AbstractProtocol;
-import org.apache.griffin.core.api.transmit.GriffinProtocolEvent;
-import org.apache.griffin.core.api.transmit.ProtocolEventEnums;
-
-public class TaskAssignResponseDTO extends AbstractProtocol {
- @Override
- public ProtocolEventEnums getProtocolEvent() {
- return ProtocolEventEnums.TASK_ASSIGN_RESPONSE;
- }
-}
diff --git a/core/src/main/java/org/apache/griffin/core/api/dao/DQBusinessRuleDao.java b/core/src/main/java/org/apache/griffin/core/common/dao/DQBusinessRuleDao.java
similarity index 62%
rename from core/src/main/java/org/apache/griffin/core/api/dao/DQBusinessRuleDao.java
rename to core/src/main/java/org/apache/griffin/core/common/dao/DQBusinessRuleDao.java
index 4197235f..1a791e2e 100644
--- a/core/src/main/java/org/apache/griffin/core/api/dao/DQBusinessRuleDao.java
+++ b/core/src/main/java/org/apache/griffin/core/common/dao/DQBusinessRuleDao.java
@@ -1,6 +1,6 @@
-package org.apache.griffin.core.api.dao;
+package org.apache.griffin.core.common.dao;
-import org.apache.griffin.core.api.entity.GriffinDQBusinessRule;
+import org.apache.griffin.core.common.entity.GriffinDQBusinessRule;
import org.springframework.stereotype.Component;
import java.util.List;
diff --git a/core/src/main/java/org/apache/griffin/core/common/dao/DQContentDao.java b/core/src/main/java/org/apache/griffin/core/common/dao/DQContentDao.java
new file mode 100644
index 00000000..97182d9f
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/common/dao/DQContentDao.java
@@ -0,0 +1,8 @@
+package org.apache.griffin.core.common.dao;
+
+
+import org.apache.griffin.core.common.entity.GriffinDQContent;
+
+public interface DQContentDao {
+ GriffinDQContent getById(Long id);
+}
diff --git a/core/src/main/java/org/apache/griffin/core/api/dao/DQContentInstanceMapDao.java b/core/src/main/java/org/apache/griffin/core/common/dao/DQContentInstanceMapDao.java
similarity index 60%
rename from core/src/main/java/org/apache/griffin/core/api/dao/DQContentInstanceMapDao.java
rename to core/src/main/java/org/apache/griffin/core/common/dao/DQContentInstanceMapDao.java
index 2ea57545..94ad5018 100644
--- a/core/src/main/java/org/apache/griffin/core/api/dao/DQContentInstanceMapDao.java
+++ b/core/src/main/java/org/apache/griffin/core/common/dao/DQContentInstanceMapDao.java
@@ -1,6 +1,6 @@
-package org.apache.griffin.core.api.dao;
+package org.apache.griffin.core.common.dao;
-import org.apache.griffin.core.api.entity.GriffinDQContentInstanceMap;
+import org.apache.griffin.core.common.entity.GriffinDQContentInstanceMap;
import org.springframework.stereotype.Component;
@Component
diff --git a/core/src/main/java/org/apache/griffin/core/api/entity/DQResoueceEnums.java b/core/src/main/java/org/apache/griffin/core/common/entity/DQResoueceEnums.java
similarity index 51%
rename from core/src/main/java/org/apache/griffin/core/api/entity/DQResoueceEnums.java
rename to core/src/main/java/org/apache/griffin/core/common/entity/DQResoueceEnums.java
index ec8f5fcf..1e7f94b2 100644
--- a/core/src/main/java/org/apache/griffin/core/api/entity/DQResoueceEnums.java
+++ b/core/src/main/java/org/apache/griffin/core/common/entity/DQResoueceEnums.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.entity;
+package org.apache.griffin.core.common.entity;
public enum DQResoueceEnums {
HIVE, KAFKA;
diff --git a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQBusinessRule.java b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQBusinessRule.java
similarity index 70%
rename from core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQBusinessRule.java
rename to core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQBusinessRule.java
index 7fbd00b5..214010b0 100644
--- a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQBusinessRule.java
+++ b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQBusinessRule.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.entity;
+package org.apache.griffin.core.common.entity;
import lombok.Data;
diff --git a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContent.java b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContent.java
similarity index 86%
rename from core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContent.java
rename to core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContent.java
index 2f70c713..de773467 100644
--- a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContent.java
+++ b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContent.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.entity;
+package org.apache.griffin.core.common.entity;
import lombok.Data;
diff --git a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContentInstanceMap.java b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContentInstanceMap.java
similarity index 75%
rename from core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContentInstanceMap.java
rename to core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContentInstanceMap.java
index a7f7b711..d46d7cc4 100644
--- a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQContentInstanceMap.java
+++ b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQContentInstanceMap.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.entity;
+package org.apache.griffin.core.common.entity;
import lombok.Data;
diff --git a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQTable.java b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQTable.java
similarity index 78%
rename from core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQTable.java
rename to core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQTable.java
index 74dbb4c9..acbde9e5 100644
--- a/core/src/main/java/org/apache/griffin/core/api/entity/GriffinDQTable.java
+++ b/core/src/main/java/org/apache/griffin/core/common/entity/GriffinDQTable.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.entity;
+package org.apache.griffin.core.common.entity;
import lombok.Data;
diff --git a/core/src/main/java/org/apache/griffin/core/api/utils/SpringUtils.java b/core/src/main/java/org/apache/griffin/core/common/utils/SpringUtils.java
similarity index 94%
rename from core/src/main/java/org/apache/griffin/core/api/utils/SpringUtils.java
rename to core/src/main/java/org/apache/griffin/core/common/utils/SpringUtils.java
index ad4e1059..c84c1c51 100644
--- a/core/src/main/java/org/apache/griffin/core/api/utils/SpringUtils.java
+++ b/core/src/main/java/org/apache/griffin/core/common/utils/SpringUtils.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.utils;
+package org.apache.griffin.core.common.utils;
import org.springframework.beans.BeansException;
diff --git a/core/src/main/java/org/apache/griffin/core/api/context/WorkerContext.java b/core/src/main/java/org/apache/griffin/core/common/utils/context/WorkerContext.java
similarity index 98%
rename from core/src/main/java/org/apache/griffin/core/api/context/WorkerContext.java
rename to core/src/main/java/org/apache/griffin/core/common/utils/context/WorkerContext.java
index 48e5be75..49f99949 100644
--- a/core/src/main/java/org/apache/griffin/core/api/context/WorkerContext.java
+++ b/core/src/main/java/org/apache/griffin/core/common/utils/context/WorkerContext.java
@@ -1,4 +1,4 @@
-package org.apache.griffin.core.api.context;
+package org.apache.griffin.core.common.utils.context;
import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
diff --git a/core/src/main/java/org/apache/griffin/core/master/bootstrap/GriffinMasterMain.java b/core/src/main/java/org/apache/griffin/core/master/bootstrap/GriffinMasterMain.java
index 0fa74c08..cd8a4cb4 100644
--- a/core/src/main/java/org/apache/griffin/core/master/bootstrap/GriffinMasterMain.java
+++ b/core/src/main/java/org/apache/griffin/core/master/bootstrap/GriffinMasterMain.java
@@ -4,10 +4,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
-@ComponentScan({"org.apache.griffin.core.master", "org.apache.griffin.core.api"})
+@ComponentScan({"org.apache.griffin.core.master", "org.apache.griffin.core.common"})
+//@ComponentScan({"org.apache.griffin.core.master.service"})
@SpringBootApplication
@EnableScheduling
public class GriffinMasterMain {
@@ -16,7 +18,10 @@ public class GriffinMasterMain {
.getLogger(GriffinMasterMain.class);
public static void main(String[] args) {
- SpringApplication.run(GriffinMasterMain.class, args);
+ ConfigurableApplicationContext run = SpringApplication.run(GriffinMasterMain.class, args);
LOGGER.info("application started");
+
+ Object taskManagerServiceImpl = run.getBean("taskManagerServiceImpl");
+ System.out.println(taskManagerServiceImpl);
}
}
diff --git a/core/src/main/java/org/apache/griffin/core/master/name/ExecuteNodeNameResloverProvider.java b/core/src/main/java/org/apache/griffin/core/master/name/ExecuteNodeNameResloverProvider.java
new file mode 100644
index 00000000..8de84ddb
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/master/name/ExecuteNodeNameResloverProvider.java
@@ -0,0 +1,28 @@
+package org.apache.griffin.core.master.name;
+
+import io.grpc.NameResolver;
+import io.grpc.NameResolverProvider;
+
+import java.net.URI;
+
+public class ExecuteNodeNameResloverProvider extends NameResolverProvider {
+ @Override
+ protected boolean isAvailable() {
+ return false;
+ }
+
+ @Override
+ protected int priority() {
+ return 0;
+ }
+
+ @Override
+ public NameResolver newNameResolver(URI uri, NameResolver.Args args) {
+ return null;
+ }
+
+ @Override
+ public String getDefaultScheme() {
+ return null;
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/master/server/TaskManager.java b/core/src/main/java/org/apache/griffin/core/master/server/TaskManager.java
new file mode 100644
index 00000000..258dce68
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/master/server/TaskManager.java
@@ -0,0 +1,45 @@
+package org.apache.griffin.core.master.server;
+
+
+import com.google.common.collect.Maps;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.griffin.api.common.GRPCCode;
+import org.apache.griffin.api.proto.protocol.ExecuteNodeServiceGrpc;
+import org.apache.griffin.api.proto.protocol.SayHelloRequest;
+import org.apache.griffin.api.proto.protocol.SayHelloResponse;
+import org.apache.griffin.core.master.transport.DQCConnection;
+import org.apache.griffin.core.master.transport.DQCConnectionManager;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.net.UnknownHostException;
+
+@Component
+@Slf4j
+public class TaskManager {
+
+ @Autowired
+ private DQCConnectionManager dqcConnectionManager;
+
+ public void registerWorker(String hostName, int port) throws UnknownHostException {
+ try {
+ dqcConnectionManager.registerWorker(hostName, port);
+ } catch (UnknownHostException uhe) {
+ throw uhe;
+ } catch (Exception e) {
+ log.error("Connect to ExecuteNode Failed. Host: {}, Port: {}", hostName, port);
+ }
+ }
+
+ public void submitDQTask(Long instanceId) {
+ DQCConnection aliveClient = dqcConnectionManager.getAliveClient();
+ if (aliveClient == null) {
+
+ }
+ if (aliveClient.submitDQTask(instanceId)) {
+ // todo add task and client info to cache
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/master/service/TaskAssignService.java b/core/src/main/java/org/apache/griffin/core/master/service/TaskAssignService.java
index a324973d..05e8a761 100644
--- a/core/src/main/java/org/apache/griffin/core/master/service/TaskAssignService.java
+++ b/core/src/main/java/org/apache/griffin/core/master/service/TaskAssignService.java
@@ -9,7 +9,7 @@ import org.springframework.util.Assert;
import javax.annotation.PostConstruct;
-@Component
+//@Component
public class TaskAssignService {
@Value("${task.assign.strategy}")
@@ -24,7 +24,7 @@ public class TaskAssignService {
}
- public String assignTask(long instanceId) {
- return strategy.assignTask(instanceId);
- }
+// public String assignTask(long instanceId) {
+// return strategy.assignTask(instanceId);
+// }
}
diff --git a/core/src/main/java/org/apache/griffin/core/master/service/TaskManagerServiceImpl.java b/core/src/main/java/org/apache/griffin/core/master/service/TaskManagerServiceImpl.java
new file mode 100644
index 00000000..250b70d9
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/master/service/TaskManagerServiceImpl.java
@@ -0,0 +1,53 @@
+package org.apache.griffin.core.master.service;
+
+
+import io.grpc.stub.StreamObserver;
+import lombok.extern.slf4j.Slf4j;
+import net.devh.boot.grpc.server.service.GrpcService;
+import org.apache.griffin.api.common.GRPCCode;
+import org.apache.griffin.api.proto.protocol.*;
+import org.apache.griffin.core.master.server.TaskManager;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.net.UnknownHostException;
+
+@GrpcService
+@Slf4j
+public class TaskManagerServiceImpl extends TaskManagerServiceGrpc.TaskManagerServiceImplBase {
+
+ @Autowired
+ private TaskManager taskManager;
+
+ @Override
+ public void registDQWorkerNode(RegistDQWorkerNodeRequest request, StreamObserver<RegistDQWorkerNodeResponse> responseObserver) {
+ String hostName = request.getHostName();
+ int port = request.getPort();
+ GRPCCode code = GRPCCode.SUCCESS;
+ try {
+ taskManager.registerWorker(hostName, port);
+ } catch (UnknownHostException uhe) {
+ log.error("TaskManager registerWorker failed. Host: {}, port: {} is unreachable", hostName, port);
+ code = GRPCCode.CLIENT_ERROR;
+ } catch (Exception e) {
+ log.error("TaskManager registerWorker Unknown Error. Host: {}, port: {}", hostName, port, e);
+ code = GRPCCode.SERVER_ERROR;
+ }
+ responseObserver.onNext(RegistDQWorkerNodeResponse.newBuilder()
+ .setCode(code.getCode())
+ .build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void reportDQWorkNodeStatus(ReportDQWorkNodeStatusRequest request, StreamObserver<ReportDQWorkNodeStatusResponse> responseObserver) {
+ super.reportDQWorkNodeStatus(request, responseObserver);
+ }
+
+ @Override
+ public void sayHello(SayHelloRequest request, StreamObserver<SayHelloResponse> responseObserver) {
+ responseObserver.onNext(SayHelloResponse.newBuilder()
+ .setCode(GRPCCode.SUCCESS.getCode())
+ .build());
+ responseObserver.onCompleted();
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/master/strategy/AbstractAssignStrategy.java b/core/src/main/java/org/apache/griffin/core/master/strategy/AbstractAssignStrategy.java
index 4cd844a7..9f57c2b7 100644
--- a/core/src/main/java/org/apache/griffin/core/master/strategy/AbstractAssignStrategy.java
+++ b/core/src/main/java/org/apache/griffin/core/master/strategy/AbstractAssignStrategy.java
@@ -1,14 +1,11 @@
package org.apache.griffin.core.master.strategy;
-import org.apache.griffin.core.api.context.DQCApplicationContext;
-public abstract class AbstractAssignStrategy {
+import org.apache.griffin.core.master.transport.DQCConnection;
- protected DQCApplicationContext dqcApplicationContext;
+import java.util.List;
- public AbstractAssignStrategy(DQCApplicationContext dqcApplicationContext) {
- this.dqcApplicationContext = dqcApplicationContext;
- }
+public abstract class AbstractAssignStrategy {
- public abstract String assignTask(long instanceId);
+ public abstract DQCConnection assignTask(List<DQCConnection> clientList);
}
diff --git a/core/src/main/java/org/apache/griffin/core/master/strategy/LooperAssignStrategy.java b/core/src/main/java/org/apache/griffin/core/master/strategy/LooperAssignStrategy.java
index 4d85fc4a..238ed8e6 100644
--- a/core/src/main/java/org/apache/griffin/core/master/strategy/LooperAssignStrategy.java
+++ b/core/src/main/java/org/apache/griffin/core/master/strategy/LooperAssignStrategy.java
@@ -1,15 +1,14 @@
package org.apache.griffin.core.master.strategy;
-import org.apache.griffin.core.api.context.DQCApplicationContext;
-public class LooperAssignStrategy extends AbstractAssignStrategy {
+import org.apache.griffin.core.master.transport.DQCConnection;
- public LooperAssignStrategy(DQCApplicationContext dqcApplicationContext) {
- super(dqcApplicationContext);
- }
+import java.util.List;
+
+public class LooperAssignStrategy extends AbstractAssignStrategy {
@Override
- public String assignTask(long instanceId) {
+ public DQCConnection assignTask(List<DQCConnection> clientList) {
return null;
}
}
diff --git a/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnection.java b/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnection.java
index 0f7b902a..d2d4c7be 100644
--- a/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnection.java
+++ b/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnection.java
@@ -1,27 +1,47 @@
package org.apache.griffin.core.master.transport;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+import io.grpc.stub.StreamObserver;
+import lombok.Builder;
+import lombok.Data;
+import org.apache.griffin.api.common.GRPCCode;
+import org.apache.griffin.api.proto.protocol.*;
+
+import java.net.UnknownHostException;
import java.nio.channels.ServerSocketChannel;
import java.util.concurrent.Future;
/**
* the obj has a socketChannel to worker node
*/
+@Data
+@Builder
public class DQCConnection {
// worker hostName
private String hostName;
- // worker hostIP
- private String hostIP;
// worker hostPort
private int hostPort;
- // todo
- private ServerSocketChannel channel;
+ // client
+ private ExecuteNodeServiceGrpc.ExecuteNodeServiceBlockingStub client;
+
+ private boolean isAlive;
- /**
- * Send msg async
- * @param msg message
- * @return Future
- */
- public Future send(byte[] msg) {
- return null;
+ public boolean sayHello() throws UnknownHostException {
+ SayHelloResponse sayHelloResponse = client.sayHello(SayHelloRequest.newBuilder().build());
+ if (sayHelloResponse.getCode() == GRPCCode.SUCCESS.getCode()) {
+ return true;
+ } else {
+ throw new UnknownHostException(hostName + ":" + hostPort);
+ }
}
+
+ public boolean submitDQTask(Long instanceId) {
+ SubmitDQTaskRequest submitDQTaskRequest = SubmitDQTaskRequest.newBuilder()
+ .setInstanceId(instanceId)
+ .build();
+ SubmitDQTaskResponse submitDQTaskResponse = client.submitDQTask(submitDQTaskRequest);
+ return submitDQTaskResponse.getCode() == GRPCCode.SUCCESS.getCode();
+ }
+
}
diff --git a/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnectionManager.java b/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnectionManager.java
new file mode 100644
index 00000000..9cddce5c
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/master/transport/DQCConnectionManager.java
@@ -0,0 +1,92 @@
+package org.apache.griffin.core.master.transport;
+
+import com.google.common.collect.Lists;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.griffin.api.proto.protocol.ExecuteNodeServiceGrpc;
+import org.apache.griffin.core.master.strategy.AbstractAssignStrategy;
+import org.apache.griffin.core.master.strategy.AssignStrategyFactory;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.net.ConnectException;
+import java.net.UnknownHostException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Component
+@Slf4j
+public class DQCConnectionManager {
+
+ private List<DQCConnection> clientList = Lists.newCopyOnWriteArrayList();
+ private String assignStrategyClassName = "org.apache.griffin.core.master.strategy.LooperAssignStrategy";
+ private AbstractAssignStrategy strategy;
+ @PostConstruct
+ public void init() {
+ strategy = AssignStrategyFactory.getStrategy(assignStrategyClassName);
+ if (strategy == null) {
+ log.error("DQCConnectionManager init error.");
+ System.exit(-1);
+ }
+ }
+
+ public void registerWorker(String hostName, int port) throws UnknownHostException {
+ try {
+ // the client connect to en
+ ManagedChannel channel = ManagedChannelBuilder.forAddress(hostName, port)
+ .usePlaintext()
+ .build();
+ ExecuteNodeServiceGrpc.ExecuteNodeServiceBlockingStub clientStub = ExecuteNodeServiceGrpc.newBlockingStub(channel);
+
+ DQCConnection dqcConnection = DQCConnection.builder()
+ .hostName(hostName)
+ .hostPort(port)
+ .client(clientStub)
+ .build();
+ // check Connection is alive
+ if (dqcConnection.sayHello()) {
+ clientList.add(dqcConnection);
+ }
+ } catch (UnknownHostException uhe) {
+ throw uhe;
+ } catch (Exception e) {
+ log.error("Connect to ExecuteNode Failed. Host: {}, Port: {}", hostName, port);
+ }
+ }
+
+ public DQCConnection getAliveClient() {
+ return strategy.assignTask(clientList);
+ }
+
+ // check client health
+ @Scheduled(cron = "0 * * * * ?")
+ public void chekClientHealth() {
+ if (CollectionUtils.isEmpty(clientList)) {
+ log.warn("There is no dqcConnection, please start up ExecuteNode.");
+ return;
+ }
+ clientList.forEach(client -> {
+ String hostName = client.getHostName();
+ int hostPort = client.getHostPort();
+ try {
+ if (!client.isAlive()) {
+ ManagedChannel channel = ManagedChannelBuilder.forAddress(hostName, hostPort)
+ .usePlaintext()
+ .build();
+ ExecuteNodeServiceGrpc.ExecuteNodeServiceBlockingStub clientStub = ExecuteNodeServiceGrpc.newBlockingStub(channel);
+ client.setClient(clientStub);
+ }
+ if (client.sayHello()) {
+ client.setAlive(true);
+ }
+ } catch (UnknownHostException e) {
+ log.error("Connect to ExecuteNode Failed. Host: {}, Port: {}",
+ hostName, hostPort);
+ client.setAlive(false);
+ }
+ });
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/worker/bootstrap/GriffinWorkerMain.java b/core/src/main/java/org/apache/griffin/core/worker/bootstrap/GriffinWorkerMain.java
index 9516bcb4..64adb5a7 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/bootstrap/GriffinWorkerMain.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/bootstrap/GriffinWorkerMain.java
@@ -7,7 +7,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.annotation.EnableScheduling;
-@ComponentScan({"org.apache.griffin.core.worker", "org.apache.griffin.core.api"})
+@ComponentScan({"org.apache.griffin.core.worker", "org.apache.griffin.core.common"})
@SpringBootApplication
@EnableScheduling
public class GriffinWorkerMain {
diff --git a/core/src/main/java/org/apache/griffin/core/worker/client/TaskManagerClient.java b/core/src/main/java/org/apache/griffin/core/worker/client/TaskManagerClient.java
new file mode 100644
index 00000000..0872d71b
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/worker/client/TaskManagerClient.java
@@ -0,0 +1,17 @@
+package org.apache.griffin.core.worker.client;
+
+
+import net.devh.boot.grpc.client.inject.GrpcClient;
+import org.apache.griffin.api.proto.protocol.TaskManagerServiceGrpc;
+
+public class TaskManagerClient {
+
+ // todo connect to master?
+ @GrpcClient("taskManagerService")
+ private TaskManagerServiceGrpc.TaskManagerServiceBlockingStub client;
+
+ public void test() {
+
+// TaskManagerServiceGrpc.newBlockingStub()
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/worker/entity/bo/DQInstance.java b/core/src/main/java/org/apache/griffin/core/worker/entity/bo/DQInstance.java
index e4608f9c..303c7d13 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/entity/bo/DQInstance.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/entity/bo/DQInstance.java
@@ -16,7 +16,7 @@ import java.util.List;
* 一个实例包含多个子任务
*/
@Data
-public class DQInstance {
+public class DQInstance implements Comparable<DQInstance> {
private Long id;
private Long dqcId;
@@ -68,4 +68,9 @@ public class DQInstance {
}
return isFinishRecord;
}
+
+ @Override
+ public int compareTo(DQInstance o) {
+ return 0;
+ }
}
diff --git a/core/src/main/java/org/apache/griffin/core/worker/entity/bo/task/DQBaseTask.java b/core/src/main/java/org/apache/griffin/core/worker/entity/bo/task/DQBaseTask.java
index fffc684d..5fdf4ed9 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/entity/bo/task/DQBaseTask.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/entity/bo/task/DQBaseTask.java
@@ -1,9 +1,9 @@
package org.apache.griffin.core.worker.entity.bo.task;
-import com.beust.jcommander.internal.Lists;
+import com.google.common.collect.Lists;
import lombok.Data;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.griffin.core.api.context.WorkerContext;
+import org.apache.griffin.core.common.utils.context.WorkerContext;
import org.apache.griffin.core.worker.entity.dispatcher.JobStatus;
import org.apache.griffin.core.worker.entity.enums.DQEngineEnum;
import org.apache.griffin.core.worker.entity.enums.DQTaskStatus;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/factory/DQInstanceFactory.java b/core/src/main/java/org/apache/griffin/core/worker/factory/DQInstanceFactory.java
index 386bd8b9..7ea41cb1 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/factory/DQInstanceFactory.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/factory/DQInstanceFactory.java
@@ -1,9 +1,9 @@
package org.apache.griffin.core.worker.factory;
-import org.apache.griffin.core.api.dao.DQBusinessRuleDao;
-import org.apache.griffin.core.api.dao.DQContentDao;
-import org.apache.griffin.core.api.entity.GriffinDQBusinessRule;
-import org.apache.griffin.core.api.entity.GriffinDQContent;
+import org.apache.griffin.core.common.dao.DQBusinessRuleDao;
+import org.apache.griffin.core.common.dao.DQContentDao;
+import org.apache.griffin.core.common.entity.GriffinDQBusinessRule;
+import org.apache.griffin.core.common.entity.GriffinDQContent;
import org.apache.griffin.core.worker.dao.DQInstanceDao;
import org.apache.griffin.core.worker.entity.bo.DQInstance;
import org.apache.griffin.core.worker.entity.bo.task.DQBaseTask;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/factory/DQStageFactory.java b/core/src/main/java/org/apache/griffin/core/worker/factory/DQStageFactory.java
index 519c7162..b8f4ec56 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/factory/DQStageFactory.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/factory/DQStageFactory.java
@@ -1,6 +1,6 @@
package org.apache.griffin.core.worker.factory;
-import org.apache.griffin.core.api.utils.SpringUtils;
+import org.apache.griffin.core.common.utils.SpringUtils;
import org.apache.griffin.core.worker.dao.DQStageDao;
import org.apache.griffin.core.worker.entity.bo.DQInstance;
import org.apache.griffin.core.worker.entity.enums.DQStageTypeEnum;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/factory/DQTaskFactory.java b/core/src/main/java/org/apache/griffin/core/worker/factory/DQTaskFactory.java
index 8bbd6336..11d6d900 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/factory/DQTaskFactory.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/factory/DQTaskFactory.java
@@ -1,7 +1,7 @@
package org.apache.griffin.core.worker.factory;
-import org.apache.griffin.core.api.entity.DQResoueceEnums;
-import org.apache.griffin.core.api.entity.GriffinDQBusinessRule;
+import org.apache.griffin.core.common.entity.DQResoueceEnums;
+import org.apache.griffin.core.common.entity.GriffinDQBusinessRule;
import org.apache.griffin.core.worker.dao.DQTaskDao;
import org.apache.griffin.core.worker.entity.bo.task.DQBaseTask;
import org.slf4j.Logger;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/schedule/TaskDispatcherScheduler.java b/core/src/main/java/org/apache/griffin/core/worker/schedule/TaskDispatcherScheduler.java
index d29bcb81..4ddc34cd 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/schedule/TaskDispatcherScheduler.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/schedule/TaskDispatcherScheduler.java
@@ -3,7 +3,7 @@ package org.apache.griffin.core.worker.schedule;
import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.griffin.core.api.context.WorkerContext;
+import org.apache.griffin.core.common.utils.context.WorkerContext;
import org.apache.griffin.core.worker.entity.bo.DQInstance;
import org.apache.griffin.core.worker.entity.enums.DQInstanceStatus;
import org.apache.griffin.core.worker.entity.enums.DQStageStatus;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/service/DQInstanceService.java b/core/src/main/java/org/apache/griffin/core/worker/service/DQInstanceService.java
index 276ebdaa..904de4ed 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/service/DQInstanceService.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/service/DQInstanceService.java
@@ -1,7 +1,7 @@
package org.apache.griffin.core.worker.service;
-import org.apache.griffin.core.api.entity.GriffinDQContentInstanceMap;
-import org.apache.griffin.core.api.dao.DQContentInstanceMapDao;
+import org.apache.griffin.core.common.entity.GriffinDQContentInstanceMap;
+import org.apache.griffin.core.common.dao.DQContentInstanceMapDao;
import org.apache.griffin.core.worker.dao.DQInstanceDao;
import org.apache.griffin.core.worker.entity.bo.DQInstance;
import org.apache.griffin.core.worker.entity.enums.DQInstanceStatus;
diff --git a/core/src/main/java/org/apache/griffin/core/worker/service/ExecuteNodeServiceImpl.java b/core/src/main/java/org/apache/griffin/core/worker/service/ExecuteNodeServiceImpl.java
new file mode 100644
index 00000000..ebe9091f
--- /dev/null
+++ b/core/src/main/java/org/apache/griffin/core/worker/service/ExecuteNodeServiceImpl.java
@@ -0,0 +1,22 @@
+package org.apache.griffin.core.worker.service;
+
+
+import io.grpc.stub.StreamObserver;
+import org.apache.griffin.api.proto.protocol.*;
+
+public class ExecuteNodeServiceImpl extends ExecuteNodeServiceGrpc.ExecuteNodeServiceImplBase {
+ @Override
+ public void submitDQTask(SubmitDQTaskRequest request, StreamObserver<SubmitDQTaskResponse> responseObserver) {
+ super.submitDQTask(request, responseObserver);
+ }
+
+ @Override
+ public void stopDQTask(StopDQTaskRequest request, StreamObserver<StopDQTaskResponse> responseObserver) {
+ super.stopDQTask(request, responseObserver);
+ }
+
+ @Override
+ public void querySingleDQTask(SQuerySingleDQTaskRequest request, StreamObserver<QuerySingleDQTaskResponse> responseObserver) {
+ super.querySingleDQTask(request, responseObserver);
+ }
+}
diff --git a/core/src/main/java/org/apache/griffin/core/worker/service/WorkCoreService.java b/core/src/main/java/org/apache/griffin/core/worker/service/WorkCoreService.java
index 2a5f9863..8d7e0bd4 100644
--- a/core/src/main/java/org/apache/griffin/core/worker/service/WorkCoreService.java
+++ b/core/src/main/java/org/apache/griffin/core/worker/service/WorkCoreService.java
@@ -1,6 +1,6 @@
package org.apache.griffin.core.worker.service;
-import org.apache.griffin.core.api.context.WorkerContext;
+import org.apache.griffin.core.common.utils.context.WorkerContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
diff --git a/dispatcher/pom.xml b/dispatcher/pom.xml
index da5cd950..e534cbf4 100644
--- a/dispatcher/pom.xml
+++ b/dispatcher/pom.xml
@@ -156,74 +156,74 @@ under the License.
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>de.ruedigermoeller</groupId>
- <artifactId>fst</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>jetty-all</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>de.ruedigermoeller</groupId>-->
+<!-- <artifactId>fst</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
- <!-- to access Hive using JDBC -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <!– to access Hive using JDBC –>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-jdbc</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>*</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.mortbay.jetty</groupId>-->
+<!-- <artifactId>servlet-api-2.5</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<!-- to access confluent schema registry -->
<!-- <dependency>-->
diff --git a/pom.xml b/pom.xml
index 9cca4fe1..29f78a97 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,6 +101,7 @@ under the License.
<module>dispatcher</module>
<module>scheduler</module>
<module>anomalydetection</module>
+ <module>api</module>
</modules>
<dependencyManagement>
diff --git a/scheduler/pom.xml b/scheduler/pom.xml
index 0cd03a89..712d73d8 100644
--- a/scheduler/pom.xml
+++ b/scheduler/pom.xml
@@ -156,74 +156,74 @@ under the License.
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>de.ruedigermoeller</groupId>
- <artifactId>fst</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>jetty-all</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>de.ruedigermoeller</groupId>-->
+<!-- <artifactId>fst</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
- <!-- to access Hive using JDBC -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <!– to access Hive using JDBC –>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-jdbc</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>*</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.mortbay.jetty</groupId>-->
+<!-- <artifactId>servlet-api-2.5</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<!-- to access confluent schema registry -->
<!-- <dependency>-->
diff --git a/service/hibernate_mysql_pom.xml b/service/hibernate_mysql_pom.xml
index 5a1be463..0d0e92fd 100644
--- a/service/hibernate_mysql_pom.xml
+++ b/service/hibernate_mysql_pom.xml
@@ -103,23 +103,23 @@ under the License.
<version>2.6.3</version>
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <artifactId>servlet-api</artifactId>
- <groupId>javax.servlet</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- </dependency>-->
<!-- to access confluent schema registry -->
<dependency>
diff --git a/service/pom.xml b/service/pom.xml
index c79f0004..5d680762 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -156,74 +156,74 @@ under the License.
</dependency>
<!-- to access metastore from hive-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <!--<scope>provided</scope>-->
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>de.ruedigermoeller</groupId>
- <artifactId>fst</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hadoop</groupId>-->
+<!-- <artifactId>hadoop-client</artifactId>-->
+<!-- <version>${hadoop.version}</version>-->
+<!-- <!–<scope>provided</scope>–>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-metastore</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>jetty-all</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>de.ruedigermoeller</groupId>-->
+<!-- <artifactId>fst</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
- <!-- to access Hive using JDBC -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api-2.5</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <!– to access Hive using JDBC –>-->
+<!-- <dependency>-->
+<!-- <groupId>org.apache.hive</groupId>-->
+<!-- <artifactId>hive-jdbc</artifactId>-->
+<!-- <version>${hive.version}</version>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.aggregate</groupId>-->
+<!-- <artifactId>*</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.eclipse.jetty.orbit</groupId>-->
+<!-- <artifactId>javax.servlet</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>javax.servlet</groupId>-->
+<!-- <artifactId>servlet-api</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.mortbay.jetty</groupId>-->
+<!-- <artifactId>servlet-api-2.5</artifactId>-->
+<!-- </exclusion>-->
+<!-- <exclusion>-->
+<!-- <groupId>org.slf4j</groupId>-->
+<!-- <artifactId>slf4j-log4j12</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<!-- to access confluent schema registry -->
<dependency>