You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2020/06/09 01:04:57 UTC
[submarine] branch master updated: SUBMARINE-460. Enable
interpreter module
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 8c7a219 SUBMARINE-460. Enable interpreter module
8c7a219 is described below
commit 8c7a219683cc30fc12fe20ddf9fe7fed4748145b
Author: Xun Liu <li...@apache.org>
AuthorDate: Sun Mar 29 20:36:56 2020 +0800
SUBMARINE-460. Enable interpreter module
### What is this PR for?
zeppelin released version 0.9-preview1, so we can enable the interpreter module for submarine.
We can test our own interpreter and `submarine-security/spark-security`,
When the `zeppelin-0.9` version is released, the zeppelin version number in the pom needs to be updated.
### What type of PR is it?
[Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-460
### How should this be tested?
* [CI Pass](https://travis-ci.org/github/liuxunorg/submarine/builds/695333876)
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Xun Liu <li...@apache.org>
Closes #306 from liuxunorg/SUBMARINE-460 and squashes the following commits:
175c0de [Xun Liu] SUBMARINE-460. Enable interpreter module
---
.travis.yml | 41 +++++++++++-----------
pom.xml | 2 +-
.../pom.xml | 6 ++--
.../submarine/interpreter/AbstractInterpreter.java | 0
.../apache/submarine/interpreter/Interpreter.java | 0
.../interpreter/InterpreterException.java | 0
.../submarine/interpreter/InterpreterGroup.java | 0
.../submarine/interpreter/InterpreterProcess.java | 0
.../submarine/interpreter/InterpreterResult.java | 0
.../interpreter/InterpreterResultMessage.java | 0
.../src/main/resources/log4j.properties | 0
submarine-workbench/interpreter/pom.xml | 4 +--
.../interpreter/python-interpreter/pom.xml | 8 ++---
.../interpreter/PythonInterpreterTest.java | 2 +-
.../interpreter/spark-interpreter/pom.xml | 10 ++----
submarine-workbench/pom.xml | 2 +-
16 files changed, 35 insertions(+), 40 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 750084a..51543c5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -44,20 +44,20 @@ env:
global:
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
- SUBMARINE="org.apache.submarine"
- - EXCLUDE_SUBMARINE="!${SUBMARINE}:submarine-all,!${SUBMARINE}:submarine-client,!${SUBMARINE}:submarine-dist"
- - EXCLUDE_SERVER="!${SUBMARINE}:submarine-server-api,!${SUBMARINE}:submarine-server-core,!${SUBMARINE}:submarine-server-rpc"
- - EXCLUDE_CLIENT="!${SUBMARINE}:submarine-client"
- - EXCLUDE_ALL="!${SUBMARINE}:submarine-all"
- - EXCLUDE_WORKBENCH="!${SUBMARINE}:submarine-workbench,!${SUBMARINE}:submarine-workbench-web,!${SUBMARINE}:submarine-workbench-web-ng"
- - EXCLUDE_INTERPRETER="" # Template disable by SUBMARINE-381 "!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter
- - EXCLUDE_SUBMITTER_K8S="!${SUBMARINE}:submarine-submitter-k8s"
- - EXCLUDE_SUBMITTER_YARN="!${SUBMARINE}:submarine-submitter-yarn"
- - EXCLUDE_SUBMITTER="!${SUBMARINE}:submarine-server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}"
- - EXCLUDE_COMMONS="!${SUBMARINE}:submarine-commons-cluster,!${SUBMARINE}:submarine-commons-metastore,!${SUBMARINE}:submarine-commons-rpc,!${SUBMARINE}:submarine-commons-runtime,!${SUBMARINE}:submarine-commons-utils"
- - EXCLUDE_CLOUD="!${SUBMARINE}:submarine-cloud"
- - EXCLUDE_DIST="!${SUBMARINE}:submarine-dist"
- - EXCLUDE_TEST="!${SUBMARINE}:submarine-test,!${SUBMARINE}:submarine-test-e2e,!${SUBMARINE}:submarine-test-k8s"
- - EXCLUDE_SPARK_SECURTITY="!${SUBMARINE}:submarine-spark-security"
+ - EXCLUDE_SUBMARINE="!:submarine-all,!:submarine-client,!:submarine-dist"
+ - EXCLUDE_SERVER="!:submarine-server-api,!:submarine-server-core,!:submarine-server-rpc"
+ - EXCLUDE_CLIENT="!:submarine-client"
+ - EXCLUDE_ALL="!:submarine-all"
+ - EXCLUDE_WORKBENCH="!:submarine-workbench,!:submarine-workbench-web,!:submarine-workbench-web-ng"
+ - EXCLUDE_INTERPRETER="!:submarine-interpreter,!:submarine-interpreter-core,!:submarine-python-interpreter,!:submarine-spark-interpreter"
+ - EXCLUDE_SUBMITTER_K8S="!:submarine-submitter-k8s"
+ - EXCLUDE_SUBMITTER_YARN="!:submarine-submitter-yarn"
+ - EXCLUDE_SUBMITTER="!:submarine-server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}"
+ - EXCLUDE_COMMONS="!:submarine-commons-cluster,!:submarine-commons-metastore,!:submarine-commons-rpc,!:submarine-commons-runtime,!:submarine-commons-utils"
+ - EXCLUDE_CLOUD="!:submarine-cloud"
+ - EXCLUDE_DIST="!:submarine-dist"
+ - EXCLUDE_TEST="!:submarine-test,!:submarine-test-e2e,!:submarine-test-k8s"
+ - EXCLUDE_SPARK_SECURTITY="!:submarine-spark-security"
- MOZ_HEADLESS=1
before_install:
@@ -105,7 +105,7 @@ matrix:
- PROFILE="-Phadoop-2.9"
- BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1"
- MODULES=""
- - TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},org.apache.submarine:submarine-test-k8s"
+ - TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},:submarine-test-k8s"
- TEST_PROJECTS=""
- name: Test submarine test-e2e
@@ -230,12 +230,11 @@ matrix:
- TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER},${EXCLUDE_SPARK_SECURTITY}"
- TEST_PROJECTS=""
- # Template disable by SUBMARINE-381
- #- name: Test submarine interpreter
- # language: java
- # jdk: "openjdk8"
- # dist: xenial
- # env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
+ - name: Test submarine interpreter
+ language: java
+ jdk: "openjdk8"
+ dist: xenial
+ env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
- name: Test submarine workbench-web VUE
language: node_js
diff --git a/pom.xml b/pom.xml
index fb9d013..eb4f4ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,7 +123,7 @@
<nimbus-jose-jwt.version>7.9</nimbus-jose-jwt.version>
<mybatis-generator.version>1.3.7</mybatis-generator.version>
<derby.version>10.15.1.3</derby.version>
- <zeppelin.version>0.9.0-SNAPSHOT</zeppelin.version>
+ <zeppelin.version>0.9.0-preview1</zeppelin.version>
<jgit.version>5.5.1.201910021850-r</jgit.version>
<atomix.version>3.1.5</atomix.version>
<spark.scala.version>2.11.8</spark.scala.version>
diff --git a/submarine-workbench/interpreter/interpreter-engine/pom.xml b/submarine-workbench/interpreter/interpreter-core/pom.xml
similarity index 96%
rename from submarine-workbench/interpreter/interpreter-engine/pom.xml
rename to submarine-workbench/interpreter/interpreter-core/pom.xml
index f8bce7b..6cd4e3a 100644
--- a/submarine-workbench/interpreter/interpreter-engine/pom.xml
+++ b/submarine-workbench/interpreter/interpreter-core/pom.xml
@@ -30,10 +30,10 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>submarine-interpreter-engine</artifactId>
+ <artifactId>submarine-interpreter-core</artifactId>
<version>0.4.0-SNAPSHOT</version>
- <name>Submarine: Interpreter Engine</name>
- <description>Submarine Interpreter Engine</description>
+ <name>Submarine: Interpreter Core</name>
+ <description>Submarine Interpreter Core</description>
<dependencies>
<dependency>
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/AbstractInterpreter.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/AbstractInterpreter.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/AbstractInterpreter.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/AbstractInterpreter.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/Interpreter.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/Interpreter.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/Interpreter.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/Interpreter.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterException.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterException.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterException.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterException.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterGroup.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterGroup.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterGroup.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterGroup.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterResult.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterResult.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterResult.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterResult.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterResultMessage.java b/submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterResultMessage.java
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterResultMessage.java
rename to submarine-workbench/interpreter/interpreter-core/src/main/java/org/apache/submarine/interpreter/InterpreterResultMessage.java
diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/resources/log4j.properties b/submarine-workbench/interpreter/interpreter-core/src/main/resources/log4j.properties
similarity index 100%
rename from submarine-workbench/interpreter/interpreter-engine/src/main/resources/log4j.properties
rename to submarine-workbench/interpreter/interpreter-core/src/main/resources/log4j.properties
diff --git a/submarine-workbench/interpreter/pom.xml b/submarine-workbench/interpreter/pom.xml
index 79d6ad0..d2fcde9 100644
--- a/submarine-workbench/interpreter/pom.xml
+++ b/submarine-workbench/interpreter/pom.xml
@@ -33,11 +33,11 @@
<artifactId>submarine-interpreter</artifactId>
<version>0.4.0-SNAPSHOT</version>
<packaging>pom</packaging>
- <name>Submarine: Interpreter Parent</name>
+ <name>Submarine: Interpreter</name>
<description>Submarine Interpreter Parent</description>
<modules>
- <module>interpreter-engine</module>
+ <module>interpreter-core</module>
<module>python-interpreter</module>
<module>spark-interpreter</module>
</modules>
diff --git a/submarine-workbench/interpreter/python-interpreter/pom.xml b/submarine-workbench/interpreter/python-interpreter/pom.xml
index f16fdde..2e6a0b3 100644
--- a/submarine-workbench/interpreter/python-interpreter/pom.xml
+++ b/submarine-workbench/interpreter/python-interpreter/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.submarine</groupId>
- <artifactId>interpreter</artifactId>
+ <artifactId>submarine-interpreter</artifactId>
<version>0.4.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -38,7 +38,7 @@
<dependencies>
<dependency>
<groupId>org.apache.submarine</groupId>
- <artifactId>submarine-interpreter-engine</artifactId>
+ <artifactId>submarine-interpreter-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -213,7 +213,7 @@
<goal>shade</goal>
</goals>
<configuration>
- <outputFile>target/submarine-${artifactId}-${project.version}-shade.jar</outputFile>
+ <outputFile>target/${artifactId}-${project.version}-shade.jar</outputFile>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -226,7 +226,7 @@
</filters>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.submarine.interpreter.AbstractInterpreter</mainClass>
+ <mainClass>org.apache.submarine.interpreter.InterpreterProcess</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
diff --git a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/PythonInterpreterTest.java b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/PythonInterpreterTest.java
index 8096ba7..8180e4b 100644
--- a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/PythonInterpreterTest.java
+++ b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/PythonInterpreterTest.java
@@ -63,7 +63,7 @@ public class PythonInterpreterTest {
public void calcOnePlusOne() throws InterpreterException {
String code = "1+1";
InterpreterResult result = pythonInterpreterForCancel.interpret(code);
- LOG.info("result = {}", result);
+ LOG.info("result = {}", result.toString());
assertEquals(result.code(), InterpreterResult.Code.SUCCESS);
// 1 + 1 = 2 + '\n'
diff --git a/submarine-workbench/interpreter/spark-interpreter/pom.xml b/submarine-workbench/interpreter/spark-interpreter/pom.xml
index 599828b..9f0ca98 100644
--- a/submarine-workbench/interpreter/spark-interpreter/pom.xml
+++ b/submarine-workbench/interpreter/spark-interpreter/pom.xml
@@ -30,7 +30,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-interpreter</artifactId>
+ <artifactId>submarine-spark-interpreter</artifactId>
<version>0.4.0-SNAPSHOT</version>
<name>Submarine: Interpreter Spark</name>
<description>Submarine spark Interpreter</description>
@@ -38,7 +38,7 @@
<dependencies>
<dependency>
<groupId>org.apache.submarine</groupId>
- <artifactId>submarine-interpreter-engine</artifactId>
+ <artifactId>submarine-interpreter-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -199,7 +199,7 @@
<goal>shade</goal>
</goals>
<configuration>
- <outputFile>target/submarine-${artifactId}-${project.version}-shade.jar</outputFile>
+ <outputFile>target/${artifactId}-${project.version}-shade.jar</outputFile>
<filters>
<filter>
<artifact>*:*</artifact>
@@ -218,10 +218,6 @@
</transformers>
<relocations>
<relocation>
- <pattern>org.apache.zeppelin</pattern>
- <shadedPattern>${shaded.dependency.prefix}.org.apache.zeppelin</shadedPattern>
- </relocation>
- <relocation>
<pattern>io.grpc</pattern>
<shadedPattern>${shaded.dependency.prefix}.io.grpc</shadedPattern>
</relocation>
diff --git a/submarine-workbench/pom.xml b/submarine-workbench/pom.xml
index d5b6d15..da9f0c9 100644
--- a/submarine-workbench/pom.xml
+++ b/submarine-workbench/pom.xml
@@ -37,7 +37,7 @@
<description>Submarine Workbench</description>
<modules>
- <!--module>interpreter</module-->
+ <module>interpreter</module>
<module>workbench-web</module>
<module>workbench-web-ng</module>
</modules>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org