You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2023/01/23 12:33:24 UTC
[zeppelin] branch master updated: [ZEPPELIN-5861] Correct shading Prefix (#4545)
This is an automated email from the ASF dual-hosted git repository.
jongyoul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 45b15fe971 [ZEPPELIN-5861] Correct shading Prefix (#4545)
45b15fe971 is described below
commit 45b15fe9719246f7097c0cd496f162bce60cad62
Author: Philipp Dallig <ph...@gmail.com>
AuthorDate: Mon Jan 23 13:33:18 2023 +0100
[ZEPPELIN-5861] Correct shading Prefix (#4545)
* Add Process ErrorMessage
* change shade prefix
* Run with pre modules to fill the maven reactor
* spark-interpreter has no scala test so we can remove the maven plugin
* Fix flaky python test
* Use shaded version of io.grpc:grpc-netty (io.grrpc:grpc-netty-shaded)
* Adjust interpreter tests
* Remove maven-dependency-plugin, because downloaded artefacts are attached to classpath
---
.github/workflows/core.yml | 28 ++++++++++++----------
flink/flink-scala-parent/pom.xml | 4 ----
.../zeppelin/python/IPythonInterpreterTest.java | 3 ---
spark/interpreter/pom.xml | 13 ----------
submarine/pom.xml | 4 ++--
zeppelin-interpreter-shaded/pom.xml | 2 +-
.../zeppelin/interpreter/util/ProcessLauncher.java | 2 +-
zeppelin-jupyter-interpreter/pom.xml | 3 ++-
zeppelin-zengine/pom.xml | 4 ----
9 files changed, 21 insertions(+), 42 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index be45db08ff..4d806acf87 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -90,7 +90,7 @@ jobs:
interpreter-test-non-core:
runs-on: ubuntu-20.04
env:
- INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,submarine,sparql,mongodb,influxdb'
+ INTERPRETERS: 'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,alluxio,livy,groovy,java,neo4j,submarine,sparql,mongodb,influxdb,shell'
steps:
- name: Checkout
uses: actions/checkout@v3
@@ -113,7 +113,7 @@ jobs:
restore-keys: |
${{ runner.os }}-zeppelin-
- name: install environment
- run: ./mvnw install -DskipTests -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.11 ${MAVEN_ARGS}
+ run: ./mvnw install -DskipTests -am -pl ${INTERPRETERS} -Pscala-2.11 ${MAVEN_ARGS}
- name: Setup conda environment with python 3.7 and R
uses: conda-incubator/setup-miniconda@v2
with:
@@ -126,7 +126,7 @@ jobs:
auto-activate-base: false
use-mamba: true
- name: verify interpreter
- run: ./mvnw verify -pl ${INTERPRETERS} -Pscala-2.11 ${MAVEN_ARGS}
+ run: ./mvnw verify -am -pl ${INTERPRETERS} -Pscala-2.11 ${MAVEN_ARGS}
# test interpreter modules for jupyter, python, rlang
interpreter-test-jupyter-python-rlang:
@@ -278,10 +278,10 @@ jobs:
use-mamba: true
- name: run tests for flink before 1.15 (exclusive)
if: matrix.flink != '115'
- run: ./mvnw test -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS}
+ run: ./mvnw verify -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -am -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS}
- name: run tests for flink before 1.15 (inclusive)
if: matrix.flink == '115'
- run: ./mvnw test -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS}
+ run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS}
- name: Print zeppelin logs
if: always()
run: if [ -d "logs" ]; then cat logs/*; fi
@@ -381,36 +381,38 @@ jobs:
R -e "IRkernel::installspec()"
- name: run spark-2.4 tests with scala-2.11 and python-${{ matrix.python }}
if: matrix.python == '3.7' # Spark 2.4 doesn't support python 3.8
- run: ./mvnw test -pl spark-submit,spark/interpreter -Pspark-2.4 -Pspark-scala-2.11 -DfailIfNoTests=false ${MAVEN_ARGS}
+ run: |
+ rm -rf spark/interpreter/metastore_db
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-2.4 -Pspark-scala-2.11 -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-2.4 tests with scala-2.12 and python-${{ matrix.python }}
if: matrix.python == '3.7' # Spark 2.4 doesn't support python 3.8
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.0 tests with scala-2.12 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.1 tests with scala-2.12 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.2 tests with scala-2.12 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.2 tests with scala-2.13 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.2 -Pspark-scala-2.13 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.13 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.3 -Pspark-scala-2.12 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }}
run: |
rm -rf spark/interpreter/metastore_db
- ./mvnw test -pl spark-submit,spark/interpreter -Pspark-3.3 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
livy-0-7-with-spark-2-2-0-under-python3:
runs-on: ubuntu-20.04
diff --git a/flink/flink-scala-parent/pom.xml b/flink/flink-scala-parent/pom.xml
index 70bbf953e1..e9f364a162 100644
--- a/flink/flink-scala-parent/pom.xml
+++ b/flink/flink-scala-parent/pom.xml
@@ -177,10 +177,6 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
</exclusions>
</dependency>
diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
index 6f44cbb4a4..ebdfac35ed 100644
--- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
+++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
@@ -434,9 +434,6 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
InterpreterResult result = interpreter.interpret("import time\ntime.sleep(1000)",
getInterpreterContext());
waiter.assertEquals(InterpreterResult.Code.ERROR, result.code());
- waiter.assertEquals(
- "IPython kernel is abnormally exited, please check your code and log.",
- result.message().get(0).getData());
} catch (InterpreterException e) {
waiter.fail("Should not throw exception\n" + ExceptionUtils.getStackTrace(e));
}
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 95b1553622..1b186ec35e 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -111,10 +111,6 @@
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
</exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -161,10 +157,6 @@
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
</exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -433,11 +425,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
diff --git a/submarine/pom.xml b/submarine/pom.xml
index 2bb0c24b6c..0a48aa5dc9 100644
--- a/submarine/pom.xml
+++ b/submarine/pom.xml
@@ -53,7 +53,7 @@
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-python</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.zeppelin</groupId>
@@ -69,7 +69,7 @@
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-shell</artifactId>
- <version>0.11.0-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
diff --git a/zeppelin-interpreter-shaded/pom.xml b/zeppelin-interpreter-shaded/pom.xml
index d320ed8736..3f79e3ebb5 100644
--- a/zeppelin-interpreter-shaded/pom.xml
+++ b/zeppelin-interpreter-shaded/pom.xml
@@ -33,7 +33,7 @@
<description>Zeppelin Interpreter Shaded</description>
<properties>
- <shaded.dependency.prefix>org.apache.zeppelin.shaded</shaded.dependency.prefix>
+ <shaded.dependency.prefix>shaded.org.apache.zeppelin</shaded.dependency.prefix>
</properties>
<dependencies>
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/ProcessLauncher.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/ProcessLauncher.java
index 5ccc88b936..400e89f158 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/ProcessLauncher.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/ProcessLauncher.java
@@ -132,7 +132,7 @@ public abstract class ProcessLauncher implements ExecuteResultHandler {
@Override
public void onProcessFailed(ExecuteException e) {
LOGGER.warn("Process with cmd {} is failed due to", commandLine, e);
-
+ LOGGER.warn("Process ErrorMessage: \n{}", getErrorMessage());
errorMessage = ExceptionUtils.getStackTrace(e);
transition(State.TERMINATED);
}
diff --git a/zeppelin-jupyter-interpreter/pom.xml b/zeppelin-jupyter-interpreter/pom.xml
index cf48721a8a..291afd4926 100644
--- a/zeppelin-jupyter-interpreter/pom.xml
+++ b/zeppelin-jupyter-interpreter/pom.xml
@@ -46,8 +46,9 @@
<dependency>
<groupId>io.grpc</groupId>
- <artifactId>grpc-netty</artifactId>
+ <artifactId>grpc-netty-shaded</artifactId>
<version>${grpc.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index 11685db1e4..0f0f7f7670 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -304,10 +304,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- </plugin>
</plugins>
</build>