You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2021/02/01 03:03:26 UTC
[zeppelin] branch master updated: [ZEPPELIN-5213]. Move
spark-submit as a separated interpreter module
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new abbae06 [ZEPPELIN-5213]. Move spark-submit as a separated interpreter module
abbae06 is described below
commit abbae06204b4d0d199aa37eeb886e9b8ae7f0bf5
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Fri Jan 22 11:42:40 2021 +0800
[ZEPPELIN-5213]. Move spark-submit as a separated interpreter module
### What is this PR for?
This PR is to move spark-submit from spark module to another separated interpreter module. Because it doesn't make sense to run spark-submit in the same JVM of spark interpreter. It would cause some very confused things. e.g. If running it in yarn mode, the yarn app would never be registered unless SparkContext is created. But if you only `%spark.submit`, SparkContext would never be created.
### What type of PR is it?
[ Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5213
### How should this be tested?
* Manually tested
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #4032 from zjffdu/ZEPPELIN-5213 and squashes the following commits:
382755877 [Jeff Zhang] [ZEPPELIN-5213]. Move spark-submit as a separated interpreter module
---
.github/workflows/core.yml | 26 ++++-----
pom.xml | 1 +
spark-submit/pom.xml | 68 ++++++++++++++++++++++
.../spark/submit}/SparkSubmitInterpreter.java | 6 +-
.../src/main/resources/interpreter-setting.json | 22 +++++++
spark/interpreter/pom.xml | 6 --
.../src/main/resources/interpreter-setting.json | 13 -----
.../zeppelin/integration/SparkIntegrationTest.java | 22 ++++---
.../zeppelin/notebook/ParagraphTextParser.java | 2 +-
.../zeppelin/notebook/ParagraphTextParserTest.java | 8 +++
10 files changed, 130 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 08b9b35..c0e7c21 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -59,10 +59,10 @@ jobs:
conda info
- name: install plugins and interpreter
run: |
- mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B
+ mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B
mvn package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B
- name: run tests with ${{ matrix.hadoop }}
- run: mvn verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false
+ run: mvn verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false
test-interpreter-modules:
runs-on: ubuntu-18.04
env:
@@ -125,7 +125,7 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- mvn install -DskipTests -DskipRat -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown,flink/interpreter,jdbc,shell -am
+ mvn install -DskipTests -DskipRat -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,flink/interpreter,jdbc,shell -am
mvn package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B
- name: run tests
run: mvn test -DskipRat -pl zeppelin-interpreter-integration -Pintegration -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest
@@ -194,10 +194,10 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B
+ mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B
mvn clean package -pl zeppelin-plugins -amd -DskipTests -B
- name: run tests
- run: mvn test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,SparkIntegrationTest30 -DfailIfNoTests=false
+ run: mvn test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,SparkIntegrationTest30 -DfailIfNoTests=false
jdbcIntegrationTest-and-unit-test-of-Spark-2-4-with-Scala-2-11:
runs-on: ubuntu-18.04
steps:
@@ -231,10 +231,10 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B
+ mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B
mvn clean package -pl zeppelin-plugins -amd -DskipTests -B
- name: run tests
- run: mvn test -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B -Dtest=JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false
+ run: mvn test -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B -Dtest=JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false
spark-2-4-and-scale-2-12:
runs-on: ubuntu-18.04
@@ -266,9 +266,9 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- mvn install -DskipTests -DskipRat -pl spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B
+ mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B
- name: run tests
- run: mvn test -DskipRat -pl spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false
+ run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false
spark-2-3-and-scale-2-11-and-other-interpreter:
runs-on: ubuntu-18.04
@@ -300,9 +300,9 @@ jobs:
R -e "IRkernel::installspec()"
- name: install environment
run: |
- mvn install -DskipTests -DskipRat -pl spark/spark-dependencies -am -Pspark-2.3 -Pspark-scala-2.11 -Phadoop2 -B
+ mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.3 -Pspark-scala-2.11 -Phadoop2 -B
- name: run tests
- run: mvn test -DskipRat -pl spark/spark-dependencies -am -Pspark-2.3 -Pspark-scala-2.11 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
+ run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.3 -Pspark-scala-2.11 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
spark-2-2-and-scale-2-10-and-other-interpreter:
runs-on: ubuntu-18.04
@@ -333,9 +333,9 @@ jobs:
run: |
R -e "IRkernel::installspec()"
- name: install environment
- run: mvn install -DskipTests -DskipRat -pl spark/spark-dependencies -am -Pspark-2.2 -Pspark-scala-2.10 -Phadoop2 -B
+ run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.2 -Pspark-scala-2.10 -Phadoop2 -B
- name: run tests
- run: mvn test -DskipRat -pl spark/spark-dependencies -am -Pspark-2.2 -Pspark-scala-2.10 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
+ run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.2 -Pspark-scala-2.10 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
test-livy-0-5-with-spark-2-2-0-under-python3:
runs-on: ubuntu-18.04
steps:
diff --git a/pom.xml b/pom.xml
index c7c5d2c..9552016 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,7 @@
<module>kotlin</module>
<module>groovy</module>
<module>spark</module>
+ <module>spark-submit</module>
<module>submarine</module>
<module>markdown</module>
<module>mongodb</module>
diff --git a/spark-submit/pom.xml b/spark-submit/pom.xml
new file mode 100644
index 0000000..8877c78
--- /dev/null
+++ b/spark-submit/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zeppelin-interpreter-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.10.0-SNAPSHOT</version>
+ <relativePath>../zeppelin-interpreter-parent/pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>zeppelin-spark-submit</artifactId>
+ <packaging>jar</packaging>
+ <version>0.10.0-SNAPSHOT</version>
+ <name>Zeppelin: Spark-Submit interpreter</name>
+
+ <properties>
+ <interpreter.name>spark-submit</interpreter.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>zeppelin-shell</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSubmitInterpreter.java b/spark-submit/src/main/java/org/apache/zeppelin/spark/submit/SparkSubmitInterpreter.java
similarity index 95%
rename from spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSubmitInterpreter.java
rename to spark-submit/src/main/java/org/apache/zeppelin/spark/submit/SparkSubmitInterpreter.java
index d7b26c6..7dce340 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkSubmitInterpreter.java
+++ b/spark-submit/src/main/java/org/apache/zeppelin/spark/submit/SparkSubmitInterpreter.java
@@ -16,7 +16,7 @@
*/
-package org.apache.zeppelin.spark;
+package org.apache.zeppelin.spark.submit;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterOutput;
@@ -32,7 +32,7 @@ import java.util.Properties;
/**
- * Support %spark.submit which run spark-submit command. Internally,
+ * Support %spark-submit which run spark-submit command. Internally,
* it would run shell command via ShellInterpreter.
*
*/
@@ -66,7 +66,7 @@ public class SparkSubmitInterpreter extends ShellInterpreter {
private InterpreterContext context;
private boolean isSparkUrlSent = false;
- public SparkSubmitOutputListener(InterpreterContext context) {
+ SparkSubmitOutputListener(InterpreterContext context) {
this.context = context;
}
diff --git a/spark-submit/src/main/resources/interpreter-setting.json b/spark-submit/src/main/resources/interpreter-setting.json
new file mode 100644
index 0000000..5726432
--- /dev/null
+++ b/spark-submit/src/main/resources/interpreter-setting.json
@@ -0,0 +1,22 @@
+[
+ {
+ "group": "spark-submit",
+ "name": "submit",
+ "className": "org.apache.zeppelin.spark.submit.SparkSubmitInterpreter",
+ "defaultInterpreter": true,
+ "properties": {
+ "SPARK_HOME": {
+ "envName": "SPARK_HOME",
+ "propertyName": "SPARK_HOME",
+ "defaultValue": "",
+ "description": "Location of spark distribution",
+ "type": "string"
+ }
+ },
+ "editor": {
+ "language": "sh",
+ "editOnDblClick": false,
+ "completionSupport": false
+ }
+ }
+]
\ No newline at end of file
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index fb56929..443d03c 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -112,12 +112,6 @@
<dependency>
<groupId>org.apache.zeppelin</groupId>
- <artifactId>zeppelin-shell</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-python</artifactId>
<version>${project.version}</version>
<exclusions>
diff --git a/spark/interpreter/src/main/resources/interpreter-setting.json b/spark/interpreter/src/main/resources/interpreter-setting.json
index 8f28823..9d70f9c 100644
--- a/spark/interpreter/src/main/resources/interpreter-setting.json
+++ b/spark/interpreter/src/main/resources/interpreter-setting.json
@@ -349,18 +349,5 @@
"completionKey": "TAB",
"completionSupport": false
}
- },
-
- {
- "group": "spark",
- "name": "submit",
- "className": "org.apache.zeppelin.spark.SparkSubmitInterpreter",
- "properties": {
- },
- "editor": {
- "language": "sh",
- "editOnDblClick": false,
- "completionSupport": false
- }
}
]
diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
index 50990ba..a29668a 100644
--- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
+++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
@@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterNotFoundException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
@@ -108,7 +109,7 @@ public abstract class SparkIntegrationTest {
InterpreterResult interpreterResult = sparkInterpreter.interpret("sc.version", context);
assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
String detectedSparkVersion = interpreterResult.message().get(0).getData();
- assertTrue(detectedSparkVersion +" doesn't contain " + this.sparkVersion, detectedSparkVersion.contains(this.sparkVersion));
+ assertTrue(detectedSparkVersion + " doesn't contain " + this.sparkVersion, detectedSparkVersion.contains(this.sparkVersion));
interpreterResult = sparkInterpreter.interpret("sc.range(1,10).sum()", context);
assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("45"));
@@ -145,13 +146,6 @@ public abstract class SparkIntegrationTest {
assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(interpreterResult.toString(), InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType());
assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("eruptions waiting"));
-
- // test SparkSubmitInterpreter
- context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
- Interpreter sparkSubmitInterpreter = interpreterFactory.getInterpreter("spark.submit", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("test").createExecutionContext());
- interpreterResult = sparkSubmitInterpreter.interpret("--class org.apache.spark.examples.SparkPi " + sparkHome + "/examples/jars/spark-examples*.jar ", context);
-
- assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
}
@Test
@@ -255,6 +249,18 @@ public abstract class SparkIntegrationTest {
}
}
+ @Test
+ public void testSparkSubmit() throws InterpreterException {
+ InterpreterSetting sparkSubmitInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark-submit");
+ sparkSubmitInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
+ // test SparkSubmitInterpreter
+ InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
+ Interpreter sparkSubmitInterpreter = interpreterFactory.getInterpreter("spark-submit", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("test").createExecutionContext());
+ InterpreterResult interpreterResult = sparkSubmitInterpreter.interpret("--class org.apache.spark.examples.SparkPi " + sparkHome + "/examples/jars/spark-examples*.jar ", context);
+
+ assertEquals(interpreterResult.toString(), InterpreterResult.Code.SUCCESS, interpreterResult.code());
+ }
+
private boolean isSpark2() {
return this.sparkVersion.startsWith("2.");
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphTextParser.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphTextParser.java
index 1f9fe07..db421ea 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphTextParser.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphTextParser.java
@@ -67,7 +67,7 @@ public class ParagraphTextParser {
}
}
- private static final Pattern REPL_PATTERN = Pattern.compile("^(\\s*)%(\\w+(?:\\.\\w+)*)");
+ private static final Pattern REPL_PATTERN = Pattern.compile("^(\\s*)%([a-zA-Z0-9_\\-]+(?:\\.\\w+)*)");
private static int parseLocalProperties(
final String text, int startPos,
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTextParserTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTextParserTest.java
index 31f750f..3857a3b 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTextParserTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTextParserTest.java
@@ -47,6 +47,14 @@ public class ParagraphTextParserTest {
}
@Test
+ public void testSparkSubmit() {
+ ParagraphTextParser.ParseResult parseResult = ParagraphTextParser.parse(
+ "%spark-submit --class A a.jar");
+ assertEquals("spark-submit", parseResult.getIntpText());
+ assertEquals("--class A a.jar", parseResult.getScriptText());
+ }
+
+ @Test
public void testParagraphTextLocalPropertiesAndText() {
ParagraphTextParser.ParseResult parseResult = ParagraphTextParser.parse("%spark.pyspark(pool=pool_1) sc.version");
assertEquals("spark.pyspark", parseResult.getIntpText());