You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2019/03/02 14:00:32 UTC
[zeppelin] branch branch-0.8 updated: ZEPPELIN-4005.
SparkRInterpreter is broken for spark 2.1.3 and 2.2.2
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.8
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.8 by this push:
new 95bfacf ZEPPELIN-4005. SparkRInterpreter is broken for spark 2.1.3 and 2.2.2
95bfacf is described below
commit 95bfacf906154710f58c156396805c79578b11dd
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Fri Feb 15 17:25:55 2019 +0800
ZEPPELIN-4005. SparkRInterpreter is broken for spark 2.1.3 and 2.2.2
### What is this PR for?
The security patch about the security communication between JVM and R process is backported to spark 2.13 and 2.2.2. And Zeppelin didn't catch that. This PR fix it and also update the test case.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-4005
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3307 from zjffdu/ZEPPELIN-4005 and squashes the following commits:
fcae651a9 [Jeff Zhang] ZEPPELIN-4005. SparkRInterpreter is broken for spark 2.1.3 and 2.2.2
---
.../org/apache/zeppelin/spark/SparkVersion.java | 21 ++++++++++++++++-----
.../org/apache/zeppelin/spark/SparkVersionTest.java | 8 ++++++++
.../zeppelin/rest/ZeppelinSparkClusterTest21.java | 2 +-
.../zeppelin/rest/ZeppelinSparkClusterTest22.java | 2 +-
.../interpreter/SparkIntegrationTest21.java | 2 +-
.../interpreter/SparkIntegrationTest22.java | 2 +-
6 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index df3c754..947a36f 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -43,6 +43,9 @@ public class SparkVersion {
private int version;
+ private int majorVersion;
+ private int minorVersion;
+ private int patchVersion;
private String versionString;
SparkVersion(String versionString) {
@@ -57,11 +60,11 @@ public class SparkVersion {
}
String versions[] = numberPart.split("\\.");
- int major = Integer.parseInt(versions[0]);
- int minor = Integer.parseInt(versions[1]);
- int patch = Integer.parseInt(versions[2]);
+ this.majorVersion = Integer.parseInt(versions[0]);
+ this.minorVersion = Integer.parseInt(versions[1]);
+ this.patchVersion = Integer.parseInt(versions[2]);
// version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
- version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch));
+ version = Integer.parseInt(String.format("%d%02d%02d", majorVersion, minorVersion, patchVersion));
} catch (Exception e) {
logger.error("Can not recognize Spark version " + versionString +
". Assume it's a future release", e);
@@ -112,7 +115,10 @@ public class SparkVersion {
}
public boolean isSecretSocketSupported() {
- return this.newerThanEquals(SPARK_2_3_1);
+ return this.newerThanEquals(SparkVersion.SPARK_2_4_0) ||
+ this.newerThanEqualsPatchVersion(SPARK_2_3_1) ||
+ this.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.2.2")) ||
+ this.newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.1.3"));
}
public boolean equals(Object versionToCompare) {
return version == ((SparkVersion) versionToCompare).version;
@@ -126,6 +132,11 @@ public class SparkVersion {
return version >= versionToCompare.version;
}
+ public boolean newerThanEqualsPatchVersion(SparkVersion versionToCompare) {
+ return version / 100 == versionToCompare.version / 100 &&
+ version % 100 >= versionToCompare.version % 100;
+ }
+
public boolean olderThan(SparkVersion versionToCompare) {
return version < versionToCompare.version;
}
diff --git a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
index 3dc8f4e..f87caed 100644
--- a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
+++ b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
@@ -64,6 +64,14 @@ public class SparkVersionTest {
assertFalse(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_1_0));
assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0));
+ // test newerThanEqualsPatchVersion
+ assertTrue(SparkVersion.fromVersionString("2.3.1")
+ .newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.3.0")));
+ assertFalse(SparkVersion.fromVersionString("2.3.1")
+ .newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.3.2")));
+ assertFalse(SparkVersion.fromVersionString("2.3.1")
+ .newerThanEqualsPatchVersion(SparkVersion.fromVersionString("2.2.0")));
+
// conversion
assertEquals(10200, SparkVersion.SPARK_1_2_0.toNumber());
assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString());
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest21.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest21.java
index f6b390d..db54b64 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest21.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest21.java
@@ -34,7 +34,7 @@ public class ZeppelinSparkClusterTest21 extends ZeppelinSparkClusterTest {
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.1.2"}
+ {"2.1.3"}
});
}
}
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest22.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest22.java
index cae24ef..eca09c6 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest22.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest22.java
@@ -34,7 +34,7 @@ public class ZeppelinSparkClusterTest22 extends ZeppelinSparkClusterTest {
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.2.1"}
+ {"2.2.2"}
});
}
}
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest21.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest21.java
index bcb61e5..4238528 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest21.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest21.java
@@ -33,7 +33,7 @@ public class SparkIntegrationTest21 extends SparkIntegrationTest{
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.1.2"}
+ {"2.1.3"}
});
}
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest22.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest22.java
index 97f9b06..02ab5dc 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest22.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkIntegrationTest22.java
@@ -33,7 +33,7 @@ public class SparkIntegrationTest22 extends SparkIntegrationTest{
@Parameterized.Parameters
public static List<Object[]> data() {
return Arrays.asList(new Object[][]{
- {"2.2.1"}
+ {"2.2.2"}
});
}