You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2016/09/09 11:08:28 UTC
zeppelin git commit: [ZEPPELIN-1390] SparkInterpreter does not work
for Spark2 version of HDP 2.5
Repository: zeppelin
Updated Branches:
refs/heads/master 56bbff3d3 -> d005c7967
[ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5
### What is this PR for?
Spark2 version of HDP 2.5 is "2.0.0.2.5.0.0-1245". Currently, Zeppelin parses this version to integer 2002500, which is higher than version "2.0.0" (i.e. 200 in integer). Therefore, Zeppelin thinks it's not supported version and fail.
I fixed some codes to consider major, minor, and patch version. Now, version parsed using only first 3 numbers (major, minor, patch) and parsed into a 5-digit integer. For example, 2.0.0 -> 20000, 1.6.2 -> 10602, 1.10.1 -> 11001. This makes HDP 2.5 compatible, and also make it more robust than before. Since maybe Spark's minor version can be a 2-digit number in the future
### What type of PR is it?
[Bug Fix]
### What is the Jira issue?
[ZEPPELIN-1390](https://issues.apache.org/jira/browse/ZEPPELIN-1390)
### How should this be tested?
I added SparkVersion test codes.
Author: Jun <i2...@gmail.com>
Closes #1381 from tae-jun/ZEPPELIN-1390 and squashes the following commits:
bae5264 [Jun] Merge remote-tracking branch 'origin/master' into ZEPPELIN-1390
f5531ab [Jun] [ZEPPELIN-1390] Fix Python and R number versions
6ad8049 [Jun] [ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d005c796
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d005c796
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d005c796
Branch: refs/heads/master
Commit: d005c7967a57a572cc6d6620185ac6aedbfc56e3
Parents: 56bbff3
Author: Jun <i2...@gmail.com>
Authored: Sun Sep 4 10:53:32 2016 +0900
Committer: Mina Lee <mi...@apache.org>
Committed: Fri Sep 9 13:08:16 2016 +0200
----------------------------------------------------------------------
.../main/java/org/apache/zeppelin/spark/SparkVersion.java | 10 ++++++++--
spark/src/main/resources/R/zeppelin_sparkr.R | 2 +-
spark/src/main/resources/python/zeppelin_pyspark.py | 6 +++---
.../java/org/apache/zeppelin/spark/SparkVersionTest.java | 9 +++++++--
4 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d005c796/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index 17f2de7..c8a7c79 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -52,13 +52,19 @@ public class SparkVersion {
if (pos > 0) {
numberPart = versionString.substring(0, pos);
}
- version = Integer.parseInt(numberPart.replaceAll("\\.", ""));
+
+ String versions[] = numberPart.split("\\.");
+ int major = Integer.parseInt(versions[0]);
+ int minor = Integer.parseInt(versions[1]);
+ int patch = Integer.parseInt(versions[2]);
+ // version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
+ version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch));
} catch (Exception e) {
logger.error("Can not recognize Spark version " + versionString +
". Assume it's a future release", e);
// assume it is future release
- version = 999;
+ version = 99999;
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d005c796/spark/src/main/resources/R/zeppelin_sparkr.R
----------------------------------------------------------------------
diff --git a/spark/src/main/resources/R/zeppelin_sparkr.R b/spark/src/main/resources/R/zeppelin_sparkr.R
index d951774..8e66421 100644
--- a/spark/src/main/resources/R/zeppelin_sparkr.R
+++ b/spark/src/main/resources/R/zeppelin_sparkr.R
@@ -42,7 +42,7 @@ assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv)
# setup spark env
assign(".sc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkContext"), envir = SparkR:::.sparkREnv)
assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
-if (version >= 200) {
+if (version >= 20000) {
assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv)
assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d005c796/spark/src/main/resources/python/zeppelin_pyspark.py
----------------------------------------------------------------------
diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/src/main/resources/python/zeppelin_pyspark.py
index 9a40556..3e6535f 100644
--- a/spark/src/main/resources/python/zeppelin_pyspark.py
+++ b/spark/src/main/resources/python/zeppelin_pyspark.py
@@ -107,9 +107,9 @@ class PyZeppelinContext(dict):
class SparkVersion(object):
- SPARK_1_4_0 = 140
- SPARK_1_3_0 = 130
- SPARK_2_0_0 = 200
+ SPARK_1_4_0 = 10400
+ SPARK_1_3_0 = 10300
+ SPARK_2_0_0 = 20000
def __init__(self, versionNumber):
self.version = versionNumber
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d005c796/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
index 5783c1e..3dc8f4e 100644
--- a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
+++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
@@ -24,7 +24,7 @@ public class SparkVersionTest {
@Test
public void testUnknownSparkVersion() {
- assertEquals(999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
+ assertEquals(99999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
}
@Test
@@ -33,6 +33,8 @@ public class SparkVersionTest {
assertFalse(SparkVersion.fromVersionString("1.5.9").isUnsupportedVersion());
assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion());
assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion());
+ // should support spark2 version of HDP 2.5
+ assertFalse(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion());
}
@Test
@@ -40,6 +42,9 @@ public class SparkVersionTest {
// test equals
assertEquals(SparkVersion.SPARK_1_2_0, SparkVersion.fromVersionString("1.2.0"));
assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
+ assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
+ // test spark2 version of HDP 2.5
+ assertEquals(SparkVersion.SPARK_2_0_0, SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245"));
// test newer than
assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_2_0));
@@ -60,7 +65,7 @@ public class SparkVersionTest {
assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0));
// conversion
- assertEquals(120, SparkVersion.SPARK_1_2_0.toNumber());
+ assertEquals(10200, SparkVersion.SPARK_1_2_0.toNumber());
assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString());
}
}