You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by fe...@apache.org on 2017/01/04 05:54:13 UTC
zeppelin git commit: [ZEPPELIN-1609] using pyspark(python3) with livy
interperter
Repository: zeppelin
Updated Branches:
refs/heads/master c2a91e96c -> a5b437d7f
[ZEPPELIN-1609] using pyspark(python3) with livy interperter
### What is this PR for?
using pyspark(python3) with livy interperter
### What type of PR is it?
Improvement
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-1609
### How should this be tested?
%livy.pyspark3
3/2
check result is "1.5"
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? - NO
* Is there breaking changes for older versions? - NO
* Does this needs documentation? - NO
Author: purechoc <pu...@gmail.com>
Author: purechoc <pu...@ncsoft.com>
Closes #1587 from purechoc/zeppelin-livy-pyspark3 and squashes the following commits:
6f7391f [purechoc] update template
7c0f70d [purechoc] refactor code
476ae25 [purechoc] Merge remote-tracking branch 'upstream3/master' into zeppelin-livy-pyspark3
55d486f [purechoc] Trigger
e5db640 [purechoc] add properties
51140f2 [purechoc] Trigger
3fc98cd [purechoc] add configuration
d0e2293 [purechoc] Trigger
7e42108 [purechoc] add %livy.python3
3775e47 [purechoc] add %livy.python3
c0babf6 [purechoc] add %livy.python3
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/a5b437d7
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/a5b437d7
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/a5b437d7
Branch: refs/heads/master
Commit: a5b437d7fe5adcdb4546b91af5f77b0252d2fd64
Parents: c2a91e9
Author: purechoc <pu...@gmail.com>
Authored: Thu Dec 29 09:08:18 2016 +0900
Committer: Felix Cheung <fe...@apache.org>
Committed: Tue Jan 3 21:54:09 2017 -0800
----------------------------------------------------------------------
conf/zeppelin-site.xml.template | 2 +-
.../zeppelin/livy/LivyPySpark3Interpreter.java | 46 ++++++++++++++++++++
.../src/main/resources/interpreter-setting.json | 20 +++++++++
.../zeppelin/conf/ZeppelinConfiguration.java | 1 +
4 files changed, 68 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index 2edb056..08104c7 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -190,7 +190,7 @@
<property>
<name>zeppelin.interpreters</name>
- <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlIn
terpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
+ <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlIn
terpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java
new file mode 100644
index 0000000..4a0314c
--- /dev/null
+++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.livy;
+
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * Livy PySpark interpreter for Zeppelin.
+ */
+public class LivyPySpark3Interpreter extends BaseLivyInterprereter {
+
+ public LivyPySpark3Interpreter(Properties property) {
+ super(property);
+ }
+
+ @Override
+ public String getSessionKind() {
+ return "pyspark3";
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index 8a4e4d7..4be537a 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -155,6 +155,26 @@
},
{
"group": "livy",
+ "name": "pyspark3",
+ "className": "org.apache.zeppelin.livy.LivyPySpark3Interpreter",
+ "properties": {
+ },
+ "option": {
+ "remote": true,
+ "port": -1,
+ "perNote": "shared",
+ "perUser": "scoped",
+ "isExistingProcess": false,
+ "setPermission": false,
+ "users": []
+ },
+ "editor": {
+ "language": "python",
+ "editOnDblClick": false
+ }
+ },
+ {
+ "group": "livy",
"name": "sparkr",
"className": "org.apache.zeppelin.livy.LivySparkRInterpreter",
"properties": {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 499c338..001585d 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -535,6 +535,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.livy.LivySparkInterpreter,"
+ "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
+ "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
+ + "org.apache.zeppelin.livy.LivyPySpark3Interpreter,"
+ "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ "org.apache.zeppelin.file.HDFSFileInterpreter,"