You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by bz...@apache.org on 2016/06/23 13:16:53 UTC
zeppelin git commit: [ZEPPELIN-1045] Apply new mechanism to
PythonInterpreter
Repository: zeppelin
Updated Branches:
refs/heads/master 83602f5c2 -> ff4973d43
[ZEPPELIN-1045] Apply new mechanism to PythonInterpreter
### What is this PR for?
This PR is applying new interpreter register mechanism to python interpreter.
### What type of PR is it?
Improvement
### What is the Jira issue?
[ZEPPELIN-1045](https://issues.apache.org/jira/browse/ZEPPELIN-1045)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Mina Lee <mi...@apache.org>
Closes #1063 from minahlee/ZEPPELIN-1045 and squashes the following commits:
66b8f73 [Mina Lee] Add zeppelin.python.maxResult property to python interpreter
5013890 [Mina Lee] Apply new mechanism to PythonInterpreter
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/ff4973d4
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/ff4973d4
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/ff4973d4
Branch: refs/heads/master
Commit: ff4973d4354583fa39f7c90149f1e50bb6beebc7
Parents: 83602f5
Author: Mina Lee <mi...@apache.org>
Authored: Wed Jun 22 15:41:16 2016 -0700
Committer: Alexander Bezzubov <bz...@apache.org>
Committed: Thu Jun 23 22:16:39 2016 +0900
----------------------------------------------------------------------
docs/interpreter/python.md | 7 +++-
.../zeppelin/python/PythonInterpreter.java | 22 ++++---------
.../apache/zeppelin/python/PythonProcess.java | 2 +-
python/src/main/resources/bootstrap.py | 5 ++-
.../src/main/resources/interpreter-setting.json | 21 ++++++++++++
.../zeppelin/python/PythonInterpreterTest.java | 34 +++++++++-----------
6 files changed, 53 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/docs/interpreter/python.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/python.md b/docs/interpreter/python.md
index 0476c68..34134a1 100644
--- a/docs/interpreter/python.md
+++ b/docs/interpreter/python.md
@@ -16,12 +16,17 @@ group: manual
<th>Description</th>
</tr>
<tr>
- <td>python</td>
+ <td>zeppelin.python</td>
<td>python</td>
<td>Path of the already installed Python binary (could be python2 or python3).
If python is not in your $PATH you can set the absolute directory (example : /usr/bin/python)
</td>
</tr>
+ <tr>
+ <td>zeppelin.python.maxResult</td>
+ <td>1000</td>
+ <td>Max number of dataframe rows to display.</td>
+ </tr>
</table>
## Enabling Python Interpreter
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
----------------------------------------------------------------------
diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
index b1da981..9dd4ed7 100644
--- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
@@ -20,7 +20,6 @@ package org.apache.zeppelin.python;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@@ -50,27 +49,16 @@ public class PythonInterpreter extends Interpreter {
public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py";
public static final String ZEPPELIN_PYTHON = "zeppelin.python";
public static final String DEFAULT_ZEPPELIN_PYTHON = "python";
+ public static final String MAX_RESULT = "zeppelin.python.maxResult";
private Integer port;
private GatewayServer gatewayServer;
- private long pythonPid;
private Boolean py4J = false;
private InterpreterContext context;
+ private int maxResult;
PythonProcess process = null;
- static {
- Interpreter.register(
- "python",
- "python",
- PythonInterpreter.class.getName(),
- new InterpreterPropertyBuilder()
- .add(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON,
- "Python directory. Default : python (assume python is in your $PATH)")
- .build()
- );
- }
-
public PythonInterpreter(Properties property) {
super(property);
}
@@ -80,6 +68,7 @@ public class PythonInterpreter extends Interpreter {
logger.info("Starting Python interpreter .....");
logger.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON));
+ maxResult = Integer.valueOf(getProperty(MAX_RESULT));
process = getPythonProcess();
try {
@@ -223,7 +212,7 @@ public class PythonInterpreter extends Interpreter {
return context.getGui();
}
- public Integer getPy4JPort() {
+ public Integer getPy4jPort() {
return port;
}
@@ -247,4 +236,7 @@ public class PythonInterpreter extends Interpreter {
return port;
}
+ public int getMaxResult() {
+ return maxResult;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
----------------------------------------------------------------------
diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
index 364d372..348ced6 100644
--- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
+++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
@@ -92,7 +92,7 @@ public class PythonProcess {
String output = "";
String line;
while (!(line = reader.readLine()).contains("*!?flush reader!?*")) {
- logger.debug("Readed line from python shell : " + line);
+ logger.debug("Read line from python shell : " + line);
if (line.equals("...")) {
logger.warn("Syntax error ! ");
output += "Syntax error ! ";
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/resources/bootstrap.py
----------------------------------------------------------------------
diff --git a/python/src/main/resources/bootstrap.py b/python/src/main/resources/bootstrap.py
index 4f0dc5e..04a5f53 100644
--- a/python/src/main/resources/bootstrap.py
+++ b/python/src/main/resources/bootstrap.py
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# PYTHON 2 / 3 comptability :
+# PYTHON 2 / 3 compatibility :
# bootstrap.py must be runnable with Python 2 or 3
# Remove interactive mode displayhook
@@ -36,7 +36,7 @@ signal.signal(signal.SIGINT, intHandler)
def help():
print ('%html')
print ('<h2>Python Interpreter help</h2>')
- print ('<h3>Python 2 & 3 comptability</h3>')
+ print ('<h3>Python 2 & 3 compatibility</h3>')
print ('<p>The interpreter is compatible with Python 2 & 3.<br/>')
print ('To change Python version, ')
print ('change in the interpreter configuration the python to the ')
@@ -100,4 +100,3 @@ class PyZeppelinContext():
print (self.errorMsg)
z = PyZeppelinContext("")
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json
new file mode 100644
index 0000000..8508bd0
--- /dev/null
+++ b/python/src/main/resources/interpreter-setting.json
@@ -0,0 +1,21 @@
+[
+ {
+ "group": "python",
+ "name": "python",
+ "className": "org.apache.zeppelin.python.PythonInterpreter",
+ "properties": {
+ "zeppelin.python": {
+ "envName": null,
+ "propertyName": "zeppelin.python",
+ "defaultValue": "python",
+ "description": "Python directory. It is set to python by default.(assume python is in your $PATH)"
+ },
+ "zeppelin.python.maxResult": {
+ "envName": null,
+ "propertyName": "zeppelin.python.maxResult",
+ "defaultValue": "1000",
+ "description": "Max number of dataframe rows to display."
+ }
+ }
+ }
+]
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
----------------------------------------------------------------------
diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
index 2944903..dbd1346 100644
--- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
+++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
@@ -17,6 +17,7 @@
package org.apache.zeppelin.python;
+import static org.apache.zeppelin.python.PythonInterpreter.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -53,13 +54,17 @@ public class PythonInterpreterTest {
Logger logger = LoggerFactory.getLogger(PythonProcess.class);
- public static final String ZEPPELIN_PYTHON = "zeppelin.python";
- public static final String DEFAULT_ZEPPELIN_PYTHON = "python";
-
PythonInterpreter pythonInterpreter = null;
PythonProcess mockPythonProcess;
String cmdHistory;
+ public static Properties getPythonTestProperties() {
+ Properties p = new Properties();
+ p.setProperty(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON);
+ p.setProperty(MAX_RESULT, "1000");
+ return p;
+ }
+
@Before
public void beforeTest() {
cmdHistory = "";
@@ -79,20 +84,15 @@ public class PythonInterpreterTest {
logger.error("Can't initiate python process", e);
}
- Properties properties = new Properties();
- properties.put(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON);
- pythonInterpreter = spy(new PythonInterpreter(properties));
+ pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties()));
when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess);
-
try {
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError");
} catch (IOException e) {
e.printStackTrace();
}
-
-
}
@Test
@@ -111,7 +111,7 @@ public class PythonInterpreterTest {
py4j JavaGateway is not running
*/
pythonInterpreter.open();
- assertNull(pythonInterpreter.getPy4JPort());
+ assertNull(pythonInterpreter.getPy4jPort());
assertTrue(cmdHistory.contains("def help()"));
assertTrue(cmdHistory.contains("class PyZeppelinContext():"));
@@ -122,8 +122,7 @@ public class PythonInterpreterTest {
}
@Test
- public void testPy4JInstalled() {
-
+ public void testPy4jInstalled() {
/*
If Py4J installed, bootstrap_input.py
@@ -137,7 +136,7 @@ public class PythonInterpreterTest {
e.printStackTrace();
}
pythonInterpreter.open();
- Integer py4jPort = pythonInterpreter.getPy4JPort();
+ Integer py4jPort = pythonInterpreter.getPy4jPort();
assertNotNull(py4jPort);
assertTrue(cmdHistory.contains("def help()"));
@@ -147,8 +146,7 @@ public class PythonInterpreterTest {
assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")"));
assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input"));
-
- assertTrue(checkSocketAdress(py4jPort));
+ assertTrue(checkSocketAddress(py4jPort));
}
@@ -162,12 +160,12 @@ public class PythonInterpreterTest {
e.printStackTrace();
}
pythonInterpreter.open();
- Integer py4jPort = pythonInterpreter.getPy4JPort();
+ Integer py4jPort = pythonInterpreter.getPy4jPort();
assertNotNull(py4jPort);
pythonInterpreter.close();
- assertFalse(checkSocketAdress(py4jPort));
+ assertFalse(checkSocketAddress(py4jPort));
try {
verify(mockPythonProcess, times(1)).close();
} catch (IOException e) {
@@ -189,7 +187,7 @@ public class PythonInterpreterTest {
- private boolean checkSocketAdress(Integer py4jPort) {
+ private boolean checkSocketAddress(Integer py4jPort) {
Socket s = new Socket();
SocketAddress sa = new InetSocketAddress("localhost", py4jPort);
Boolean working = null;