You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2017/07/02 11:03:30 UTC
zeppelin git commit: [ZEPPELIN-2710] Programmatically update progress
bar
Repository: zeppelin
Updated Branches:
refs/heads/master e500373cf -> 968bda800
[ZEPPELIN-2710] Programmatically update progress bar
### What is this PR for?
This PR adds setProgress(n) method to InterpreterContext.
So user can simply update progress bar manually in the code.
This can be useful when user runs some loops that takes lots of time (e.g. training) and user want to display progress with progress bar.
### What type of PR is it?
Feature
### Todos
* [x] - add setProgress() method
* [x] - Unittest
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-2710
### How should this be tested?
run following example and see if progress bar is updating
```
%python
import time
ic = z.getInterpreterContext()
for i in range(0, 100):
time.sleep(0.05)
ic.setProgress(i)
```
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Lee moon soo <mo...@apache.org>
Closes #2454 from Leemoonsoo/manual_progress_update and squashes the following commits:
592f97a [Lee moon soo] Remove unnecessary print
f0c8132 [Lee moon soo] Add unittest for setProgress
777f085 [Lee moon soo] Add setProgress method to InterpreterContext
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/968bda80
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/968bda80
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/968bda80
Branch: refs/heads/master
Commit: 968bda80031abfd2b980ed8f2dd257530c0643be
Parents: e500373
Author: Lee moon soo <mo...@apache.org>
Authored: Fri Jun 30 09:20:43 2017 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Jul 2 20:03:26 2017 +0900
----------------------------------------------------------------------
.../interpreter/InterpreterContext.java | 25 ++++++--
.../remote/RemoteInterpreterServer.java | 18 +++++-
.../remote/RemoteInterpreterTest.java | 61 ++++++++++++++++++++
.../remote/mock/MockInterpreterA.java | 3 +
4 files changed, 100 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/968bda80/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
index d0db27b..4288ea3 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
@@ -62,7 +62,9 @@ public class InterpreterContext {
private String className;
private RemoteEventClientWrapper client;
private RemoteWorksController remoteWorksController;
+ private final Map<String, Integer> progressMap;
+ // visible for testing
public InterpreterContext(String noteId,
String paragraphId,
String replName,
@@ -77,7 +79,7 @@ public class InterpreterContext {
InterpreterOutput out
) {
this(noteId, paragraphId, replName, paragraphTitle, paragraphText, authenticationInfo,
- config, gui, angularObjectRegistry, resourcePool, runners, out, null);
+ config, gui, angularObjectRegistry, resourcePool, runners, out, null, null);
}
public InterpreterContext(String noteId,
@@ -92,7 +94,8 @@ public class InterpreterContext {
ResourcePool resourcePool,
List<InterpreterContextRunner> runners,
InterpreterOutput out,
- RemoteWorksController remoteWorksController
+ RemoteWorksController remoteWorksController,
+ Map<String, Integer> progressMap
) {
this.noteId = noteId;
this.paragraphId = paragraphId;
@@ -107,6 +110,7 @@ public class InterpreterContext {
this.runners = runners;
this.out = out;
this.remoteWorksController = remoteWorksController;
+ this.progressMap = progressMap;
}
public InterpreterContext(String noteId,
@@ -122,10 +126,11 @@ public class InterpreterContext {
List<InterpreterContextRunner> contextRunners,
InterpreterOutput output,
RemoteWorksController remoteWorksController,
- RemoteInterpreterEventClient eventClient) {
+ RemoteInterpreterEventClient eventClient,
+ Map<String, Integer> progressMap) {
this(noteId, paragraphId, replName, paragraphTitle, paragraphText, authenticationInfo,
config, gui, angularObjectRegistry, resourcePool, contextRunners, output,
- remoteWorksController);
+ remoteWorksController, progressMap);
this.client = new RemoteEventClient(eventClient);
}
@@ -196,4 +201,16 @@ public class InterpreterContext {
public InterpreterOutput out() {
return out;
}
+
+ /**
+ * Set progress of paragraph manually
+ * @param n integer from 0 to 100
+ */
+ public void setProgress(int n) {
+ if (progressMap != null) {
+ n = Math.max(n, 0);
+ n = Math.min(n, 100);
+ progressMap.put(paragraphId, new Integer(n));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/968bda80/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
index 719d2dd..4df3088 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
@@ -24,6 +24,8 @@ import java.lang.reflect.Method;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.thrift.TException;
import org.apache.thrift.server.TThreadPoolServer;
@@ -82,6 +84,9 @@ public class RemoteInterpreterServer
private final long DEFAULT_SHUTDOWN_TIMEOUT = 2000;
+ // Hold information for manual progress update
+ private ConcurrentMap<String, Integer> progressMap = new ConcurrentHashMap<>();
+
public RemoteInterpreterServer(int port) throws TTransportException {
this.port = port;
@@ -330,6 +335,8 @@ public class RemoteInterpreterServer
}
}
+ progressMap.remove(interpreterContext.getParagraphId());
+
InterpreterResult result;
if (job.getStatus() == Status.ERROR) {
result = new InterpreterResult(Code.ERROR, Job.getStack(job.getException()));
@@ -557,8 +564,13 @@ public class RemoteInterpreterServer
public int getProgress(String noteId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
- Interpreter intp = getInterpreter(noteId, className);
- return intp.getProgress(convert(interpreterContext, null));
+ Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
+ if (manuallyProvidedProgress != null) {
+ return manuallyProvidedProgress;
+ } else {
+ Interpreter intp = getInterpreter(noteId, className);
+ return intp.getProgress(convert(interpreterContext, null));
+ }
}
@@ -603,7 +615,7 @@ public class RemoteInterpreterServer
GUI.fromJson(ric.getGui()),
interpreterGroup.getAngularObjectRegistry(),
interpreterGroup.getResourcePool(),
- contextRunners, output, remoteWorksController, eventClient);
+ contextRunners, output, remoteWorksController, eventClient, progressMap);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/968bda80/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
index 2ba7a76..95235e5 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -911,4 +911,65 @@ public class RemoteInterpreterTest {
intp.close();
}
+ @Test
+ public void testSetProgress() throws InterruptedException {
+ // given MockInterpreterA set progress through InterpreterContext
+ Properties p = new Properties();
+ p.setProperty("progress", "50");
+ final RemoteInterpreter intpA = createMockInterpreterA(p);
+
+ intpGroup.put("note", new LinkedList<Interpreter>());
+ intpGroup.get("note").add(intpA);
+ intpA.setInterpreterGroup(intpGroup);
+
+ intpA.open();
+
+ final InterpreterContext context1 = new InterpreterContext(
+ "noteId",
+ "id1",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ new AngularObjectRegistry(intpGroup.getId(), null),
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null);
+
+ InterpreterContext context2 = new InterpreterContext(
+ "noteId",
+ "id2",
+ null,
+ "title",
+ "text",
+ new AuthenticationInfo(),
+ new HashMap<String, Object>(),
+ new GUI(),
+ new AngularObjectRegistry(intpGroup.getId(), null),
+ new LocalResourcePool("pool1"),
+ new LinkedList<InterpreterContextRunner>(), null);
+
+
+ assertEquals(0, intpA.getProgress(context1));
+ assertEquals(0, intpA.getProgress(context2));
+
+ // when interpreter update progress through InterpreterContext
+ Thread t = new Thread() {
+ public void run() {
+ InterpreterResult ret = intpA.interpret("1000", context1);
+ }
+ };
+ t.start();
+
+ // then progress need to be updated in given context
+ while(intpA.getProgress(context1) == 0) Thread.yield();
+ assertEquals(50, intpA.getProgress(context1));
+ assertEquals(0, intpA.getProgress(context2));
+
+ t.join();
+ assertEquals(0, intpA.getProgress(context1));
+ assertEquals(0, intpA.getProgress(context2));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/968bda80/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
index 81a9164..50d9888 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
@@ -53,6 +53,9 @@ public class MockInterpreterA extends Interpreter {
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
+ if (property.containsKey("progress")) {
+ context.setProgress(Integer.parseInt(getProperty("progress")));
+ }
try {
Thread.sleep(Long.parseLong(st));
this.lastSt = st;