[zeppelin] branch master updated: [ZEPPELIN-5494] Make shell interpreter concurrency configurable

[zj...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new c8b1210  [ZEPPELIN-5494] Make shell interpreter concurrency configurable
c8b1210 is described below

commit c8b1210d55aa3fe5247298e1daaa35fe6b45bbe8
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Thu Aug 26 10:16:30 2021 +0800

    [ZEPPELIN-5494] Make shell interpreter concurrency configurable
    
    ### What is this PR for?
    
    Introduce property `zeppelin.concurrency.max` to make shell concurrency configurable, by default it is 10.
    
    ### What type of PR is it?
    [Improvement ]
    
    ### Todos
    * [ ] - Task
    
    ### What is the Jira issue?
    * https://issues.apache.org/jira/browse/ZEPPELIN-5494
    
    ### How should this be tested?
    * CI pass
    
    ### Screenshots (if appropriate)
    
    ### Questions:
    * Does the licenses files need update? No
    * Is there breaking changes for older versions? No
    * Does this needs documentation? No
    
    Author: Jeff Zhang <zj...@apache.org>
    
    Closes #4207 from zjffdu/ZEPPELIN-5494 and squashes the following commits:
    
    988a4839c5 [Jeff Zhang] address comment
    a2cace6c85 [Jeff Zhang] [ZEPPELIN-5494] Make shell interpreter concurrency configurable
---
 docs/interpreter/shell.md                                  |  5 +++++
 .../java/org/apache/zeppelin/shell/ShellInterpreter.java   | 14 +++++++++++++-
 shell/src/main/resources/interpreter-setting.json          |  7 +++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/docs/interpreter/shell.md b/docs/interpreter/shell.md
index 631c133..865b915 100644
--- a/docs/interpreter/shell.md
+++ b/docs/interpreter/shell.md
@@ -79,6 +79,11 @@ At the "Interpreters" menu in Zeppelin dropdown menu, you can set the property v
     <td></td>
     <td>Internal and external IP mapping of zeppelin server</td>
   </tr>
+  <tr>
+    <td>zeppelin.concurrency.max</td>
+    <td>10</td>
+    <td>Max concurrency of shell interpreter</td>
+  </tr>
 </table>
 
 ## Example
diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
index c43a8be..b865a6d 100644
--- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
+++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
@@ -57,6 +57,7 @@ public class ShellInterpreter extends KerberosInterpreter {
   private static final String DEFAULT_TIMEOUT = "60000";
   private static final String DEFAULT_CHECK_INTERVAL = "10000";
   private static final String DIRECTORY_USER_HOME = "shell.working.directory.user.home";
+  private static final String MAX_CONCURRENCY = "shell.concurrency.max";
 
   private final boolean isWindows = System.getProperty("os.name").startsWith("Windows");
   private final String shell = isWindows ? "cmd /c" : "bash -c";
@@ -213,7 +214,18 @@ public class ShellInterpreter extends KerberosInterpreter {
   @Override
   public Scheduler getScheduler() {
     return SchedulerFactory.singleton().createOrGetParallelScheduler(
-        ShellInterpreter.class.getName() + this.hashCode(), 10);
+        ShellInterpreter.class.getName() + this.hashCode(), getMaxConcurrent());
+  }
+
+  private int getMaxConcurrent() {
+    String maxConcurrencyProperty = getProperty(MAX_CONCURRENCY, "10");
+    try {
+      return Integer.valueOf(maxConcurrencyProperty);
+    } catch (NumberFormatException e) {
+      LOGGER.error("Fail to parse {} with value: {}", MAX_CONCURRENCY,
+              maxConcurrencyProperty);
+      return 10;
+    }
   }
 
   @Override
diff --git a/shell/src/main/resources/interpreter-setting.json b/shell/src/main/resources/interpreter-setting.json
index 7bbed28..e1cba3b 100644
--- a/shell/src/main/resources/interpreter-setting.json
+++ b/shell/src/main/resources/interpreter-setting.json
@@ -53,6 +53,13 @@
         "defaultValue": false,
         "description": "Enable ZeppelinContext variable interpolation into paragraph text",
         "type": "checkbox"
+      },
+      "zeppelin.concurrency.max": {
+        "envName": null,
+        "propertyName": "zeppelin.concurrency.max",
+        "defaultValue": "10",
+        "description": "Max concurrency of shell interpreter",
+        "type": "number"
       }
     },
     "editor": {