You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/03/21 17:07:35 UTC

Change in asterixdb[cheshire-cat]: [NO ISSUE] Make UDF upload timeout configurable

From Ian Maxon <im...@uci.edu>:

Ian Maxon has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10623 )


Change subject: [NO ISSUE] Make UDF upload timeout configurable
......................................................................

[NO ISSUE] Make UDF upload timeout configurable

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Add a config variable for UDF Upload timeout.
  For very large libraries, this is necessary as
  it can take some time for them to be distributed
  and extracted on every node in the cluster

Change-Id: Ic318a3b70d5d44c1dbd26f543eb6ef5fe62002ac
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
2 files changed, 7 insertions(+), 1 deletion(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/23/10623/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index f164938..6b84faa 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -58,6 +58,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
@@ -83,6 +84,7 @@
     protected Path workingDir;
     protected String sysAuthHeader;
     private ILibraryManager libraryManager;
+    private int timeout;
 
     private static final Logger LOGGER = LogManager.getLogger();
 
@@ -91,6 +93,7 @@
         super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort);
         this.compilationProvider = compilationProvider;
         this.receptionist = appCtx.getReceptionist();
+        this.timeout = this.appCtx.getServiceContext().getAppConfig().getInt(NCConfig.Option.UDF_UPLOAD_TIMEOUT);
     }
 
     @Override
@@ -152,7 +155,7 @@
         InternalRequestResponse responseMsg;
         try {
             ncMb.sendMessageToPrimaryCC(requestMessage);
-            responseMsg = (InternalRequestResponse) responseFuture.get(120000, TimeUnit.MILLISECONDS);
+            responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
         } finally {
             ncMb.deregisterMessageFuture(responseFuture.getFutureId());
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 22b240a..d75af2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -98,6 +98,7 @@
         PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, new String[0]),
         PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
         PYTHON_ARGS(STRING_ARRAY, (String[]) null),
+        UDF_UPLOAD_TIMEOUT(POSITIVE_INTEGER, 600),
         CREDENTIAL_FILE(
                 OptionTypes.STRING,
                 (Function<IApplicationConfig, String>) appConfig -> FileUtil
@@ -245,6 +246,8 @@
                     return "Python args to pass to Python interpreter";
                 case PYTHON_CMD_AUTOLOCATE:
                     return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter";
+                case UDF_UPLOAD_TIMEOUT:
+                    return "Timeout to upload a UDF in seconds";
                 case CREDENTIAL_FILE:
                     return "Path to HTTP basic credentials";
                 default:

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10623
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Ic318a3b70d5d44c1dbd26f543eb6ef5fe62002ac
Gerrit-Change-Number: 10623
Gerrit-PatchSet: 1
Gerrit-Owner: Ian Maxon <im...@uci.edu>
Gerrit-MessageType: newchange

Change in asterixdb[cheshire-cat]: [NO ISSUE] Make UDF upload timeout configurable

Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Ian Maxon <im...@uci.edu>:

Ian Maxon has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10623 )


Change subject: [NO ISSUE] Make UDF upload timeout configurable
......................................................................

[NO ISSUE] Make UDF upload timeout configurable

- user model changes: no
- storage format changes: no
- interface changes: no

Details:

- Add a config variable for UDF Upload timeout.
  For very large libraries, this is necessary as
  it can take some time for them to be distributed
  and extracted on every node in the cluster

Change-Id: Ic318a3b70d5d44c1dbd26f543eb6ef5fe62002ac
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
2 files changed, 7 insertions(+), 1 deletion(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/23/10623/1

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index f164938..6b84faa 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -58,6 +58,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
@@ -83,6 +84,7 @@
     protected Path workingDir;
     protected String sysAuthHeader;
     private ILibraryManager libraryManager;
+    private int timeout;
 
     private static final Logger LOGGER = LogManager.getLogger();
 
@@ -91,6 +93,7 @@
         super(ctx, paths, appCtx, compilationProvider, httpServerProtocol, httpServerPort);
         this.compilationProvider = compilationProvider;
         this.receptionist = appCtx.getReceptionist();
+        this.timeout = this.appCtx.getServiceContext().getAppConfig().getInt(NCConfig.Option.UDF_UPLOAD_TIMEOUT);
     }
 
     @Override
@@ -152,7 +155,7 @@
         InternalRequestResponse responseMsg;
         try {
             ncMb.sendMessageToPrimaryCC(requestMessage);
-            responseMsg = (InternalRequestResponse) responseFuture.get(120000, TimeUnit.MILLISECONDS);
+            responseMsg = (InternalRequestResponse) responseFuture.get(timeout, TimeUnit.SECONDS);
         } finally {
             ncMb.deregisterMessageFuture(responseFuture.getFutureId());
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index 22b240a..d75af2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -98,6 +98,7 @@
         PYTHON_ADDITIONAL_PACKAGES(STRING_ARRAY, new String[0]),
         PYTHON_USE_BUNDLED_MSGPACK(BOOLEAN, true),
         PYTHON_ARGS(STRING_ARRAY, (String[]) null),
+        UDF_UPLOAD_TIMEOUT(POSITIVE_INTEGER, 600),
         CREDENTIAL_FILE(
                 OptionTypes.STRING,
                 (Function<IApplicationConfig, String>) appConfig -> FileUtil
@@ -245,6 +246,8 @@
                     return "Python args to pass to Python interpreter";
                 case PYTHON_CMD_AUTOLOCATE:
                     return "Whether or not to attempt to automatically set PYTHON_CMD to a usable interpreter";
+                case UDF_UPLOAD_TIMEOUT:
+                    return "Timeout to upload a UDF in seconds";
                 case CREDENTIAL_FILE:
                     return "Path to HTTP basic credentials";
                 default:

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10623
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Ic318a3b70d5d44c1dbd26f543eb6ef5fe62002ac
Gerrit-Change-Number: 10623
Gerrit-PatchSet: 1
Gerrit-Owner: Ian Maxon <im...@uci.edu>
Gerrit-MessageType: newchange