You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by mm...@apache.org on 2018/03/20 01:46:06 UTC

[incubator-pulsar] branch master updated: Added trigger Function capability (#1402)

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 4cb4f11  Added trigger Function capability (#1402)
4cb4f11 is described below

commit 4cb4f114d552cee94ca3a9ea585e02bd1cdffd26
Author: Sanjeev Kulkarni <sa...@gmail.com>
AuthorDate: Mon Mar 19 18:46:03 2018 -0700

    Added trigger Function capability (#1402)
    
    * Added trigger functionality
    
    * Finished logic for trigger
    
    * Added admin side of changes
    
    * Added trigger impl in broker as well
    
    * Switched to comparing to message Id
---
 .../pulsar/broker/admin/impl/FunctionsBase.java    |  14 +++
 .../org/apache/pulsar/admin/cli/CmdFunctions.java  |  24 +++++
 .../org/apache/pulsar/client/admin/Functions.java  |  20 ++++
 .../client/admin/internal/FunctionsImpl.java       |  20 ++++
 .../instance/src/main/python/python_instance.py    |  37 ++++---
 .../functions/worker/rest/api/FunctionsImpl.java   | 113 ++++++++++++++++++++-
 .../worker/rest/api/v2/FunctionApiV2Resource.java  |  11 ++
 7 files changed, 215 insertions(+), 24 deletions(-)

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
index 7080474..e1318ba 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -139,4 +139,18 @@ public class FunctionsBase extends AdminResource implements Supplier<WorkerServi
         return functions.getAssignments();
     }
 
+    @POST
+    @Path("/{tenant}/{namespace}/{functionName}/trigger")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response triggerFunction(final @PathParam("tenant") String tenant,
+                                    final @PathParam("namespace") String namespace,
+                                    final @PathParam("functionName") String functionName,
+                                    final @FormDataParam("data") String triggerValue,
+                                    final @FormDataParam("dataStream") InputStream triggerStream) {
+
+        return functions.triggerFunction(
+                tenant, namespace, functionName, triggerValue, triggerStream);
+
+    }
+
 }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index b38aed2..5dcf070 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -77,6 +77,7 @@ public class CmdFunctions extends CmdBase {
     private final GetFunctionStatus statuser;
     private final ListFunctions lister;
     private final StateGetter stateGetter;
+    private final TriggerFunction triggerer;
 
     /**
      * Base command
@@ -617,6 +618,22 @@ public class CmdFunctions extends CmdBase {
         }
     }
 
+    @Parameters(commandDescription = "Trigger function")
+    class TriggerFunction extends FunctionCommand {
+        @Parameter(names = "--triggerValue", description = "The value the function needs to be triggered with")
+        protected String triggerValue;
+        @Parameter(names = "--triggerFile", description = "The fileName that contains data the function needs to be triggered with")
+        protected String triggerFile;
+        @Override
+        void runCmd() throws Exception {
+            if (triggerFile == null && triggerValue == null) {
+                throw new RuntimeException("One of triggerValue/triggerFile has to be present");
+            }
+            String retval = fnAdmin.functions().triggerFunction(tenant, namespace, functionName, triggerValue, triggerFile);
+            System.out.println(retval);
+        }
+    }
+
     public CmdFunctions(PulsarAdmin admin) {
         super("functions", admin);
         this.fnAdmin = (PulsarAdminWithFunctions) admin;
@@ -628,6 +645,7 @@ public class CmdFunctions extends CmdBase {
         statuser = new GetFunctionStatus();
         lister = new ListFunctions();
         stateGetter = new StateGetter();
+        triggerer = new TriggerFunction();
         jcommander.addCommand("localrun", getLocalRunner());
         jcommander.addCommand("create", getCreater());
         jcommander.addCommand("delete", getDeleter());
@@ -636,6 +654,7 @@ public class CmdFunctions extends CmdBase {
         jcommander.addCommand("getstatus", getStatuser());
         jcommander.addCommand("list", getLister());
         jcommander.addCommand("querystate", getStateGetter());
+        jcommander.addCommand("trigger", getTriggerer());
     }
 
     @VisibleForTesting
@@ -675,4 +694,9 @@ public class CmdFunctions extends CmdBase {
     StateGetter getStateGetter() {
         return stateGetter;
     }
+
+    @VisibleForTesting
+    TriggerFunction getTriggerer() {
+        return triggerer;
+    }
 }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java
index 390d88d..55ba089 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java
@@ -140,4 +140,24 @@ public interface Functions {
      */
     FunctionStatusList getFunctionStatus(String tenant, String namespace, String function) throws PulsarAdminException;
 
+    /**
+     * Triggers the function by writing to the input topic.
+     *
+     * @param tenant
+     *            Tenant name
+     * @param namespace
+     *            Namespace name
+     * @param function
+     *            Function name
+     * @param triggerValue
+     *            The input that will be written to input topic
+     * @param triggerFile
+     *            The file which contains the input that will be written to input topic
+     *
+     * @throws PulsarAdminException
+     *             Unexpected error
+     */
+    String triggerFunction(String tenant, String namespace, String function, String triggerValue, String triggerFile) throws PulsarAdminException;
+
+
 }
diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
index f184ea2..7394823 100644
--- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
+++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java
@@ -139,4 +139,24 @@ public class FunctionsImpl extends BaseResource implements Functions {
             throw getApiException(e);
         }
     }
+
+    @Override
+    public String triggerFunction(String tenant, String namespace, String functionName, String triggerValue, String triggerFile) throws PulsarAdminException {
+        try {
+            final FormDataMultiPart mp = new FormDataMultiPart();
+            if (triggerFile != null) {
+                mp.bodyPart(new FileDataBodyPart("dataStream",
+                        new File(triggerFile),
+                        MediaType.APPLICATION_OCTET_STREAM_TYPE));
+            }
+            if (triggerValue != null) {
+                mp.bodyPart(new FormDataBodyPart("data", triggerValue, MediaType.TEXT_PLAIN_TYPE));
+            }
+            String response = request(functions.path(tenant).path(namespace).path(functionName).path("trigger"))
+                    .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), String.class);
+            return response;
+        } catch (Exception e) {
+            throw getApiException(e);
+        }
+    }
 }
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index 88a1961..cc58526 100644
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -179,33 +179,32 @@ class PythonInstance(object):
       self.contextimpl.set_current_message_context(msg.message.message_id(), msg.topic)
       output_object = None
       self.saved_log_handler = None
+      if self.log_topic_handler is not None:
+        self.saved_log_handler = log.remove_all_handlers()
+        log.add_handler(self.log_topic_handler)
+      start_time = time.time()
+      self.current_stats.increment_processed(int(start_time) * 1000)
+      self.total_stats.increment_processed(int(start_time) * 1000)
+      successfully_executed = False
       try:
-        if self.log_topic_handler is not None:
-          self.saved_log_handler = log.remove_all_handlers()
-          log.add_handler(self.log_topic_handler)
-        start_time = time.time()
-        self.current_stats.increment_processed(int(start_time) * 1000)
-        self.total_stats.increment_processed(int(start_time) * 1000)
         if self.function_class is not None:
           output_object = self.function_class.process(input_object, self.contextimpl)
         else:
           output_object = self.function_purefunction.process(input_object)
-        end_time = time.time()
-        latency = (end_time - start_time) * 1000
-        self.total_stats.increment_successfully_processed(latency)
-        self.current_stats.increment_successfully_processed(latency)
-        self.process_result(output_object, msg)
+        successfully_executed = True
       except Exception as e:
-        if self.log_topic_handler is not None:
-          log.remove_all_handlers()
-          log.add_handler(self.saved_log_handler)
         Log.exception("Exception while executing user method")
         self.total_stats.record_user_exception(e)
         self.current_stats.record_user_exception(e)
-      finally:
-        if self.log_topic_handler is not None:
-          log.remove_all_handlers()
-          log.add_handler(self.saved_log_handler)
+      end_time = time.time()
+      latency = (end_time - start_time) * 1000
+      self.total_stats.increment_successfully_processed(latency)
+      self.current_stats.increment_successfully_processed(latency)
+      if self.log_topic_handler is not None:
+        log.remove_all_handlers()
+        log.add_handler(self.saved_log_handler)
+      if successfully_executed:
+        self.process_result(output_object, msg)
 
   def done_producing(self, consumer, orig_message, result, sent_message):
     if result == pulsar.Result.Ok and self.auto_ack and self.atleast_once:
@@ -224,7 +223,7 @@ class PythonInstance(object):
         self.current_stats.nserialization_exceptions += 1
         self.total_stats.nserialization_exceptions += 1
       if output_bytes is not None:
-        props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : str(base64.b64encode(msg.message.message_id().serialize()))}
+        props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64.b64encode(msg.message.message_id().serialize())}
         try:
           self.producer.send_async(output_bytes, partial(self.done_producing, msg.consumer, msg.message), properties=props)
         except Exception as e:
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index 7e49b5d..40d2c0b 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -23,13 +23,11 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import com.google.gson.Gson;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.lang.reflect.Array;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
@@ -43,6 +41,10 @@ import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.Reader;
 import org.apache.pulsar.common.policies.data.ErrorData;
 import org.apache.pulsar.functions.proto.Function;
 import org.apache.pulsar.functions.proto.Function.FunctionConfig;
@@ -442,6 +444,88 @@ public class FunctionsImpl {
                 new Gson().toJson(ret)).build();
     }
 
+    @POST
+    @Path("/{tenant}/{namespace}/{functionName}/trigger")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response triggerFunction(final @PathParam("tenant") String tenant,
+                                    final @PathParam("namespace") String namespace,
+                                    final @PathParam("name") String functionName,
+                                    final @FormDataParam("data") String input,
+                                    final @FormDataParam("dataStream") InputStream uploadedInputStream) {
+        FunctionConfig functionConfig;
+        // validate parameters
+        try {
+            validateTriggerRequestParams(tenant, namespace, functionName, input, uploadedInputStream);
+        } catch (IllegalArgumentException e) {
+            log.error("Invalid trigger function request @ /{}/{}/{}",
+                    tenant, namespace, functionName, e);
+            return Response.status(Status.BAD_REQUEST)
+                    .type(MediaType.APPLICATION_JSON)
+                    .entity(new ErrorData(e.getMessage())).build();
+        }
+
+        FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager();
+        if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) {
+            log.error("Function in getFunction does not exist @ /{}/{}/{}",
+                    tenant, namespace, functionName);
+            return Response.status(Status.NOT_FOUND)
+                    .type(MediaType.APPLICATION_JSON)
+                    .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build();
+        }
+
+        FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName);
+        String inputTopicToWrite;
+        if (functionMetaData.getFunctionConfig().getInputsList().size() > 0) {
+            inputTopicToWrite = functionMetaData.getFunctionConfig().getInputsList().get(0);
+        } else {
+            inputTopicToWrite = functionMetaData.getFunctionConfig().getCustomSerdeInputs().entrySet().iterator().next().getKey();
+        }
+        String outputTopic = functionMetaData.getFunctionConfig().getOutput();
+        Reader reader = null;
+        Producer producer = null;
+        try {
+            if (outputTopic != null && !outputTopic.isEmpty()) {
+                reader = worker().getClient().newReader().topic(outputTopic).startMessageId(MessageId.latest).create();
+            }
+            producer = worker().getClient().newProducer().topic(inputTopicToWrite).create();
+            byte[] targetArray;
+            if (uploadedInputStream != null) {
+                targetArray = new byte[uploadedInputStream.available()];
+                uploadedInputStream.read(targetArray);
+            } else {
+                targetArray = input.getBytes();
+            }
+            MessageId msgId = producer.send(targetArray);
+            if (reader == null) {
+                return Response.status(Status.OK).build();
+            }
+            long curTime = System.currentTimeMillis();
+            long maxTime = curTime + 1000;
+            while (curTime < maxTime) {
+                Message msg = reader.readNext(10000, TimeUnit.MILLISECONDS);
+                if (msg == null) break;
+                if (msg.getProperties().containsKey("__pfn_input_msg_id__") &&
+                        msg.getProperties().containsKey("__pfn_input_topic__")) {
+                    MessageId newMsgId = MessageId.fromByteArray(Base64.getDecoder().decode((String) msg.getProperties().get("__pfn_input_msg_id__")));
+                    if (msgId.equals(newMsgId) && msg.getProperties().get("__pfn_input_topic__").equals(inputTopicToWrite)) {
+                        return Response.status(Status.OK).entity(msg.getData()).build();
+                    }
+                }
+                curTime = System.currentTimeMillis();
+            }
+            return Response.status(Status.REQUEST_TIMEOUT).build();
+        } catch (Exception e) {
+            return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+        } finally {
+            if (reader != null) {
+                reader.closeAsync();
+            }
+            if (producer != null) {
+                producer.closeAsync();
+            }
+        }
+    }
+
     private void validateListFunctionRequestParams(String tenant, String namespace) throws IllegalArgumentException {
 
         if (tenant == null) {
@@ -550,4 +634,23 @@ public class FunctionsImpl {
         }
     }
 
+    private void validateTriggerRequestParams(String tenant,
+                                              String namespace,
+                                              String functionName,
+                                              String input,
+                                              InputStream uploadedInputStream) {
+        if (tenant == null) {
+            throw new IllegalArgumentException("Tenant is not provided");
+        }
+        if (namespace == null) {
+            throw new IllegalArgumentException("Namespace is not provided");
+        }
+        if (functionName == null) {
+            throw new IllegalArgumentException("Function Name is not provided");
+        }
+        if (uploadedInputStream == null && input == null) {
+            throw new IllegalArgumentException("Trigger Data is not provided");
+        }
+    }
+
 }
diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
index 7effcd8..1d7a8b6 100644
--- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
+++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java
@@ -128,4 +128,15 @@ public class FunctionApiV2Resource extends FunctionApiResource {
     public Response getAssignments() {
         return functions.getAssignments();
     }
+
+    @POST
+    @Path("/{tenant}/{namespace}/{functionName}/trigger")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response triggerFunction(final @PathParam("tenant") String tenant,
+                                    final @PathParam("namespace") String namespace,
+                                    final @PathParam("name") String functionName,
+                                    final @FormDataParam("data") String input,
+                                    final @FormDataParam("dataStream") InputStream uploadedInputStream) {
+        return functions.triggerFunction(tenant, namespace, functionName, input, uploadedInputStream);
+    }
 }

-- 
To stop receiving notification emails like this one, please contact
mmerli@apache.org.