You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2020/05/21 07:02:53 UTC
[flink-statefun] 05/08: [FLINK-17518] [e2e] Add remote functions
for remote module E2E
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit bae971d60ec918dba5d68dfdaec02c51fdc55aa4
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
AuthorDate: Wed May 20 19:30:02 2020 +0800
[FLINK-17518] [e2e] Add remote functions for remote module E2E
---
statefun-e2e-tests/pom.xml | 1 +
.../statefun-remote-module-e2e/pom.xml | 105 +++++++++++++++
.../main/protobuf/remote-module-verification.proto | 35 +++++
.../src/main/python/functions.py | 97 ++++++++++++++
.../main/python/remote_module_verification_pb2.py | 149 +++++++++++++++++++++
5 files changed, 387 insertions(+)
diff --git a/statefun-e2e-tests/pom.xml b/statefun-e2e-tests/pom.xml
index 5fe37c1..9456ed1 100644
--- a/statefun-e2e-tests/pom.xml
+++ b/statefun-e2e-tests/pom.xml
@@ -32,6 +32,7 @@ under the License.
<module>statefun-e2e-tests-common</module>
<module>statefun-sanity-e2e</module>
<module>statefun-routable-kafka-e2e</module>
+ <module>statefun-remote-module-e2e</module>
<module>statefun-exactly-once-e2e</module>
</modules>
diff --git a/statefun-e2e-tests/statefun-remote-module-e2e/pom.xml b/statefun-e2e-tests/statefun-remote-module-e2e/pom.xml
new file mode 100644
index 0000000..e904370
--- /dev/null
+++ b/statefun-e2e-tests/statefun-remote-module-e2e/pom.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>statefun-e2e-tests</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <version>2.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>statefun-remote-module-e2e</artifactId>
+
+ <properties>
+ <testcontainers.version>1.12.5</testcontainers.version>
+ <kafka.version>2.2.0</kafka.version>
+ </properties>
+
+ <dependencies>
+ <!-- Kafka clients -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ <version>${kafka.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Protobuf -->
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+
+ <!-- logging -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.7.15</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- End-to-end test common -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>statefun-e2e-tests-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Testcontainers KafkaContainer -->
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>kafka</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.os72</groupId>
+ <artifactId>protoc-jar-maven-plugin</artifactId>
+ <version>${protoc-jar-maven-plugin.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <excludePackageNames>org.apache.flink.statefun.examples.greeter.generated</excludePackageNames>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git a/statefun-e2e-tests/statefun-remote-module-e2e/src/main/protobuf/remote-module-verification.proto b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/protobuf/remote-module-verification.proto
new file mode 100644
index 0000000..1cafcf1
--- /dev/null
+++ b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/protobuf/remote-module-verification.proto
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+package org.apache.flink.statefun.e2e.remote;
+option java_package = "org.apache.flink.statefun.e2e.remote.generated";
+option java_multiple_files = false;
+
+message Invoke {
+}
+
+message InvokeCount {
+ int32 count = 1;
+}
+
+message InvokeResult {
+ string id = 1;
+ int32 invoke_count = 2;
+}
diff --git a/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/functions.py b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/functions.py
new file mode 100644
index 0000000..be2abe4
--- /dev/null
+++ b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/functions.py
@@ -0,0 +1,97 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from remote_module_verification_pb2 import Invoke, InvokeResult, InvokeCount
+
+from statefun import StatefulFunctions
+from statefun import RequestReplyHandler
+from statefun import kafka_egress_record
+
+import uuid
+
+functions = StatefulFunctions()
+
+
+@functions.bind("org.apache.flink.statefun.e2e.remote/counter")
+def counter(context, invoke: Invoke):
+ """
+ Keeps count of the number of invocations, and forwards that count
+ to be sent to the Kafka egress. We do the extra forwarding instead
+ of directly sending to Kafka, so that we cover inter-function
+ messaging in our E2E test.
+ """
+ invoke_count = context.state('invoke_count').unpack(InvokeCount)
+ if not invoke_count:
+ invoke_count = InvokeCount()
+ invoke_count.count = 1
+ else:
+ invoke_count.count += 1
+ context.state('invoke_count').pack(invoke_count)
+
+ response = InvokeResult()
+ response.id = context.address.identity
+ response.invoke_count = invoke_count.count
+
+ context.pack_and_send(
+ "org.apache.flink.statefun.e2e.remote/forward-function",
+ # use random keys to simulate both local handovers and
+ # cross-partition messaging via the feedback loop
+ uuid.uuid4().hex,
+ response
+ )
+
+
+@functions.bind("org.apache.flink.statefun.e2e.remote/forward-function")
+def forward_to_egress(context, invoke_result: InvokeResult):
+ """
+ Simply forwards the results to the Kafka egress.
+ """
+ egress_message = kafka_egress_record(
+ topic="invoke-results",
+ key=invoke_result.id,
+ value=invoke_result
+ )
+ context.pack_and_send_egress(
+ "org.apache.flink.statefun.e2e.remote/invoke-results",
+ egress_message
+ )
+
+
+handler = RequestReplyHandler(functions)
+
+#
+# Serve the endpoint
+#
+
+from flask import request
+from flask import make_response
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.route('/service', methods=['POST'])
+def handle():
+ response_data = handler(request.data)
+ response = make_response(response_data)
+ response.headers.set('Content-Type', 'application/octet-stream')
+ return response
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/remote_module_verification_pb2.py b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/remote_module_verification_pb2.py
new file mode 100644
index 0000000..8e0a556
--- /dev/null
+++ b/statefun-e2e-tests/statefun-remote-module-e2e/src/main/python/remote_module_verification_pb2.py
@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: remote-module-verification.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+ name='remote-module-verification.proto',
+ package='org.apache.flink.statefun.e2e.remote',
+ syntax='proto3',
+ serialized_options=_b('\n.org.apache.flink.statefun.e2e.remote.generatedP\000'),
+ serialized_pb=_b('\n remote-module-verification.proto\x12$org.apache.flink.statefun.e2e.remote\"\x08\n\x06Invoke\"\x1c\n\x0bInvokeCount\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\"0\n\x0cInvokeResult\x12\n\n\x02id\x18\x01 \x01(\t\x12\x14\n\x0cinvoke_count\x18\x02 \x01(\x05\x42\x32\n.org.apache.flink.statefun.e2e.remote.generatedP\x00\x62\x06proto3')
+)
+
+
+
+
+_INVOKE = _descriptor.Descriptor(
+ name='Invoke',
+ full_name='org.apache.flink.statefun.e2e.remote.Invoke',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=74,
+ serialized_end=82,
+)
+
+
+_INVOKECOUNT = _descriptor.Descriptor(
+ name='InvokeCount',
+ full_name='org.apache.flink.statefun.e2e.remote.InvokeCount',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='count', full_name='org.apache.flink.statefun.e2e.remote.InvokeCount.count', index=0,
+ number=1, type=5, cpp_type=1, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=84,
+ serialized_end=112,
+)
+
+
+_INVOKERESULT = _descriptor.Descriptor(
+ name='InvokeResult',
+ full_name='org.apache.flink.statefun.e2e.remote.InvokeResult',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='id', full_name='org.apache.flink.statefun.e2e.remote.InvokeResult.id', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=_b("").decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ _descriptor.FieldDescriptor(
+ name='invoke_count', full_name='org.apache.flink.statefun.e2e.remote.InvokeResult.invoke_count', index=1,
+ number=2, type=5, cpp_type=1, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=114,
+ serialized_end=162,
+)
+
+DESCRIPTOR.message_types_by_name['Invoke'] = _INVOKE
+DESCRIPTOR.message_types_by_name['InvokeCount'] = _INVOKECOUNT
+DESCRIPTOR.message_types_by_name['InvokeResult'] = _INVOKERESULT
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Invoke = _reflection.GeneratedProtocolMessageType('Invoke', (_message.Message,), dict(
+ DESCRIPTOR = _INVOKE,
+ __module__ = 'remote_module_verification_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.e2e.remote.Invoke)
+ ))
+_sym_db.RegisterMessage(Invoke)
+
+InvokeCount = _reflection.GeneratedProtocolMessageType('InvokeCount', (_message.Message,), dict(
+ DESCRIPTOR = _INVOKECOUNT,
+ __module__ = 'remote_module_verification_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.e2e.remote.InvokeCount)
+ ))
+_sym_db.RegisterMessage(InvokeCount)
+
+InvokeResult = _reflection.GeneratedProtocolMessageType('InvokeResult', (_message.Message,), dict(
+ DESCRIPTOR = _INVOKERESULT,
+ __module__ = 'remote_module_verification_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.e2e.remote.InvokeResult)
+ ))
+_sym_db.RegisterMessage(InvokeResult)
+
+
+DESCRIPTOR._options = None
+# @@protoc_insertion_point(module_scope)