You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2020/03/23 08:35:22 UTC
[flink-statefun] 02/02: [FLINK-16685] [py] Add a k8s example to
statefun-python-sdk
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit b9752c895487c188b5e687b6445d06b388d02239
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Mon Mar 9 22:33:03 2020 +0100
[FLINK-16685] [py] Add a k8s example to statefun-python-sdk
This closes #62.
---
.../examples/k8s/Dockerfile.python-worker | 34 ++++++
.../examples/k8s/Dockerfile.statefun | 22 ++++
statefun-python-sdk/examples/k8s/README.md | 92 +++++++++++++++
statefun-python-sdk/examples/k8s/build-example.sh | 54 +++++++++
.../examples/k8s/event-generator.py | 74 ++++++++++++
statefun-python-sdk/examples/k8s/main.py | 65 +++++++++++
statefun-python-sdk/examples/k8s/messages.proto | 32 ++++++
statefun-python-sdk/examples/k8s/messages_pb2.py | 124 +++++++++++++++++++++
statefun-python-sdk/examples/k8s/module.yaml | 54 +++++++++
statefun-python-sdk/examples/k8s/requirements.txt | 22 ++++
.../examples/k8s/resources/Chart.yaml | 21 ++++
.../k8s/resources/templates/config-map.yaml | 47 ++++++++
.../k8s/resources/templates/master-deployment.yaml | 63 +++++++++++
.../resources/templates/master-rest-service.yaml | 28 +++++
.../k8s/resources/templates/master-service.yaml | 31 ++++++
.../templates/python-worker-deployment.yaml | 42 +++++++
.../resources/templates/python-worker-service.yaml | 27 +++++
.../k8s/resources/templates/worker-deployment.yaml | 66 +++++++++++
.../examples/k8s/resources/values.yaml | 35 ++++++
19 files changed, 933 insertions(+)
diff --git a/statefun-python-sdk/examples/k8s/Dockerfile.python-worker b/statefun-python-sdk/examples/k8s/Dockerfile.python-worker
new file mode 100644
index 0000000..06a7b65
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/Dockerfile.python-worker
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.7-alpine
+
+RUN mkdir -p /app
+WORKDIR /app
+
+COPY apache_flink_statefun-snapshot-py3-none-any.whl /app
+RUN pip install apache_flink_statefun-snapshot-py3-none-any.whl
+
+COPY requirements.txt /app
+RUN pip install -r requirements.txt
+
+COPY main.py /app
+COPY messages_pb2.py /app
+
+EXPOSE 8000
+
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w 4", "main:app"]
+
diff --git a/statefun-python-sdk/examples/k8s/Dockerfile.statefun b/statefun-python-sdk/examples/k8s/Dockerfile.statefun
new file mode 100644
index 0000000..9f3a803
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/Dockerfile.statefun
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM statefun
+
+RUN mkdir -p /opt/statefun/modules/greeter
+ADD module.yaml /opt/statefun/modules/greeter
+
+
diff --git a/statefun-python-sdk/examples/k8s/README.md b/statefun-python-sdk/examples/k8s/README.md
new file mode 100644
index 0000000..bb78b7d
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/README.md
@@ -0,0 +1,92 @@
+# Kubernetes example
+
+This example demonstrates how to deploy a stateful function application
+written in Python to Kubernetes.
+
+## Prerequisites
+
+* Helm
+* Kubernetes cluster
+* Kafka
+* [StateFun distribution](https://github.com/apache/flink-statefun#build)
+* [StateFun Python SDK](https://github.com/apache/flink-statefun/blob/master/statefun-python-sdk/build-distribution.sh)
+
+
+## Overview
+
+This examples create a stateful function application,
+that consumes `LoginEvent`s from a `logins` Kafka topic,
+and produces `seen` count per user, into the `seen` Kafka topic.
+
+The main example components contains:
+- [main.py](main.py) - A StateFun python function that implements the main logic
+- [module.yaml](module.yaml) - defines the ingress, egress and the remote function specification.
+- [resources](resources) - a Helm chart, templates to deploy StateFun cluster and the remote python worker to k8s.
+- [build-example.sh](build-example.sh) - Builds StateFun Docker images and k8s resources to deploy it.
+
+## Setup
+
+### Create Kafka Topics:
+
+This example consumes `LoginEvent`s from the `logins` topic, and produces `SeenCount` to
+the `seen` topic
+```
+ ./kafka-topics.sh --create --topic logins --zookeeper <zookeeper address>:2181 --partitions 1 --replication-factor 1
+ ./kafka-topics.sh --create --topic seen --zookeeper <zookeeper address>:2181 --partitions 1 --replication-factor 1
+```
+
+### update [module.yaml](module.yaml)
+
+Make sure that your `module.yaml` ingress/and egress sections point to your
+Kafka cluster.
+
+```
+ingresses:
+ - ingress:
+ ...
+ spec:
+ address: kafka-service:9092
+ ...
+ egresses:
+ - egress:
+ ...
+ spec:
+ address: kafka-service:9092
+```
+
+### Build the Docker images and the k8s resource yamls.
+
+This examples creates two different Docker images, one for the `Python` remote
+worker (`k8s-demo-python-worker`) and one for the statefun cluster (`k8s-demo-statefun`).
+
+- If you have a remote docker registry (i.e. `gcr.io/<project-name>`) make sure
+to update [resources/values.yaml](resources/values.yaml) relevant `image: ` sections.
+
+- Modify [resources/values.yaml](resources/values.yaml) and set the value of `checkpoint.directory`
+to a filesystem / object store. For example
+```
+checkpoint:
+ dir: gcs://my-project/my-bucket
+```
+
+
+Assuming the all prerequisites where completed run:
+
+```build-example.sh```
+
+This should create the Docker images and generate a `k8s-demo.yaml` file.
+
+## Deploy
+
+`kubectl create -f k8s-demo.yaml`
+
+## Generate events
+
+Run:
+
+```
+pip3 install kafka-python
+python3 event_generator.py --address <kafka address> --events 1000
+```
+
+This would generate 1,000 login events into the `logins` topic
\ No newline at end of file
diff --git a/statefun-python-sdk/examples/k8s/build-example.sh b/statefun-python-sdk/examples/k8s/build-example.sh
new file mode 100755
index 0000000..07e5fc0
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/build-example.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PYTHON_IMAGE_NAME="k8s-demo-python-worker"
+PYTHON_SERVICE_NAME="python-worker"
+STATEFUN_IMAGE_NAME="k8s-demo-statefun"
+PARALLELISM=3
+K8S_RESOURCES_YAML="k8s-demo.yaml"
+
+# clean
+rm -f apache_flink_statefun-*-py3-none-any.whl
+rm -rf __pycache__
+
+# copy the whl distribution, it must be first built by calling build-distribution.sh
+cp ../../dist/apache_flink_statefun-*-py3-none-any.whl apache_flink_statefun-snapshot-py3-none-any.whl 2>/dev/null
+rc=$?
+if [[ ${rc} -ne 0 ]]; then
+ echo "Failed copying the whl distribution, please build the distribution first by calling ./build-distribution.sh"
+ exit 1;
+fi
+
+# build the flask container
+docker build -f Dockerfile.python-worker . -t ${PYTHON_IMAGE_NAME}
+
+rm -f apache_flink_statefun-*-py3-none-any.whl
+
+# build the statefun Flink image
+docker build -f Dockerfile.statefun . -t ${STATEFUN_IMAGE_NAME}
+
+helm template resources \
+ --set worker.replicas=${PARALLELISM} \
+ --set worker.image=${STATEFUN_IMAGE_NAME} \
+ --set python.image=${PYTHON_IMAGE_NAME} \
+ --set python.name=${PYTHON_SERVICE_NAME} > ${K8S_RESOURCES_YAML}
+
+
+echo "Successfully created ${STATEFUN_IMAGE_NAME}, ${PYTHON_IMAGE_NAME} Docker images and ${K8S_RESOURCES_YAML}"
+echo "Upload these Docker images to your docker registry that is accssible from K8S, and"
+echo "Use: kubectl create -f ${K8S_RESOURCES_YAML}"
+
diff --git a/statefun-python-sdk/examples/k8s/event-generator.py b/statefun-python-sdk/examples/k8s/event-generator.py
new file mode 100644
index 0000000..1df04d0
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/event-generator.py
@@ -0,0 +1,74 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import sys
+import getopt
+import random
+import string
+
+from messages_pb2 import LoginEvent
+from messages_pb2 import SeenCount
+
+from kafka import KafkaProducer
+
+
+def random_user(n):
+ """generate a random user id of size n"""
+ chars = []
+ for i in range(n):
+ chars.append(random.choice(string.ascii_lowercase))
+ return ''.join(chars)
+
+
+def produce(events, address):
+ producer = KafkaProducer(bootstrap_servers=address)
+
+ for _ in range(events):
+ event = LoginEvent()
+ event.user_name = random_user(4)
+ producer.send('logins', event.SerializeToString())
+ producer.flush()
+ producer.close()
+
+
+def usage():
+ print('usage: python3 event-generator.py --address=localhost:9092 --events=1')
+ sys.exit(1)
+
+
+def parse_args():
+ address = None
+ events = None
+ opts, args = getopt.getopt(sys.argv[1:], "a:e", ["address=", "events="])
+ for opt, arg in opts:
+ if opt in ("-a", "--address"):
+ address = arg
+ elif opt in ("-e", "--events"):
+ events = arg
+ if address is None or events is None:
+ usage()
+ return address, int(events)
+
+
+def main():
+ address, events = parse_args()
+ produce(address=address, events=events)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/statefun-python-sdk/examples/k8s/main.py b/statefun-python-sdk/examples/k8s/main.py
new file mode 100644
index 0000000..6867dd8
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/main.py
@@ -0,0 +1,65 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+from messages_pb2 import LoginEvent
+from messages_pb2 import SeenCount
+
+from statefun import StatefulFunctions
+
+from statefun import RequestReplyHandler
+from statefun import kafka_egress_builder
+
+functions = StatefulFunctions()
+
+
+@functions.bind("k8s-demo/greeter")
+def greet(context, message: LoginEvent):
+ state = context.state('seen_count').unpack(SeenCount)
+ if not state:
+ state = SeenCount()
+ state.seen = 1
+ else:
+ state.seen += 1
+ context.state('seen_count').pack(state)
+
+ egress_message = kafka_egress_builder(topic="seen", key=message.user_name, value=state)
+ context.pack_and_send_egress("k8s-demo/greets-egress", egress_message)
+
+
+handler = RequestReplyHandler(functions)
+
+#
+# Serve the endpoint
+#
+
+from flask import request
+from flask import make_response
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.route('/statefun', methods=['POST'])
+def handle():
+ response_data = handler(request.data)
+ response = make_response(response_data)
+ response.headers.set('Content-Type', 'application/octet-stream')
+ return response
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/statefun-python-sdk/examples/k8s/messages.proto b/statefun-python-sdk/examples/k8s/messages.proto
new file mode 100644
index 0000000..92ac22f
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/messages.proto
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto3";
+
+// protoc flask.proto --python_out=.
+
+package k8s.demo;
+
+message LoginEvent {
+ string user_name = 1;
+}
+
+message SeenCount {
+ int64 seen = 1;
+}
+
+
diff --git a/statefun-python-sdk/examples/k8s/messages_pb2.py b/statefun-python-sdk/examples/k8s/messages_pb2.py
new file mode 100644
index 0000000..3d2d33e
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/messages_pb2.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: messages.proto
+
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+ name='messages.proto',
+ package='k8s.demo',
+ syntax='proto3',
+ serialized_options=None,
+ serialized_pb=b'\n\x0emessages.proto\x12\x08k8s.demo\"\x1f\n\nLoginEvent\x12\x11\n\tuser_name\x18\x01 \x01(\t\"\x19\n\tSeenCount\x12\x0c\n\x04seen\x18\x01 \x01(\x03\x62\x06proto3'
+)
+
+
+
+
+_LOGINEVENT = _descriptor.Descriptor(
+ name='LoginEvent',
+ full_name='k8s.demo.LoginEvent',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='user_name', full_name='k8s.demo.LoginEvent.user_name', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=28,
+ serialized_end=59,
+)
+
+
+_SEENCOUNT = _descriptor.Descriptor(
+ name='SeenCount',
+ full_name='k8s.demo.SeenCount',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='seen', full_name='k8s.demo.SeenCount.seen', index=0,
+ number=1, type=3, cpp_type=2, label=1,
+ has_default_value=False, default_value=0,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=61,
+ serialized_end=86,
+)
+
+DESCRIPTOR.message_types_by_name['LoginEvent'] = _LOGINEVENT
+DESCRIPTOR.message_types_by_name['SeenCount'] = _SEENCOUNT
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+LoginEvent = _reflection.GeneratedProtocolMessageType('LoginEvent', (_message.Message,), {
+ 'DESCRIPTOR' : _LOGINEVENT,
+ '__module__' : 'messages_pb2'
+ # @@protoc_insertion_point(class_scope:k8s.demo.LoginEvent)
+ })
+_sym_db.RegisterMessage(LoginEvent)
+
+SeenCount = _reflection.GeneratedProtocolMessageType('SeenCount', (_message.Message,), {
+ 'DESCRIPTOR' : _SEENCOUNT,
+ '__module__' : 'messages_pb2'
+ # @@protoc_insertion_point(class_scope:k8s.demo.SeenCount)
+ })
+_sym_db.RegisterMessage(SeenCount)
+
+
+# @@protoc_insertion_point(module_scope)
diff --git a/statefun-python-sdk/examples/k8s/module.yaml b/statefun-python-sdk/examples/k8s/module.yaml
new file mode 100644
index 0000000..03e6550
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/module.yaml
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+version: "1.0"
+module:
+ meta:
+ type: remote
+ spec:
+ functions:
+ - function:
+ meta:
+ kind: http
+ type: k8s-demo/greeter
+ spec:
+ endpoint: http://python-worker:8000/statefun
+ states:
+ - seen_count
+ maxNumBatchRequests: 500
+ timeout: 2min
+ ingresses:
+ - ingress:
+ meta:
+ type: statefun.kafka.io/routable-protobuf-ingress
+ id: k8s-demo/names-ingress
+ spec:
+ address: kafka-service:9092
+ consumerGroupId: my-group-id
+ topics:
+ - topic: logins
+ typeUrl: com.googleapis/k8s.demo.LoginEvent
+ targets:
+ - k8s-demo/greeter
+ egresses:
+ - egress:
+ meta:
+ type: statefun.kafka.io/generic-egress
+ id: k8s-demo/greets-egress
+ spec:
+ address: kafka-service:9092
+ deliverySemantic:
+ type: exactly-once
+ transactionTimeoutMillis: 100000
+
diff --git a/statefun-python-sdk/examples/k8s/requirements.txt b/statefun-python-sdk/examples/k8s/requirements.txt
new file mode 100644
index 0000000..c65eb74
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/requirements.txt
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apache-flink-statefun
+flask
+gunicorn==20.0.4
+
+
+
diff --git a/statefun-python-sdk/examples/k8s/resources/Chart.yaml b/statefun-python-sdk/examples/k8s/resources/Chart.yaml
new file mode 100644
index 0000000..f9a582f
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/Chart.yaml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v2
+name: statefun-demo
+description: A Helm chart for a multi-lang StateFun application deployed on Kubernetes
+type: application
+version: 1.0.0
+appVersion: 1.16.0
\ No newline at end of file
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/config-map.yaml b/statefun-python-sdk/examples/k8s/resources/templates/config-map.yaml
new file mode 100644
index 0000000..a57e7ff
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/config-map.yaml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flink-config
+ labels:
+ app: statefun
+data:
+ flink-conf.yaml: |+
+ jobmanager.rpc.address: {{ .Values.master.name }}
+ taskmanager.numberOfTaskSlots: 1
+ blob.server.port: 6124
+ jobmanager.rpc.port: 6123
+ taskmanager.rpc.port: 6122
+ classloader.parent-first-patterns.additional: org.apache.flink.statefun;org.apache.kafka;com.google.protobuf
+ state.checkpoints.dir: {{ .Values.checkpoint.dir }}
+ state.backend: rocksdb
+ state.backend.rocksdb.timer-service.factory: ROCKSDB
+ state.backend.incremental: true
+ execution.checkpointing.interval: {{ .Values.checkpoint.interval }}
+ taskmanager.memory.process.size: {{ .Values.worker.jvm_mem }}
+ parallelism.default: {{ .Values.worker.replicas }}
+
+ log4j-console.properties: |+
+ log4j.rootLogger=INFO, console
+ log4j.appender.console=org.apache.log4j.ConsoleAppender
+ log4j.appender.console.layout=org.apache.log4j.PatternLayout
+ log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+ log4j.logger.akka=INFO
+ log4j.logger.org.apache.kafka=INFO
+ log4j.logger.org.apache.hadoop=INFO
+ log4j.logger.org.apache.zookeeper=INFO
+ log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/master-deployment.yaml b/statefun-python-sdk/examples/k8s/resources/templates/master-deployment.yaml
new file mode 100644
index 0000000..e46e988
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/master-deployment.yaml
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ .Values.master.name }}
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: statefun
+ component: master
+ template:
+ metadata:
+ labels:
+ app: statefun
+ component: master
+ spec:
+ containers:
+ - name: master
+ image: {{ .Values.statefun_image }}
+ env:
+ - name: ROLE
+ value: master
+ - name: MASTER_HOST
+ value: {{ .Values.master.name }}
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob
+ - containerPort: 8081
+ name: ui
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /opt/flink/conf
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/master-rest-service.yaml b/statefun-python-sdk/examples/k8s/resources/templates/master-rest-service.yaml
new file mode 100644
index 0000000..d09861d
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/master-rest-service.yaml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{.Values.master.name}}-rest
+spec:
+ type: NodePort
+ ports:
+ - name: rest
+ port: 8081
+ targetPort: 8081
+ selector:
+ app: statefun
+ component: master
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/master-service.yaml b/statefun-python-sdk/examples/k8s/resources/templates/master-service.yaml
new file mode 100644
index 0000000..116180a
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/master-service.yaml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Values.master.name }}
+spec:
+ type: ClusterIP
+ ports:
+ - name: rpc
+ port: 6123
+ - name: blob
+ port: 6124
+ - name: ui
+ port: 8081
+ selector:
+ app: statefun
+ component: master
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/python-worker-deployment.yaml b/statefun-python-sdk/examples/k8s/resources/templates/python-worker-deployment.yaml
new file mode 100644
index 0000000..8ff3f49
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/python-worker-deployment.yaml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ .Values.python.name }}
+spec:
+ replicas: {{ .Values.python.replicas }}
+ selector:
+ matchLabels:
+ app: statefun
+ component: {{ .Values.python.name }}
+ template:
+ metadata:
+ labels:
+ app: statefun
+ component: {{ .Values.python.name }}
+ spec:
+ containers:
+ - name: worker
+ image: {{ .Values.python.image }}
+ ports:
+ - containerPort: 8000
+ name: endpoint
+ livenessProbe:
+ tcpSocket:
+ port: 8000
+ initialDelaySeconds: 30
+ periodSeconds: 60
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/python-worker-service.yaml b/statefun-python-sdk/examples/k8s/resources/templates/python-worker-service.yaml
new file mode 100644
index 0000000..bfa0fdc
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/python-worker-service.yaml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Values.python.name }}
+spec:
+ type: ClusterIP
+ ports:
+ - name: endpoint
+ port: 8000
+ selector:
+ app: statefun
+ component: {{ .Values.python.name }}
diff --git a/statefun-python-sdk/examples/k8s/resources/templates/worker-deployment.yaml b/statefun-python-sdk/examples/k8s/resources/templates/worker-deployment.yaml
new file mode 100644
index 0000000..2d1100a
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/templates/worker-deployment.yaml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ .Values.worker.name }}
+spec:
+ replicas: {{ .Values.worker.replicas }}
+ selector:
+ matchLabels:
+ app: statefun
+ component: worker
+ template:
+ metadata:
+ labels:
+ app: statefun
+ component: worker
+ spec:
+ containers:
+ - name: worker
+ image: {{ .Values.statefun_image }}
+ env:
+ - name: ROLE
+ value: worker
+ - name: MASTER_HOST
+ value: {{ .Values.master.name }}
+ resources:
+ requests:
+ memory: "{{ .Values.worker.container_mem }}"
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob
+ - containerPort: 8081
+ name: ui
+ livenessProbe:
+ tcpSocket:
+ port: 6123
+ initialDelaySeconds: 30
+ periodSeconds: 60
+ volumeMounts:
+ - name: flink-config-volume
+ mountPath: /opt/flink/conf
+ volumes:
+ - name: flink-config-volume
+ configMap:
+ name: flink-config
+ items:
+ - key: flink-conf.yaml
+ path: flink-conf.yaml
+ - key: log4j-console.properties
+ path: log4j-console.properties
diff --git a/statefun-python-sdk/examples/k8s/resources/values.yaml b/statefun-python-sdk/examples/k8s/resources/values.yaml
new file mode 100644
index 0000000..cd07790
--- /dev/null
+++ b/statefun-python-sdk/examples/k8s/resources/values.yaml
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+checkpoint:
+ dir: file:///checkpoint-dir
+ interval: 10sec
+
+master:
+ name: statefun-master
+ image: statefun-demo
+
+worker:
+ name: statefun-worker
+ image: statefun-demo
+ jvm_mem: 1g
+ container_mem: 1.5Gi
+ replicas: 3
+
+python:
+ name: statefun-python
+ image: k8s-demo-python-worker
+ replicas: 1
+