You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2021/02/01 05:49:46 UTC
[flink-statefun] 01/03: [FLINK-21154] Move *-egress.proto to the
sdk protos
This is an automated email from the ASF dual-hosted git repository.
tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 3a124bb91f431f79b8bab6b0de831056014ccdd9
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Wed Jan 27 18:52:15 2021 +0100
[FLINK-21154] Move *-egress.proto to the sdk protos
---
statefun-flink/statefun-flink-core/pom.xml | 2 +-
statefun-flink/statefun-flink-io/pom.xml | 63 ++++++++++++
statefun-python-sdk/build-distribution.sh | 6 +-
statefun-python-sdk/statefun/kafka_egress_pb2.py | 100 -------------------
statefun-python-sdk/statefun/kinesis_egress_pb2.py | 107 ---------------------
.../src/main/protobuf/io}/kafka-egress.proto | 0
.../src/main/protobuf/io}/kinesis-egress.proto | 0
.../main/protobuf/{ => sdk}/request-reply.proto | 0
8 files changed, 66 insertions(+), 212 deletions(-)
diff --git a/statefun-flink/statefun-flink-core/pom.xml b/statefun-flink/statefun-flink-core/pom.xml
index 05c9f7a..20f56d9 100644
--- a/statefun-flink/statefun-flink-core/pom.xml
+++ b/statefun-flink/statefun-flink-core/pom.xml
@@ -163,7 +163,7 @@ under the License.
<version>${project.version}</version>
<type>jar</type>
<outputDirectory>${additional-sources.dir}</outputDirectory>
- <includes>*.proto</includes>
+ <includes>sdk/*.proto</includes>
</artifactItem>
</artifactItems>
</configuration>
diff --git a/statefun-flink/statefun-flink-io/pom.xml b/statefun-flink/statefun-flink-io/pom.xml
index e723c61..e0da70e 100644
--- a/statefun-flink/statefun-flink-io/pom.xml
+++ b/statefun-flink/statefun-flink-io/pom.xml
@@ -27,6 +27,10 @@ under the License.
<relativePath>..</relativePath>
</parent>
+ <properties>
+ <additional-sources.dir>target/additional-sources</additional-sources.dir>
+ </properties>
+
<artifactId>statefun-flink-io</artifactId>
<dependencies>
@@ -37,6 +41,11 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>statefun-sdk-protos</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
@@ -48,12 +57,66 @@ under the License.
<build>
<plugins>
+ <!--
+ The following plugin is executed in the generated-sources phase,
+ and is responsible to extract the additional *.proto files located
+ at statefun-sdk-protos.jar.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>statefun-sdk-protos</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ <outputDirectory>${additional-sources.dir}</outputDirectory>
+ <includes>io/*.proto</includes>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!--
+ The following plugin invokes protoc to generate Java classes out of the *.proto
+ definitions located at: (1) src/main/protobuf (2) ${additional-sources.dir}.
+ -->
<plugin>
<groupId>com.github.os72</groupId>
<artifactId>protoc-jar-maven-plugin</artifactId>
<version>${protoc-jar-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>generate-protobuf-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <includeStdTypes>true</includeStdTypes>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <cleanOutputFolder>true</cleanOutputFolder>
+ <inputDirectories>
+ <inputDirectory>src/main/protobuf</inputDirectory>
+ <inputDirectory>${additional-sources.dir}</inputDirectory>
+ </inputDirectories>
+ <outputDirectory>${basedir}/target/generated-sources/protoc-jar</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
+
</project>
\ No newline at end of file
diff --git a/statefun-python-sdk/build-distribution.sh b/statefun-python-sdk/build-distribution.sh
index 89e564b..3f74e35 100755
--- a/statefun-python-sdk/build-distribution.sh
+++ b/statefun-python-sdk/build-distribution.sh
@@ -41,15 +41,13 @@ mkdir -p target/
# copy all the sources into target
rsync -a --exclude=target * target/
-# copy the addtional .proto files from the SDK
-rsync -a ${SDK_PROTOS_DIR}/* target/
+# copy the additional .proto files from the SDK
+find ${SDK_PROTOS_DIR} -type f -name "*proto" -exec cp {} target/ \;
cd target/
# built the Python SDK inside a Docker container.
# This build step also generates Protobuf files.
-
-
docker run -v "${BASE_DIR}/target:/app" \
--rm \
--workdir /app \
diff --git a/statefun-python-sdk/statefun/kafka_egress_pb2.py b/statefun-python-sdk/statefun/kafka_egress_pb2.py
deleted file mode 100644
index 0a34749..0000000
--- a/statefun-python-sdk/statefun/kafka_egress_pb2.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: kafka-egress.proto
-
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='kafka-egress.proto',
- package='org.apache.flink.statefun.flink.io',
- syntax='proto3',
- serialized_options=b'\n,org.apache.flink.statefun.flink.io.generatedP\001',
- serialized_pb=b'\n\x12kafka-egress.proto\x12\"org.apache.flink.statefun.flink.io\"F\n\x13KafkaProducerRecord\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\r\n\x05topic\x18\x03 \x01(\tB0\n,org.apache.flink.statefun.flink.io.generatedP\x01\x62\x06proto3'
-)
-
-
-
-
-_KAFKAPRODUCERRECORD = _descriptor.Descriptor(
- name='KafkaProducerRecord',
- full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='key', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='value_bytes', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.value_bytes', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=b"",
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='topic', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.topic', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=58,
- serialized_end=128,
-)
-
-DESCRIPTOR.message_types_by_name['KafkaProducerRecord'] = _KAFKAPRODUCERRECORD
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-KafkaProducerRecord = _reflection.GeneratedProtocolMessageType('KafkaProducerRecord', (_message.Message,), {
- 'DESCRIPTOR' : _KAFKAPRODUCERRECORD,
- '__module__' : 'kafka_egress_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.flink.io.KafkaProducerRecord)
- })
-_sym_db.RegisterMessage(KafkaProducerRecord)
-
-
-DESCRIPTOR._options = None
-# @@protoc_insertion_point(module_scope)
diff --git a/statefun-python-sdk/statefun/kinesis_egress_pb2.py b/statefun-python-sdk/statefun/kinesis_egress_pb2.py
deleted file mode 100644
index 0a5b5ac..0000000
--- a/statefun-python-sdk/statefun/kinesis_egress_pb2.py
+++ /dev/null
@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: kinesis-egress.proto
-
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='kinesis-egress.proto',
- package='org.apache.flink.statefun.flink.io',
- syntax='proto3',
- serialized_options=b'\n,org.apache.flink.statefun.flink.io.generatedP\001',
- serialized_pb=b'\n\x14kinesis-egress.proto\x12\"org.apache.flink.statefun.flink.io\"l\n\x13KinesisEgressRecord\x12\x15\n\rpartition_key\x18\x01 \x01(\t\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x0e\n\x06stream\x18\x03 \x01(\t\x12\x19\n\x11\x65xplicit_hash_key\x18\x04 \x01(\tB0\n,org.apache.flink.statefun.flink.io.generatedP\x01\x62\x06proto3'
-)
-
-
-
-
-_KINESISEGRESSRECORD = _descriptor.Descriptor(
- name='KinesisEgressRecord',
- full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- fields=[
- _descriptor.FieldDescriptor(
- name='partition_key', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.partition_key', index=0,
- number=1, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='value_bytes', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.value_bytes', index=1,
- number=2, type=12, cpp_type=9, label=1,
- has_default_value=False, default_value=b"",
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='stream', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.stream', index=2,
- number=3, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- _descriptor.FieldDescriptor(
- name='explicit_hash_key', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.explicit_hash_key', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=60,
- serialized_end=168,
-)
-
-DESCRIPTOR.message_types_by_name['KinesisEgressRecord'] = _KINESISEGRESSRECORD
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-KinesisEgressRecord = _reflection.GeneratedProtocolMessageType('KinesisEgressRecord', (_message.Message,), {
- 'DESCRIPTOR' : _KINESISEGRESSRECORD,
- '__module__' : 'kinesis_egress_pb2'
- # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.flink.io.KinesisEgressRecord)
- })
-_sym_db.RegisterMessage(KinesisEgressRecord)
-
-
-DESCRIPTOR._options = None
-# @@protoc_insertion_point(module_scope)
diff --git a/statefun-flink/statefun-flink-io/src/main/protobuf/kafka-egress.proto b/statefun-sdk-protos/src/main/protobuf/io/kafka-egress.proto
similarity index 100%
rename from statefun-flink/statefun-flink-io/src/main/protobuf/kafka-egress.proto
rename to statefun-sdk-protos/src/main/protobuf/io/kafka-egress.proto
diff --git a/statefun-flink/statefun-flink-io/src/main/protobuf/kinesis-egress.proto b/statefun-sdk-protos/src/main/protobuf/io/kinesis-egress.proto
similarity index 100%
rename from statefun-flink/statefun-flink-io/src/main/protobuf/kinesis-egress.proto
rename to statefun-sdk-protos/src/main/protobuf/io/kinesis-egress.proto
diff --git a/statefun-sdk-protos/src/main/protobuf/request-reply.proto b/statefun-sdk-protos/src/main/protobuf/sdk/request-reply.proto
similarity index 100%
rename from statefun-sdk-protos/src/main/protobuf/request-reply.proto
rename to statefun-sdk-protos/src/main/protobuf/sdk/request-reply.proto