You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by wi...@apache.org on 2021/05/07 08:50:00 UTC
[incubator-streampipes-installer] 03/03: add kafka-cloud/edge option
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes-installer.git
commit 40daea59a658f3c87c0ac0e0568907b38676669f
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Fri May 7 10:49:44 2021 +0200
add kafka-cloud/edge option
---
cli/bin/commands/add | 1 +
.../standalone/kafka-cloud/docker-compose.dev.yml | 37 +++++++++++++
.../standalone/kafka-cloud/docker-compose.yml | 62 ++++++++++++++++++++++
.../standalone/kafka-edge/docker-compose.dev.yml | 37 +++++++++++++
.../standalone/kafka-edge/docker-compose.yml | 62 ++++++++++++++++++++++
.../mosquitto-edge-2/docker-compose.dev.yml} | 15 ++----
.../mosquitto-edge-2/docker-compose.yml} | 25 +++++----
cli/environments/edge | 1 +
cli/environments/{edge => edge-kafka} | 9 ++--
9 files changed, 223 insertions(+), 26 deletions(-)
diff --git a/cli/bin/commands/add b/cli/bin/commands/add
index 44d16e7..a45122f 100755
--- a/cli/bin/commands/add
+++ b/cli/bin/commands/add
@@ -229,6 +229,7 @@ store_in_environment() {
add_service() {
+ echo $svc_store_environment
svc_dir=$STREAMPIPES_WORKDIR/deploy/standalone/${svc_name}
create_dir $svc_dir
create_docker_compose_files $svc_dir
diff --git a/cli/deploy/standalone/kafka-cloud/docker-compose.dev.yml b/cli/deploy/standalone/kafka-cloud/docker-compose.dev.yml
new file mode 100644
index 0000000..8567e08
--- /dev/null
+++ b/cli/deploy/standalone/kafka-cloud/docker-compose.dev.yml
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+services:
+ kafka-cloud:
+ ports:
+ - "9095:9095"
+ depends_on:
+ - zookeeper-cloud
+ environment:
+ # see: https://github.com/confluentinc/schema-registry/issues/648
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9095 #(local, i.e. Single-Node with IDE + Docker)
+ KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9095
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+ KAFKA_ADVERTISED_HOST_NAME: kafka-cloud
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper-cloud:2181
+ KAFKA_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
+
+ zookeeper-cloud:
+ ports:
+ - "2182:2181"
diff --git a/cli/deploy/standalone/kafka-cloud/docker-compose.yml b/cli/deploy/standalone/kafka-cloud/docker-compose.yml
new file mode 100644
index 0000000..8c710ed
--- /dev/null
+++ b/cli/deploy/standalone/kafka-cloud/docker-compose.yml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+services:
+ kafka-cloud:
+ image: fogsyio/kafka:2.2.0
+ hostname: kafka-cloud
+ depends_on:
+ - zookeeper-cloud
+ environment:
+ # see: https://github.com/confluentinc/schema-registry/issues/648
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
+ KAFKA_LISTENERS: PLAINTEXT://:9092
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+ KAFKA_ADVERTISED_HOST_NAME: kafka-cloud
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper-cloud:2181
+ KAFKA_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
+ # volumes:
+ # - kafka:/kafka
+ # - /var/run/docker.sock:/var/run/docker.sock
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1m"
+ max-file: "1"
+ networks:
+ spnet:
+
+ zookeeper-cloud:
+ image: fogsyio/zookeeper:3.4.13
+ # volumes:
+ # - zookeeper:/opt/zookeeper-3.4.13
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1m"
+ max-file: "1"
+ networks:
+ spnet:
+
+volumes:
+ kafka:
+
+networks:
+ spnet:
+ external: true
diff --git a/cli/deploy/standalone/kafka-edge/docker-compose.dev.yml b/cli/deploy/standalone/kafka-edge/docker-compose.dev.yml
new file mode 100644
index 0000000..9a4a3f0
--- /dev/null
+++ b/cli/deploy/standalone/kafka-edge/docker-compose.dev.yml
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+services:
+ kafka-edge:
+ ports:
+ - "9094:9094"
+ depends_on:
+ - zookeeper-edge
+ environment:
+ # see: https://github.com/confluentinc/schema-registry/issues/648
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092,OUTSIDE://localhost:9094 #(local, i.e. Single-Node with IDE + Docker)
+ KAFKA_LISTENERS: PLAINTEXT://:9092,OUTSIDE://:9094
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+ KAFKA_ADVERTISED_HOST_NAME: kafka-edge
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper-edge:2181
+ KAFKA_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
+
+ zookeeper-edge:
+ ports:
+ - "2181:2181"
\ No newline at end of file
diff --git a/cli/deploy/standalone/kafka-edge/docker-compose.yml b/cli/deploy/standalone/kafka-edge/docker-compose.yml
new file mode 100644
index 0000000..222b469
--- /dev/null
+++ b/cli/deploy/standalone/kafka-edge/docker-compose.yml
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+services:
+ kafka-edge:
+ image: fogsyio/kafka:2.2.0
+ hostname: kafka-edge
+ depends_on:
+ - zookeeper-edge
+ environment:
+ # see: https://github.com/confluentinc/schema-registry/issues/648
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
+ KAFKA_LISTENERS: PLAINTEXT://:9092
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+ KAFKA_ADVERTISED_HOST_NAME: kafka-edge
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper-edge:2181
+ KAFKA_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_FETCH_MESSAGE_MAX_BYTES: 5000012
+ KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
+ # volumes:
+ # - kafka:/kafka
+ # - /var/run/docker.sock:/var/run/docker.sock
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1m"
+ max-file: "1"
+ networks:
+ spnet:
+
+ zookeeper-edge:
+ image: fogsyio/zookeeper:3.4.13
+ # volumes:
+ # - zookeeper:/opt/zookeeper-3.4.13
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1m"
+ max-file: "1"
+ networks:
+ spnet:
+
+volumes:
+ kafka:
+
+networks:
+ spnet:
+ external: true
diff --git a/cli/environments/edge b/cli/deploy/standalone/mosquitto-edge-2/docker-compose.dev.yml
similarity index 87%
copy from cli/environments/edge
copy to cli/deploy/standalone/mosquitto-edge-2/docker-compose.dev.yml
index 8b18f49..45b16bc 100644
--- a/cli/environments/edge
+++ b/cli/deploy/standalone/mosquitto-edge-2/docker-compose.dev.yml
@@ -13,13 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:edge]
-activemq
-kafka
-consul
-zookeeper
-couchdb
-influxdb
-mosquitto
-mosquitto-cloud
-mosquitto-fog
+version: "3.4"
+services:
+ mosquitto-edge-2:
+ ports:
+ - "1887:1883"
diff --git a/cli/environments/edge b/cli/deploy/standalone/mosquitto-edge-2/docker-compose.yml
similarity index 76%
copy from cli/environments/edge
copy to cli/deploy/standalone/mosquitto-edge-2/docker-compose.yml
index 8b18f49..b40a7aa 100644
--- a/cli/environments/edge
+++ b/cli/deploy/standalone/mosquitto-edge-2/docker-compose.yml
@@ -13,13 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:edge]
-activemq
-kafka
-consul
-zookeeper
-couchdb
-influxdb
-mosquitto
-mosquitto-cloud
-mosquitto-fog
+version: "3.4"
+services:
+ mosquitto-edge-2:
+ image: eclipse-mosquitto:1.6.12
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "1m"
+ max-file: "1"
+ networks:
+ spnet:
+
+networks:
+ spnet:
+ external: true
diff --git a/cli/environments/edge b/cli/environments/edge
index 8b18f49..7d9eacb 100644
--- a/cli/environments/edge
+++ b/cli/environments/edge
@@ -23,3 +23,4 @@ influxdb
mosquitto
mosquitto-cloud
mosquitto-fog
+mosquitto-edge-2
diff --git a/cli/environments/edge b/cli/environments/edge-kafka
similarity index 91%
copy from cli/environments/edge
copy to cli/environments/edge-kafka
index 8b18f49..ce4237e 100644
--- a/cli/environments/edge
+++ b/cli/environments/edge-kafka
@@ -13,13 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:edge]
+[environment:edge-kafka]
activemq
-kafka
+kafka-edge
+kafka-cloud
consul
-zookeeper
couchdb
influxdb
-mosquitto
-mosquitto-cloud
-mosquitto-fog