You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/11/29 20:22:40 UTC
[streampipes] 01/01: Add docker-compose and CLI environments for Nats messaging option (#818)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch SP-818
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 30660163e65e30608b26e0241ee1f7aa3d0eb2f8
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Tue Nov 29 21:22:28 2022 +0100
Add docker-compose and CLI environments for Nats messaging option (#818)
---
installer/cli/.env | 6 +-
installer/cli/environments/{lite => backend-nats} | 12 +-
installer/cli/environments/{lite => basic-nats} | 10 +-
installer/cli/environments/lite | 3 +-
.../environments/{lite => pipeline-element-nats} | 12 +-
installer/compose/README.md | 23 +++-
installer/compose/docker-compose.nats.yml | 135 +++++++++++++++++++++
7 files changed, 169 insertions(+), 32 deletions(-)
diff --git a/installer/cli/.env b/installer/cli/.env
index 09e1aa2e7..939031080 100644
--- a/installer/cli/.env
+++ b/installer/cli/.env
@@ -26,10 +26,10 @@ COMPOSE_PROJECT_NAME=streampipes
# -------------------------------------------------------------
# Configuration to set prioritized protocol in backend service
-#
-# options: kafka, mqtt, jms
+# needs to be changed in case one of the 'nats' or 'minimal' options is selected
+# options: kafka, mqtt, jms, nats
# default: kafka
-#SP_PRIORITIZED_PROTOCOL=mqtt
+#SP_PRIORITIZED_PROTOCOL=nats
# Configuration to set MQTT host in backend service
#
diff --git a/installer/cli/environments/lite b/installer/cli/environments/backend-nats
similarity index 87%
copy from installer/cli/environments/lite
copy to installer/cli/environments/backend-nats
index 3e6053a8a..2abf3dee5 100644
--- a/installer/cli/environments/lite
+++ b/installer/cli/environments/backend-nats
@@ -13,14 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:lite]
-ui
-backend
-connect-adapters-iiot
+[environment:backend]
consul
couchdb
-kafka
-zookeeper
influxdb
-sources-watertank-simulator
-pipeline-elements-all-jvm
+nats
+extensions-all-jvm
+ui
diff --git a/installer/cli/environments/lite b/installer/cli/environments/basic-nats
similarity index 86%
copy from installer/cli/environments/lite
copy to installer/cli/environments/basic-nats
index 3e6053a8a..fb7ef1c89 100644
--- a/installer/cli/environments/lite
+++ b/installer/cli/environments/basic-nats
@@ -13,14 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:lite]
-ui
-backend
-connect-adapters-iiot
+[environment:basic]
consul
couchdb
-kafka
-zookeeper
influxdb
-sources-watertank-simulator
-pipeline-elements-all-jvm
+nats
diff --git a/installer/cli/environments/lite b/installer/cli/environments/lite
index 3e6053a8a..145cce38b 100644
--- a/installer/cli/environments/lite
+++ b/installer/cli/environments/lite
@@ -16,11 +16,10 @@
[environment:lite]
ui
backend
-connect-adapters-iiot
consul
couchdb
kafka
zookeeper
influxdb
sources-watertank-simulator
-pipeline-elements-all-jvm
+extensions-all-jvm
diff --git a/installer/cli/environments/lite b/installer/cli/environments/pipeline-element-nats
similarity index 91%
copy from installer/cli/environments/lite
copy to installer/cli/environments/pipeline-element-nats
index 3e6053a8a..d1fa850b0 100644
--- a/installer/cli/environments/lite
+++ b/installer/cli/environments/pipeline-element-nats
@@ -13,14 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[environment:lite]
-ui
+[environment:pipeline-element]
backend
-connect-adapters-iiot
consul
+extensions-all-jvm
couchdb
-kafka
-zookeeper
-influxdb
+ui
sources-watertank-simulator
-pipeline-elements-all-jvm
+nats
+influxdb
diff --git a/installer/compose/README.md b/installer/compose/README.md
index b81194577..7cd9e0cad 100644
--- a/installer/compose/README.md
+++ b/installer/compose/README.md
@@ -43,10 +43,14 @@ Tested on: **macOS, Linux, Windows 10** (CMD, PowerShell, GitBash)
> **NOTE**: On purpose, we disabled all port mappings except of http port **80** to access the StreamPipes UI to provide minimal surface for conflicting ports.
## Usage
-We provide two options to get you going:
+We provide three options to get you going:
-- **default**: a light-weight option with few pipeline elements, needs less memory
-- **full**: contains more pipeline elements, requires **>16 GB RAM** (recommended)
+- **default**: the standard installation, uses Kafka as internal message broker (recommended)
+- **nats**: the standard installation which uses Nats as message broker (recommended for new installations)
+- **full**: contains experimental Flink wrappers
+
+The ``nats`` version will become the default version in a later release. You can already try it for new installations,
+but there's not yet an automatic migration from current Kafka-based installations to Nats.
**Starting** the **default** option is as easy as simply running:
> **NOTE**: Starting might take a while since `docker-compose up` also initially pulls all Docker images from Dockerhub.
@@ -64,6 +68,17 @@ docker-compose down
# docker-compose down -v
```
+Starting the **nats** option works as follows:
+```bash
+docker-compose -f docker-compose.nats.yml up -d
+# go to after all services are started http://localhost
+```
+Stopping the **nats** option:
+```bash
+docker-compose -f docker-compose.nats.yml down
+#docker-compose -f docker-compose.nats.yml down
+```
+
Starting the **full** option is almost the same, just specify the `docker-compose.full.yml` file:
```bash
docker-compose -f docker-compose.full.yml up -d
@@ -72,7 +87,7 @@ docker-compose -f docker-compose.full.yml up -d
Stopping the **full** option:
```bash
docker-compose -f docker-compose.full.yml down
-#docker-compose -f docker-compose.full.yml down -v
+#docker-compose -f docker-compose.full.yml down
```
## Update services
diff --git a/installer/compose/docker-compose.nats.yml b/installer/compose/docker-compose.nats.yml
new file mode 100644
index 000000000..ba115d92d
--- /dev/null
+++ b/installer/compose/docker-compose.nats.yml
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+version: "3.4"
+
+# global logging
+x-logging:
+ &default-logging
+ options:
+ max-size: '12m'
+ max-file: '5'
+ driver: json-file
+
+services:
+
+ # StreamPipes services
+ backend:
+ image: "${SP_DOCKER_REGISTRY}/backend:${SP_VERSION}"
+ depends_on:
+ - consul
+ - couchdb
+ environment:
+ - SP_PRIORITIZED_PROTOCOL=nats
+ volumes:
+ - backend:/root/.streampipes
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+
+ ui:
+ image: "${SP_DOCKER_REGISTRY}/ui:${SP_VERSION}"
+ ports:
+ - "80:80"
+ depends_on:
+ - couchdb
+ - consul
+ - backend
+ volumes:
+ - nginx:/etc/nginx/
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+
+ consul:
+ image: fogsyio/consul:1.9.6
+ environment:
+ - "CONSUL_LOCAL_CONFIG={\"disable_update_check\": true}"
+ - "CONSUL_BIND_INTERFACE=eth0"
+ - "CONSUL_HTTP_ADDR=0.0.0.0"
+ entrypoint:
+ - consul
+ - agent
+ - -server
+ - -bootstrap-expect=1
+ - -data-dir=/consul/data
+ - -node=consul-one
+ - -bind={{ GetInterfaceIP "eth0" }}
+ - -client=0.0.0.0
+ - -enable-script-checks=true
+ - -ui
+ volumes:
+ - consul:/consul/data
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+ ipv4_address: ${SP_CONSUL_CONTAINER_IP}
+
+ couchdb:
+ image: couchdb:2.3.1
+ volumes:
+ - couchdb:/opt/couchdb/data
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+
+ influxdb:
+ image: fogsyio/influxdb:1.7
+ environment:
+ - INFLUXDB_DATA_ENGINE=tsm1
+ - INFLUXDB_REPORTING_DISABLED=false
+ - INFLUXDB_ADMIN_ENABLED=true
+ volumes:
+ - influxdb:/var/lib/influxdb
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+
+ nats:
+ image: nats
+ restart: unless-stopped
+ networks:
+ spnet:
+
+ # Lite version pipeline elements
+ extensions-all-jvm:
+ image: "${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}"
+ depends_on:
+ - consul
+ logging: *default-logging
+ restart: unless-stopped
+ networks:
+ spnet:
+
+volumes:
+ backend:
+ consul:
+ connect:
+ couchdb:
+ influxdb:
+ nginx:
+
+
+networks:
+ spnet:
+ driver: bridge
+ ipam:
+ config:
+ - subnet: ${SP_SUBNET}
+