You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/12/30 06:55:18 UTC
[streampipes] branch SP-953 created (now 6864a410b)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a change to branch SP-953
in repository https://gitbox.apache.org/repos/asf/streampipes.git
at 6864a410b Improve handling of Consul env variables (#953)
This branch includes the following new commits:
new 6864a410b Improve handling of Consul env variables (#953)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[streampipes] 01/01: Improve handling of Consul env variables (#953)
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch SP-953
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 6864a410bd22242cbffd37b3910c4ded27c335e2
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Dec 30 07:54:52 2022 +0100
Improve handling of Consul env variables (#953)
---
.../main/resources/archetype-resources/Dockerfile | 3 -
.../main/resources/archetype-resources/Dockerfile | 3 -
.../main/resources/archetype-resources/Dockerfile | 3 -
streampipes-backend/Dockerfile | 3 -
.../commons/constants/DefaultEnvValues.java | 5 ++
.../apache/streampipes/commons/constants/Envs.java | 9 +++
.../streampipes-connect-adapters-iiot/Dockerfile | 4 --
.../streampipes-connect-adapters/Dockerfile | 4 --
.../streampipes-extensions-all-iiot/Dockerfile | 4 --
.../streampipes-extensions-all-jvm/Dockerfile | 4 --
.../Dockerfile | 4 --
.../Dockerfile | 4 --
.../streampipes-sinks-internal-jvm/Dockerfile | 4 --
.../Dockerfile | 4 --
.../svcdiscovery/consul/ConsulProvider.java | 66 +++++++++++-----------
15 files changed, 48 insertions(+), 76 deletions(-)
diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
index 4c93ac7a1..4315b5d7f 100644
--- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-extensions-service.jar
ENTRYPOINT ["java", "-jar", "/streampipes-extensions-service.jar"]
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
index e4d5893af..448b6d3ae 100644
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
index e4d5893af..448b6d3ae 100644
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-backend/Dockerfile b/streampipes-backend/Dockerfile
index 8a5e15ba4..4ee566aab 100644
--- a/streampipes-backend/Dockerfile
+++ b/streampipes-backend/Dockerfile
@@ -15,9 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-EXPOSE 8030
-ENV CONSUL_LOCATION consul
-
COPY target/streampipes-backend.jar /streampipes-backend.jar
ENTRYPOINT ["java", "-jar", "/streampipes-backend.jar"]
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
index ec49279fa..590abb902 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
@@ -23,6 +23,11 @@ public class DefaultEnvValues {
public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
+
+ public static final String CONSUL_HOST_DEFAULT = "consul";
+ public static final String CONSUL_HOST_LOCAL = "localhost";
+ public static final int CONSUL_PORT_DEFAULT = 8500;
+
public static final int MAX_WAIT_TIME_AT_SHUTDOWN_DEFAULT = 10000;
public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
index 9543bc62f..9ad589bb6 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
@@ -21,7 +21,12 @@ public enum Envs {
SP_HOST("SP_HOST"),
SP_PORT("SP_PORT"),
+
+ @Deprecated(since = "0.90.0", forRemoval = true)
SP_CONSUL_LOCATION("CONSUL_LOCATION"),
+
+ SP_CONSUL_HOST("SP_CONSUL_HOST"),
+ SP_CONSUL_PORT("SP_CONSUL_PORT"),
SP_KAFKA_RETENTION_MS("SP_KAFKA_RETENTION_MS"),
SP_JWT_SECRET("JWT_SECRET"),
SP_JWT_SIGNING_MODE("SP_JWT_SIGNING_MODE"),
@@ -65,6 +70,10 @@ public enum Envs {
return CustomEnvs.getEnvAsBoolean(this.envVariableName);
}
+ public boolean getValueAsBooleanOrDefault(boolean defaultValue) {
+ return this.exists() ? this.getValueAsBoolean() : defaultValue;
+ }
+
public String getEnvVariableName() {
return envVariableName;
}
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile b/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
index e61f39091..745685b08 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-connect-adapters-iiot.jar /streampipes-connect-adapters-iiot.jar
ENTRYPOINT ["java", "-jar", "/streampipes-connect-adapters-iiot.jar"]
diff --git a/streampipes-extensions/streampipes-connect-adapters/Dockerfile b/streampipes-extensions/streampipes-connect-adapters/Dockerfile
index 8e5b94956..6b146f52e 100644
--- a/streampipes-extensions/streampipes-connect-adapters/Dockerfile
+++ b/streampipes-extensions/streampipes-connect-adapters/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-connect-adapters.jar /streampipes-connect-adapters.jar
ENTRYPOINT ["java", "-jar", "/streampipes-connect-adapters.jar"]
diff --git a/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile b/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
index 99ba5270b..43449cdd8 100644
--- a/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
+++ b/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile b/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
index febbb7e64..29d0b93c2 100644
--- a/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile b/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
index 3e77c29d4..472954d29 100644
--- a/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
+++ b/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-pipeline-elements-all-flink.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile b/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
index ec9f67dba..c53ca7f36 100644
--- a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile b/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
index 27d1e9cc8..9bcae8dbc 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-sinks-internal-jvm.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile b/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
index 12bee655f..070151942 100644
--- a/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
+++ b/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-sources-watertank-simulator.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
index 49d860555..69654eaba 100644
--- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
+++ b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.svcdiscovery.consul;
+import org.apache.streampipes.commons.constants.DefaultEnvValues;
import org.apache.streampipes.commons.constants.Envs;
import com.orbitz.consul.Consul;
@@ -25,29 +26,35 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
import java.net.Socket;
-import java.net.URL;
+import java.util.concurrent.TimeUnit;
public class ConsulProvider {
private static final Logger LOG = LoggerFactory.getLogger(ConsulProvider.class);
+ private static final int CHECK_INTERVAL = 1;
- private static final int CONSUL_DEFAULT_PORT = 8500;
- private static final String CONSUL_URL_REGISTER_SERVICE = "v1/agent/service/register";
+ private final String consulHost;
+ private final String consulUrlString;
+ private final int consulPort;
+
+ public ConsulProvider() {
+ this.consulHost = getConsulHost();
+ this.consulPort = getConsulPort();
+ this.consulUrlString = makeConsulUrl();
+ }
public Consul consulInstance() {
- URL consulUrl = consulURL();
boolean connected;
do {
- LOG.info("Checking if consul is available...");
- connected = checkConsulAvailable(consulUrl);
+ LOG.info("Checking if consul is available on host {} and port {}", consulHost, consulPort);
+ connected = checkConsulAvailable();
if (!connected) {
- LOG.info("Retrying in 1 second");
+ LOG.info("Retrying in {} second", CHECK_INTERVAL);
try {
- Thread.sleep(1000);
+ TimeUnit.SECONDS.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -55,31 +62,16 @@ public class ConsulProvider {
} while (!connected);
LOG.info("Successfully connected to Consul");
- return Consul.builder().withUrl(consulURL()).build();
+ return Consul.builder().withUrl(consulUrlString).build();
}
- private URL consulURL() {
- URL url = null;
-
- if (Envs.SP_CONSUL_LOCATION.exists()) {
- try {
- url = new URL("http", Envs.SP_CONSUL_LOCATION.getValue(), CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- } else {
- try {
- url = new URL("http", "localhost", CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
- return url;
+ public String makeConsulUrl() {
+ return "http://" + consulHost + ":" + consulPort;
}
- private boolean checkConsulAvailable(URL consulUrl) {
+ private boolean checkConsulAvailable() {
try {
- InetSocketAddress sa = new InetSocketAddress(consulUrl.getHost(), consulUrl.getPort());
+ InetSocketAddress sa = new InetSocketAddress(consulHost, consulPort);
Socket ss = new Socket();
ss.connect(sa, 1000);
ss.close();
@@ -89,11 +81,21 @@ public class ConsulProvider {
LOG.info("Could not connect to Consul instance...");
return false;
}
+ }
-
+ private int getConsulPort() {
+ return Envs.SP_CONSUL_PORT.getValueAsIntOrDefault(DefaultEnvValues.CONSUL_PORT_DEFAULT);
}
- public String makeConsulEndpoint() {
- return consulURL().toString() + "/" + CONSUL_URL_REGISTER_SERVICE;
+ private String getConsulHost() {
+ if (Envs.SP_CONSUL_LOCATION.exists()) {
+ return Envs.SP_CONSUL_LOCATION.getValue();
+ } else {
+ if (Envs.SP_DEBUG.getValueAsBooleanOrDefault(false)) {
+ return DefaultEnvValues.CONSUL_HOST_LOCAL;
+ } else {
+ return Envs.SP_CONSUL_HOST.getValueOrDefault(DefaultEnvValues.CONSUL_HOST_DEFAULT);
+ }
+ }
}
}