You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2022/12/30 10:01:27 UTC
[streampipes] branch dev updated: Improve handling of Consul env variables (#953)
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new af23e43f3 Improve handling of Consul env variables (#953)
af23e43f3 is described below
commit af23e43f3848ca5df36fd66e3a4cd5206ec84814
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri Dec 30 07:54:52 2022 +0100
Improve handling of Consul env variables (#953)
---
.../main/resources/archetype-resources/Dockerfile | 3 -
.../main/resources/archetype-resources/Dockerfile | 3 -
.../main/resources/archetype-resources/Dockerfile | 3 -
streampipes-backend/Dockerfile | 3 -
.../commons/constants/DefaultEnvValues.java | 5 ++
.../apache/streampipes/commons/constants/Envs.java | 9 +++
.../streampipes-connect-adapters-iiot/Dockerfile | 4 --
.../streampipes-connect-adapters/Dockerfile | 4 --
.../streampipes-extensions-all-iiot/Dockerfile | 4 --
.../streampipes-extensions-all-jvm/Dockerfile | 4 --
.../Dockerfile | 4 --
.../Dockerfile | 4 --
.../streampipes-sinks-internal-jvm/Dockerfile | 4 --
.../Dockerfile | 4 --
.../svcdiscovery/consul/ConsulProvider.java | 66 +++++++++++-----------
15 files changed, 48 insertions(+), 76 deletions(-)
diff --git a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
index 4c93ac7a1..4315b5d7f 100644
--- a/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-extensions-jvm/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-extensions-service.jar
ENTRYPOINT ["java", "-jar", "/streampipes-extensions-service.jar"]
diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
index e4d5893af..448b6d3ae 100644
--- a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
index e4d5893af..448b6d3ae 100644
--- a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
+++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile
@@ -15,9 +15,6 @@
FROM adoptopenjdk/openjdk8-openj9:alpine
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-backend/Dockerfile b/streampipes-backend/Dockerfile
index 8a5e15ba4..4ee566aab 100644
--- a/streampipes-backend/Dockerfile
+++ b/streampipes-backend/Dockerfile
@@ -15,9 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-EXPOSE 8030
-ENV CONSUL_LOCATION consul
-
COPY target/streampipes-backend.jar /streampipes-backend.jar
ENTRYPOINT ["java", "-jar", "/streampipes-backend.jar"]
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
index ec49279fa..590abb902 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/DefaultEnvValues.java
@@ -23,6 +23,11 @@ public class DefaultEnvValues {
public static final String INITIAL_ADMIN_PW_DEFAULT = "admin";
public static final String INITIAL_CLIENT_USER_DEFAULT = "sp-service-client";
public static final String INITIAL_CLIENT_SECRET_DEFAULT = "my-apache-streampipes-secret-key-change-me";
+
+ public static final String CONSUL_HOST_DEFAULT = "consul";
+ public static final String CONSUL_HOST_LOCAL = "localhost";
+ public static final int CONSUL_PORT_DEFAULT = 8500;
+
public static final int MAX_WAIT_TIME_AT_SHUTDOWN_DEFAULT = 10000;
public static final boolean INSTALL_PIPELINE_ELEMENTS = true;
diff --git a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
index 9543bc62f..9ad589bb6 100644
--- a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
+++ b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
@@ -21,7 +21,12 @@ public enum Envs {
SP_HOST("SP_HOST"),
SP_PORT("SP_PORT"),
+
+ @Deprecated(since = "0.90.0", forRemoval = true)
SP_CONSUL_LOCATION("CONSUL_LOCATION"),
+
+ SP_CONSUL_HOST("SP_CONSUL_HOST"),
+ SP_CONSUL_PORT("SP_CONSUL_PORT"),
SP_KAFKA_RETENTION_MS("SP_KAFKA_RETENTION_MS"),
SP_JWT_SECRET("JWT_SECRET"),
SP_JWT_SIGNING_MODE("SP_JWT_SIGNING_MODE"),
@@ -65,6 +70,10 @@ public enum Envs {
return CustomEnvs.getEnvAsBoolean(this.envVariableName);
}
+ public boolean getValueAsBooleanOrDefault(boolean defaultValue) {
+ return this.exists() ? this.getValueAsBoolean() : defaultValue;
+ }
+
public String getEnvVariableName() {
return envVariableName;
}
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile b/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
index e61f39091..745685b08 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-connect-adapters-iiot.jar /streampipes-connect-adapters-iiot.jar
ENTRYPOINT ["java", "-jar", "/streampipes-connect-adapters-iiot.jar"]
diff --git a/streampipes-extensions/streampipes-connect-adapters/Dockerfile b/streampipes-extensions/streampipes-connect-adapters/Dockerfile
index 8e5b94956..6b146f52e 100644
--- a/streampipes-extensions/streampipes-connect-adapters/Dockerfile
+++ b/streampipes-extensions/streampipes-connect-adapters/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-connect-adapters.jar /streampipes-connect-adapters.jar
ENTRYPOINT ["java", "-jar", "/streampipes-connect-adapters.jar"]
diff --git a/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile b/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
index 99ba5270b..43449cdd8 100644
--- a/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
+++ b/streampipes-extensions/streampipes-extensions-all-iiot/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile b/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
index febbb7e64..29d0b93c2 100644
--- a/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-extensions-all-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile b/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
index 3e77c29d4..472954d29 100644
--- a/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
+++ b/streampipes-extensions/streampipes-pipeline-elements-all-flink/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-pipeline-elements-all-flink.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile b/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
index ec9f67dba..c53ca7f36 100644
--- a/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-pipeline-elements-all-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
# needed pkgs processors-image-processing-jvm
RUN apt-get update && apt-get install ttf-dejavu-core && apt-get install fontconfig
diff --git a/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile b/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
index 27d1e9cc8..9bcae8dbc 100644
--- a/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
+++ b/streampipes-extensions/streampipes-sinks-internal-jvm/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-sinks-internal-jvm.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile b/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
index 12bee655f..070151942 100644
--- a/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
+++ b/streampipes-extensions/streampipes-sources-watertank-simulator/Dockerfile
@@ -15,10 +15,6 @@
FROM eclipse-temurin:11-jre-focal
-ENV CONSUL_LOCATION consul
-
-EXPOSE 8090
-
COPY target/streampipes-sources-watertank-simulator.jar /streampipes-processing-element-container.jar
ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
index 49d860555..69654eaba 100644
--- a/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
+++ b/streampipes-service-discovery-consul/src/main/java/org/apache/streampipes/svcdiscovery/consul/ConsulProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.svcdiscovery.consul;
+import org.apache.streampipes.commons.constants.DefaultEnvValues;
import org.apache.streampipes.commons.constants.Envs;
import com.orbitz.consul.Consul;
@@ -25,29 +26,35 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
import java.net.Socket;
-import java.net.URL;
+import java.util.concurrent.TimeUnit;
public class ConsulProvider {
private static final Logger LOG = LoggerFactory.getLogger(ConsulProvider.class);
+ private static final int CHECK_INTERVAL = 1;
- private static final int CONSUL_DEFAULT_PORT = 8500;
- private static final String CONSUL_URL_REGISTER_SERVICE = "v1/agent/service/register";
+ private final String consulHost;
+ private final String consulUrlString;
+ private final int consulPort;
+
+ public ConsulProvider() {
+ this.consulHost = getConsulHost();
+ this.consulPort = getConsulPort();
+ this.consulUrlString = makeConsulUrl();
+ }
public Consul consulInstance() {
- URL consulUrl = consulURL();
boolean connected;
do {
- LOG.info("Checking if consul is available...");
- connected = checkConsulAvailable(consulUrl);
+ LOG.info("Checking if consul is available on host {} and port {}", consulHost, consulPort);
+ connected = checkConsulAvailable();
if (!connected) {
- LOG.info("Retrying in 1 second");
+ LOG.info("Retrying in {} second", CHECK_INTERVAL);
try {
- Thread.sleep(1000);
+ TimeUnit.SECONDS.sleep(CHECK_INTERVAL);
} catch (InterruptedException e) {
e.printStackTrace();
}
@@ -55,31 +62,16 @@ public class ConsulProvider {
} while (!connected);
LOG.info("Successfully connected to Consul");
- return Consul.builder().withUrl(consulURL()).build();
+ return Consul.builder().withUrl(consulUrlString).build();
}
- private URL consulURL() {
- URL url = null;
-
- if (Envs.SP_CONSUL_LOCATION.exists()) {
- try {
- url = new URL("http", Envs.SP_CONSUL_LOCATION.getValue(), CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- } else {
- try {
- url = new URL("http", "localhost", CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
- return url;
+ public String makeConsulUrl() {
+ return "http://" + consulHost + ":" + consulPort;
}
- private boolean checkConsulAvailable(URL consulUrl) {
+ private boolean checkConsulAvailable() {
try {
- InetSocketAddress sa = new InetSocketAddress(consulUrl.getHost(), consulUrl.getPort());
+ InetSocketAddress sa = new InetSocketAddress(consulHost, consulPort);
Socket ss = new Socket();
ss.connect(sa, 1000);
ss.close();
@@ -89,11 +81,21 @@ public class ConsulProvider {
LOG.info("Could not connect to Consul instance...");
return false;
}
+ }
-
+ private int getConsulPort() {
+ return Envs.SP_CONSUL_PORT.getValueAsIntOrDefault(DefaultEnvValues.CONSUL_PORT_DEFAULT);
}
- public String makeConsulEndpoint() {
- return consulURL().toString() + "/" + CONSUL_URL_REGISTER_SERVICE;
+ private String getConsulHost() {
+ if (Envs.SP_CONSUL_LOCATION.exists()) {
+ return Envs.SP_CONSUL_LOCATION.getValue();
+ } else {
+ if (Envs.SP_DEBUG.getValueAsBooleanOrDefault(false)) {
+ return DefaultEnvValues.CONSUL_HOST_LOCAL;
+ } else {
+ return Envs.SP_CONSUL_HOST.getValueOrDefault(DefaultEnvValues.CONSUL_HOST_DEFAULT);
+ }
+ }
}
}