You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by wi...@apache.org on 2021/05/12 07:07:21 UTC
[incubator-streampipes] 03/03: [WIP] refactoring node-controller
This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch edge-extensions
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit f21a77322a2ca0905806d2bb961c30388112c216
Author: Patrick Wiener <wi...@fzi.de>
AuthorDate: Wed May 12 09:05:54 2021 +0200
[WIP] refactoring node-controller
---
.../node/container/SupportedArchitectures.java | 14 +-
.../container/management/node/NodeManager.java | 272 ++++++++++++++
.../node/controller/api/ConnectResource.java | 32 +-
.../api/ContainerDeploymentResource.java | 10 +-
.../controller/api/InvocableEntityResource.java | 18 +-
.../api/NodeInfoDescriptionResource.java | 2 +-
.../node/controller/config/EnvConfigParam.java | 3 +-
.../node/controller/config/NodeConfiguration.java | 14 +
.../container/DockerExtensionsContainer.java | 13 +-
.../controller/container/DockerKafkaContainer.java | 13 +-
.../container/DockerMosquittoContainer.java | 13 +-
.../container/DockerZookeeperContainer.java | 13 +-
.../container/StreamPipesDockerServiceID.java | 25 --
.../policies/Comparator.java => IHandler.java} | 7 +-
.../management/NodeControllerSubmitter.java | 6 +-
.../ConnectInteractionType.java} | 11 +-
.../management/connect/ConnectManager.java | 220 ------------
.../management/connect/ConnectWorkerManager.java | 75 ++++
.../management/connect/IConnectManager.java | 33 ++
.../connect/handler/MasterRegistrationHandler.java | 46 +++
.../connect/handler/WorkerFetchAssetsHandler.java | 49 +++
.../handler/WorkerFetchConfigurationsHandler.java | 67 ++++
.../connect/handler/WorkerGuessSchemaHandler.java | 42 +++
.../connect/handler/WorkerInteractionHandler.java | 88 +++++
.../handler/utils/ConnectEndpointUtils.java | 159 +++++++++
.../controller/management/node/INodeManager.java | 9 +-
...NodeManager.java => NodeClientHttpFactory.java} | 27 +-
.../management/node/NodeClientHttpHandler.java | 68 ++++
.../NodeLifeCycleType.java} | 8 +-
.../controller/management/node/NodeManager.java | 146 ++------
.../offloading/OffloadingPolicyManager.java | 40 +--
.../model/OffloadingStrategyFactory.java | 60 ----
.../{model => strategies}/OffloadingStrategy.java | 8 +-
.../strategies/OffloadingStrategyFactory.java | 61 ++++
.../OffloadingStrategyType.java} | 26 +-
.../{model => strategies}/policies/Comparator.java | 2 +-
.../policies/OffloadingPolicy.java | 2 +-
.../ThresholdViolationOffloadingPolicy.java | 2 +-
.../property/CPULoadResourceProperty.java | 2 +-
.../property/FreeDiskSpaceResourceProperty.java | 2 +-
.../property/FreeMemoryResourceProperty.java | 2 +-
.../property/ResourceProperty.java | 2 +-
.../selection/CPULoadSelectionStrategy.java | 4 +-
.../selection/PrioritySelectionStrategy.java | 38 +-
.../selection/RandomSelectionStrategy.java | 4 +-
.../selection/SelectionStrategy.java | 4 +-
.../orchestrator/DockerEngineManager.java | 35 +-
.../management/orchestrator/IContainerEngine.java | 5 -
.../docker/AbstractStreamPipesDockerContainer.java | 3 +-
.../docker/DockerContainerDeclarerSingleton.java | 13 +-
.../orchestrator/docker/model/DockerInfo.java | 3 +-
.../management/pe/IPipelineElementLifeCycle.java | 8 +-
.../management/pe/InvocableElementManager.java | 391 ---------------------
.../PipelineElementLifeCycleState.java} | 12 +-
.../management/pe/PipelineElementManager.java | 113 ++++++
.../pe/handler/ConsulInteractionHandler.java | 89 +++++
.../handler/PipelineElementInteractionHandler.java | 108 ++++++
.../PipelineElementNodeInteractionHandler.java | 113 ++++++
.../handler/PipelineElementOffloadingHandler.java | 55 +++
.../PipelineElementReconfigurationHandler.java | 130 +++++++
.../pe/handler/PipelineElementRestartOnReboot.java | 61 ++++
.../validator/RequirementsResourceValidator.java | 110 ++++++
.../validator/Validator.java} | 18 +-
.../management/resource/ResourceManager.java | 35 +-
.../management/resource/utils/ResourceChecker.java | 82 -----
.../Comparator.java => utils/HttpRequest.java} | 7 +-
.../node/controller/utils/HttpUtils.java | 150 +++++++-
.../node/controller/utils/VersionUtils.java | 62 ++++
.../monitor/health/ClusterHealthCheckMonitor.java | 16 +-
.../monitor/resource/ClusterResourceMonitor.java | 12 +-
.../http/ReconfigurationEndpointUrlGenerator.java | 2 +-
.../migration/MigrationPipelineGenerator.java | 87 +++--
.../save-pipeline/save-pipeline.component.scss | 6 +-
ui/src/scss/sp/sp-dialog.scss | 1 -
74 files changed, 2307 insertions(+), 1182 deletions(-)
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/container/SupportedArchitectures.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/container/SupportedArchitectures.java
index 0d7aa04..0fb3f4b 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/container/SupportedArchitectures.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/container/SupportedArchitectures.java
@@ -22,19 +22,19 @@ import org.apache.streampipes.vocabulary.StreamPipes;
public class SupportedArchitectures {
private static final String ARCHITECTURE_NAMESPACE = StreamPipes.NS + "architecture#";
- public static final String AMD = ARCHITECTURE_NAMESPACE + "x86_64";
+ public static final String AMD64 = ARCHITECTURE_NAMESPACE + "x86_64";
public static final String ARM32 = ARCHITECTURE_NAMESPACE + "arm32";
- public static final String ARM64 = ARCHITECTURE_NAMESPACE + "aarch64";
+ public static final String AARCH64 = ARCHITECTURE_NAMESPACE + "aarch64";
- public static String amd(){
- return AMD;
+ public static String amd64(){
+ return AMD64;
}
- public static String arm32v7(){
+ public static String arm32(){
return ARM32;
}
- public static String arm64v8(){
- return ARM64;
+ public static String aarch64(){
+ return AARCH64;
}
}
diff --git a/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/node/NodeManager.java b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/node/NodeManager.java
new file mode 100644
index 0000000..dac74c2
--- /dev/null
+++ b/streampipes-node-controller-container/src/main/java/org/apache/streampipes/node/controller/container/management/node/NodeManager.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.container.management.node;
+
+import org.apache.http.client.fluent.Request;
+import org.apache.http.client.fluent.Response;
+import org.apache.http.entity.ContentType;
+import org.apache.streampipes.model.client.version.VersionInfo;
+import org.apache.streampipes.model.message.Message;
+import org.apache.streampipes.model.message.NotificationType;
+import org.apache.streampipes.model.message.Notifications;
+import org.apache.streampipes.model.message.SuccessMessage;
+import org.apache.streampipes.model.node.*;
+import org.apache.streampipes.model.node.container.DockerContainer;
+import org.apache.streampipes.model.node.meta.GeoLocation;
+import org.apache.streampipes.node.controller.container.config.NodeControllerConfig;
+import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class NodeManager {
+ private static final Logger LOG = LoggerFactory.getLogger(NodeManager.class.getCanonicalName());
+
+ private static final String PROTOCOL = "http://";
+ private static final String SLASH = "/";
+ private static final String COLON = ":";
+ private static final String BACKEND_BASE_ROUTE = "/streampipes-backend";
+ private static final String BACKEND_VERSION_ROUTE = "/api/v2/info/versions";
+ private static final String NODE_REGISTRATION_ROUTE = "/api/v2/users/admin@streampipes.org/nodes";
+ private static final String NODE_SYNC_UPDATE_ROUTE = "/api/v2/users/admin@streampipes.org/nodes/sync";
+ private static final long RETRY_INTERVAL_MS = 5000;
+ private static final int CONNECT_TIMEOUT_MS = 10000;
+
+ private NodeInfoDescription nodeInfo = new NodeInfoDescription();
+
+ private static NodeManager instance = null;
+
+ private NodeManager() {}
+
+ public static NodeManager getInstance() {
+ if (instance == null)
+ instance = new NodeManager();
+ return instance;
+ }
+
+ public void add(NodeInfoDescription n) {
+ nodeInfo = n;
+ }
+
+ public NodeInfoDescription retrieveNodeInfoDescription() {
+ return nodeInfo;
+ }
+
+ public void init() {
+ NodeInfoDescription nodeInfoDescription =
+ NodeInfoDescriptionBuilder.create(NodeConstants.NODE_CONTROLLER_ID)
+ .withHostname(NodeConstants.NODE_HOSTNAME)
+ .withPort(NodeConstants.NODE_PORT)
+ .withNodeBroker(NodeConstants.NODE_BROKER_HOST, NodeConstants.NODE_BROKER_PORT)
+ .staticNodeMetadata(
+ NodeConstants.NODE_TYPE,
+ NodeConstants.NODE_MODEL,
+ new GeoLocation(),
+ NodeConstants.NODE_LOCATION_TAGS)
+ .withSupportedElements(NodeConstants.SUPPORTED_PIPELINE_ELEMENTS)
+ .withRegisteredContainers(NodeConstants.REGISTERED_DOCKER_CONTAINER)
+ .withNodeResources(NodeResourceBuilder.create()
+ .hardwareResource(
+ NodeConstants.NODE_CPU,
+ NodeConstants.NODE_MEMORY,
+ NodeConstants.NODE_DISK,
+ NodeConstants.NODE_GPU)
+ .softwareResource(
+ NodeConstants.NODE_OPERATING_SYSTEM,
+ NodeConstants.NODE_KERNEL_VERSION,
+ NodeConstants.NODE_CONTAINER_RUNTIME)
+ .withFieldDeviceAccessResources(NodeConstants.FIELD_DEVICE_ACCESS_RESOURCE_LIST)
+ .build())
+ .build();
+
+ add(nodeInfoDescription);
+ }
+
+ public Message updateNodeInfoDescription(NodeInfoDescription desc) {
+ LOG.info("Update node description for node controller: {}", desc.getNodeControllerId());
+ this.nodeInfo = desc;
+ return Notifications.success(NotificationType.OPERATION_SUCCESS);
+ }
+
+ public Message activate() {
+ LOG.info("Activate node controller");
+ this.nodeInfo.setActive(true);
+ return Notifications.success(NotificationType.OPERATION_SUCCESS);
+ }
+
+ public Message deactivate() {
+ LOG.info("Deactivate node controller");
+ this.nodeInfo.setActive(false);
+ return Notifications.success(NotificationType.OPERATION_SUCCESS);
+ }
+
+ public void addToRegisteredContainers(DockerContainer container) {
+ if(!registered(container)) {
+ this.nodeInfo.addRegisteredContainer(container);
+ syncWithNodeClusterManager();
+ }
+ }
+
+ public void removeFromRegisteredContainers(DockerContainer container) {
+ if(registered(container)) {
+ this.nodeInfo.removeRegisteredContainer(container);
+ syncWithNodeClusterManager();
+ }
+ }
+
+ // Interactions with core
+
+ public boolean register() {
+ boolean connected = false;
+ try {
+ String body = JacksonSerializer.getObjectMapper().writeValueAsString(this.nodeInfo);
+ String endpoint = generateRegistrationEndpoint();
+
+ LOG.info("Trying to register node at node management: " + endpoint);
+
+ while (!connected) {
+ connected = post(endpoint, body);
+ if (!connected) {
+ LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOG.info("Successfully registered node at backend");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return connected;
+ }
+
+
+ private boolean syncWithNodeClusterManager() {
+ boolean connected = false;
+ try {
+ String body = JacksonSerializer.getObjectMapper().writeValueAsString(this.nodeInfo);
+ String endpoint = generateSyncronizationEndpoint();
+
+ LOG.info("Trying to sync node updates with node management: " + endpoint);
+
+ while (!connected) {
+ connected = post(endpoint, body);
+ if (!connected) {
+ LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOG.info("Successfully registered node at backend");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return connected;
+ }
+
+ public String getStreamPipesVersion() {
+ boolean connected = false;
+ VersionInfo versionInfo = new VersionInfo();
+ try {
+ String endpoint = generateVersionEndpoint();
+
+ LOG.info("Trying to fetch StreamPipes version from core: " + endpoint);
+
+ while (!connected) {
+ Response response = get(endpoint);
+ versionInfo = deserialize(response, VersionInfo.class);
+
+ if (versionInfo.getBackendVersion() != null) {
+ connected = true;
+ }
+
+ if (!connected) {
+ LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOG.info("Successfully registered node at backend");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return versionInfo.getBackendVersion();
+ }
+
+ private Response get(String endpoint) throws IOException {
+ return Request.Get(endpoint)
+ .connectTimeout(CONNECT_TIMEOUT_MS)
+ .addHeader("Authorization", "Bearer " + NodeControllerConfig.INSTANCE.getApiKey())
+ .execute();
+ }
+
+ private boolean post(String endpoint, String body) throws IOException {
+ Response response = Request.Post(endpoint)
+ .bodyString(body, ContentType.APPLICATION_JSON)
+ .connectTimeout(CONNECT_TIMEOUT_MS)
+ //.addHeader("Authorization", "Bearer " + NodeControllerConfig.INSTANCE.getApiKey())
+ .execute();
+ return handleResponse(response);
+ }
+
+ // Helpers
+
+ private boolean registered(DockerContainer container) {
+ return this.nodeInfo.getRegisteredContainers().contains(container);
+ }
+
+ private boolean handleResponse(Response response) throws IOException {
+ SuccessMessage message = deserialize(response, SuccessMessage.class);
+ LOG.info(message.getNotifications().get(0).getDescription());
+ return message.isSuccess();
+ }
+
+ private <T> T deserialize(Response response, Class<T> clazz) throws IOException {
+ String resp = response.returnContent().asString();
+ return JacksonSerializer.getObjectMapper().readValue(resp, clazz);
+ }
+
+ private String generateRegistrationEndpoint() {
+ return generateBaseEndpoint() + NODE_REGISTRATION_ROUTE;
+ }
+
+ private String generateSyncronizationEndpoint() {
+ return generateBaseEndpoint() + NODE_SYNC_UPDATE_ROUTE;
+ }
+
+ private String generateVersionEndpoint() {
+ return generateBaseEndpoint() + BACKEND_VERSION_ROUTE;
+ }
+
+ private String generateBaseEndpoint() {
+ return PROTOCOL
+ + NodeControllerConfig.INSTANCE.backendLocation()
+ + COLON
+ + NodeControllerConfig.INSTANCE.backendPort()
+ + BACKEND_BASE_ROUTE;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ConnectResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ConnectResource.java
index 8970551..8391d99 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ConnectResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ConnectResource.java
@@ -22,7 +22,7 @@ import org.apache.streampipes.model.connect.adapter.AdapterSetDescription;
import org.apache.streampipes.model.connect.adapter.AdapterStreamDescription;
import org.apache.streampipes.model.connect.worker.ConnectWorkerContainer;
import org.apache.streampipes.model.runtime.RuntimeOptionsRequest;
-import org.apache.streampipes.node.controller.management.connect.ConnectManager;
+import org.apache.streampipes.node.controller.management.connect.ConnectWorkerManager;
import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
import javax.ws.rs.*;
@@ -40,7 +40,7 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response register(@PathParam("username") String username, ConnectWorkerContainer cw) {
- return ok(ConnectManager.getInstance().register(username, cw));
+ return ok(ConnectWorkerManager.getInstance().register(username, cw));
}
// AdapterResource
@@ -48,22 +48,22 @@ public class ConnectResource extends AbstractResource {
@Path("/adapters/{id}/assets")
@Produces("application/zip")
public Response getAdapterAssets(@PathParam("username") String username, @PathParam("id") String appId) {
- return ok(ConnectManager.getInstance().assets(username, appId, "adapter", "/"));
+ return ok(ConnectWorkerManager.getInstance().fetchAssets(username, appId, "adapter", "/"));
}
@GET
@Path("/adapters/{id}/assets/icon")
@Produces("image/png")
public Response getAdapterIconAsset(@PathParam("username") String username, @PathParam("id") String appId) {
- return ok(ConnectManager.getInstance().assets(username, appId, "adapter", "/icon"));
+ return ok(ConnectWorkerManager.getInstance().fetchAssets(username, appId, "adapter", "/icon"));
}
@GET
@Path("/adapters/{id}/assets/documentation")
@Produces(MediaType.TEXT_PLAIN)
public String getAdapterDocumentationAsset(@PathParam("username") String username, @PathParam("id") String appId) {
- return ConnectManager.getInstance()
- .assets(username, appId, "adapter", "/documentation").toString();
+ return ConnectWorkerManager.getInstance()
+ .fetchAssets(username, appId, "adapter", "/documentation").toString();
}
// ProtocolResource
@@ -71,14 +71,14 @@ public class ConnectResource extends AbstractResource {
@Path("/protocols/{id}/assets")
@Produces("application/zip")
public Response getProtocolAssets(@PathParam("username") String username, @PathParam("id") String appId) {
- return ok(ConnectManager.getInstance().assets(username, appId, "protocol", "/"));
+ return ok(ConnectWorkerManager.getInstance().fetchAssets(username, appId, "protocol", "/"));
}
@GET
@Path("/protocols/{id}/assets/icon")
@Produces("image/png")
public Response getProtocolIconAsset(@PathParam("username") String username, @PathParam("id") String appId) {
- return ok(ConnectManager.getInstance().assets(username, appId, "protocol", "/icon"));
+ return ok(ConnectWorkerManager.getInstance().fetchAssets(username, appId, "protocol", "/icon"));
}
@GET
@@ -86,8 +86,8 @@ public class ConnectResource extends AbstractResource {
@Produces(MediaType.TEXT_PLAIN)
public String getProtocolDocumentationAsset(@PathParam("username") String username,
@PathParam("id") String appId) {
- return ConnectManager.getInstance()
- .assets(username, appId, "protocol", "documentation").toString();
+ return ConnectWorkerManager.getInstance()
+ .fetchAssets(username, appId, "protocol", "documentation").toString();
}
// WorkerResource
@@ -97,7 +97,7 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response invokeStreamAdapter(@PathParam("username") String username, AdapterStreamDescription ad) {
- return ok(ConnectManager.getInstance().invoke(username, ad));
+ return ok(ConnectWorkerManager.getInstance().invoke(username, ad));
}
@POST
@@ -106,7 +106,7 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response stopStreamAdapter(@PathParam("username") String username, AdapterStreamDescription ad) {
- return ok(ConnectManager.getInstance().stop(username, ad));
+ return ok(ConnectWorkerManager.getInstance().detach(username, ad));
}
@POST
@@ -115,7 +115,7 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response invokeSetAdapter(@PathParam("username") String username, AdapterSetDescription ad) {
- return ok(ConnectManager.getInstance().invoke(username, ad));
+ return ok(ConnectWorkerManager.getInstance().invoke(username, ad));
}
@POST
@@ -124,7 +124,7 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response stopSetAdapter(@PathParam("username") String username, AdapterSetDescription ad){
- return ok(ConnectManager.getInstance().stop(username, ad));
+ return ok(ConnectWorkerManager.getInstance().detach(username, ad));
}
@POST
@@ -132,7 +132,7 @@ public class ConnectResource extends AbstractResource {
@Path("/guess/schema")
@Produces(MediaType.APPLICATION_JSON)
public Response guessSchema(@PathParam("username") String username, AdapterDescription ad) {
- return ok(ConnectManager.getInstance().guess(username, ad));
+ return ok(ConnectWorkerManager.getInstance().guessSchema(username, ad));
}
// RuntimeResolvableResource
@@ -143,6 +143,6 @@ public class ConnectResource extends AbstractResource {
@Consumes(MediaType.APPLICATION_JSON)
public Response fetchConfigurations(@PathParam("username") String username, @PathParam("id") String appId,
RuntimeOptionsRequest runtimeOptions) {
- return ok(ConnectManager.getInstance().fetchConfigurations(username, appId, runtimeOptions));
+ return ok(ConnectWorkerManager.getInstance().fetchConfigurations(username, appId, runtimeOptions));
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ContainerDeploymentResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ContainerDeploymentResource.java
index 42c2ab6..50c398c 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ContainerDeploymentResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/ContainerDeploymentResource.java
@@ -23,7 +23,8 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Doc
import org.apache.streampipes.node.controller.management.orchestrator.status.ContainerDeploymentStatus;
import org.apache.streampipes.node.controller.management.orchestrator.docker.model.ContainerStatus;
import org.apache.streampipes.node.controller.management.orchestrator.DockerEngineManager;
-import org.apache.streampipes.node.controller.management.pe.InvocableElementManager;
+import org.apache.streampipes.node.controller.management.orchestrator.storage.RunningContainerInstances;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementManager;
import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
@@ -51,7 +52,7 @@ public class ContainerDeploymentResource extends AbstractResource {
ContainerDeploymentStatus status = DockerEngineManager.getInstance().deploy(container);
if (status.getStatus() == ContainerStatus.DEPLOYED) {
- NodeManager.getInstance().registerContainer(status.getContainer());
+ NodeManager.getInstance().registerContainerWhenDeployed(status.getContainer());
}
return ok(status);
}
@@ -63,8 +64,9 @@ public class ContainerDeploymentResource extends AbstractResource {
ContainerDeploymentStatus status = DockerEngineManager.getInstance().remove(container);
if (status.getStatus() == ContainerStatus.REMOVED) {
- InvocableElementManager.getInstance().unregister();
- NodeManager.getInstance().deregisterContainer(status.getContainer());
+ PipelineElementManager.getInstance().deregister();
+ NodeManager.getInstance().unregisterContainerWhenRemoved(status.getContainer());
+ RunningContainerInstances.INSTANCE.remove(status.getContainerId());
}
return ok(status);
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
index 473d426..109df9d 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/InvocableEntityResource.java
@@ -21,7 +21,7 @@ import org.apache.streampipes.container.model.node.InvocableRegistration;
import org.apache.streampipes.model.Response;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.pipeline.PipelineElementReconfigurationEntity;
-import org.apache.streampipes.node.controller.management.pe.InvocableElementManager;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementManager;
import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
import org.apache.streampipes.rest.shared.annotation.JacksonSerialized;
import org.slf4j.Logger;
@@ -42,7 +42,7 @@ public class InvocableEntityResource extends AbstractResource {
@JacksonSerialized
@Consumes(MediaType.APPLICATION_JSON)
public void register(InvocableRegistration registration) {
- InvocableElementManager.getInstance().register(registration);
+ PipelineElementManager.getInstance().register(registration);
}
@POST
@@ -53,7 +53,7 @@ public class InvocableEntityResource extends AbstractResource {
@PathParam("elementId") String elementId, InvocableStreamPipesEntity graph) {
if (identifier.equals(DATA_PROCESSOR_PREFIX)) {
- Response elementResponse = InvocableElementManager.getInstance().invoke(graph);
+ Response elementResponse = PipelineElementManager.getInstance().invoke(graph);
if (elementResponse.isSuccess()) {
RunningInvocableInstances.INSTANCE.add(graph.getDeploymentRunningInstanceId(), graph);
}
@@ -61,7 +61,7 @@ public class InvocableEntityResource extends AbstractResource {
return ok(elementResponse);
} else if (identifier.equals(DATA_SINK_PREFIX)) {
- Response elementResponse = InvocableElementManager.getInstance().invoke(graph);
+ Response elementResponse = PipelineElementManager.getInstance().invoke(graph);
if (elementResponse.isSuccess()) {
RunningInvocableInstances.INSTANCE.add(graph.getDeploymentRunningInstanceId(), graph);
}
@@ -77,22 +77,22 @@ public class InvocableEntityResource extends AbstractResource {
public javax.ws.rs.core.Response detach(@PathParam("identifier") String identifier,
@PathParam("elementId") String elementId,
@PathParam("runningInstanceId") String runningInstanceId) {
- String endpoint = RunningInvocableInstances.INSTANCE.get(runningInstanceId).getBelongsTo();
- Response resp = InvocableElementManager.getInstance().detach(endpoint + SLASH + runningInstanceId);
+ InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
+ Response resp = PipelineElementManager.getInstance().detach(graph, runningInstanceId);
RunningInvocableInstances.INSTANCE.remove(runningInstanceId);
return ok(resp);
}
- // Adaptation
+ // Live-Reconfiguration
@POST
@JacksonSerialized
- @Path("/adapt/{runningInstanceId}")
+ @Path("/reconfigure/{runningInstanceId}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public javax.ws.rs.core.Response adapt( @PathParam("runningInstanceId") String runningInstanceId,
PipelineElementReconfigurationEntity reconfigurationEntity) {
InvocableStreamPipesEntity graph = RunningInvocableInstances.INSTANCE.get(runningInstanceId);
- return ok(InvocableElementManager.getInstance().adapt(graph, reconfigurationEntity));
+ return ok(PipelineElementManager.getInstance().reconfigure(graph, reconfigurationEntity));
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/NodeInfoDescriptionResource.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/NodeInfoDescriptionResource.java
index ed11292..614501c 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/NodeInfoDescriptionResource.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/api/NodeInfoDescriptionResource.java
@@ -36,7 +36,7 @@ public class NodeInfoDescriptionResource extends AbstractResource {
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getNodeInfo() {
- return ok(NodeManager.getInstance().getNode());
+ return ok(NodeManager.getInstance().getNodeInfoDescription());
}
@PUT
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/EnvConfigParam.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/EnvConfigParam.java
index 9746a0a..0854466 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/EnvConfigParam.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/EnvConfigParam.java
@@ -48,7 +48,8 @@ public enum EnvConfigParam {
NODE_GPU_TYPE("SP_NODE_GPU_TYPE", "n/a"),
NODE_ACCESSIBLE_FIELD_DEVICE("SP_NODE_ACCESSIBLE_FIELD_DEVICE", ""),
CONSUL_LOCATION("CONSUL_LOCATION", "consul"),
- SUPPORTED_PIPELINE_ELEMENTS("SP_SUPPORTED_PIPELINE_ELEMENTS", "");
+ SUPPORTED_PIPELINE_ELEMENTS("SP_SUPPORTED_PIPELINE_ELEMENTS", ""),
+ AUTO_OFFLOADING_STRATEGY("SP_AUTO_OFFLOADING_STRATEGY", "default");
private final String environmentKey;
private final String defaultValue;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/NodeConfiguration.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/NodeConfiguration.java
index d27dc46..872464f 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/NodeConfiguration.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/config/NodeConfiguration.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.node.controller.config;
import org.apache.commons.validator.routines.UrlValidator;
import org.apache.streampipes.model.node.resources.fielddevice.FieldDeviceAccessResource;
import org.apache.streampipes.node.controller.config.utils.ConfigUtils;
+import org.apache.streampipes.node.controller.management.offloading.strategies.OffloadingStrategyType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +58,7 @@ public final class NodeConfiguration {
private static int resourceMonitorFreqSecs;
private static int relayEventBufferSize;
private static String consulHost;
+ private static OffloadingStrategyType autoOffloadingStrategy;
private static HashMap<String, String> configMap;
@@ -272,6 +274,14 @@ public final class NodeConfiguration {
NodeConfiguration.supportedPipelineElements.addAll(supportedPipelineElements);
}
+ public static OffloadingStrategyType getAutoOffloadingStrategy() {
+ return autoOffloadingStrategy;
+ }
+
+ public static void setAutoOffloadingStrategy(OffloadingStrategyType autoOffloadingStrategy) {
+ NodeConfiguration.autoOffloadingStrategy = autoOffloadingStrategy;
+ }
+
public static HashMap<String, String> getConfigMap() {
return configMap;
}
@@ -478,6 +488,10 @@ public final class NodeConfiguration {
List<String> supportedPipelineElements = Arrays.asList(value.split(";").clone());
addSupportedPipelineElements(supportedPipelineElements);
break;
+ case AUTO_OFFLOADING_STRATEGY:
+ configMap.put(envKey, value);
+ setAutoOffloadingStrategy(OffloadingStrategyType.fromString(value));
+ break;
default:
throw new IllegalArgumentException("Invalid environment config param: " + configParam);
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerExtensionsContainer.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerExtensionsContainer.java
index a455714..ef06b75 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerExtensionsContainer.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerExtensionsContainer.java
@@ -23,14 +23,16 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Abs
public class DockerExtensionsContainer extends AbstractStreamPipesDockerContainer {
+ public static String SP_SVC_EXTENSIONS_ID = "svc/org.apache.streampipes.pe.extensions";
+
@Override
public DockerContainer declareDockerContainer() {
- return DockerContainerBuilder.create(StreamPipesDockerServiceID.SP_SVC_EXTENSIONS_ID)
+ return DockerContainerBuilder.create(SP_SVC_EXTENSIONS_ID)
.withImage("apachestreampipes/extensions-all-jvm:" + getStreamPipesVersion())
.supportedArchitectures(
- SupportedArchitectures.amd(),
- SupportedArchitectures.arm32v7(),
- SupportedArchitectures.arm64v8())
+ SupportedArchitectures.amd64(),
+ SupportedArchitectures.arm32(),
+ SupportedArchitectures.aarch64())
.supportedOperatingSystemTypes(
SupportedOsType.linux(),
SupportedOsType.darwin())
@@ -41,8 +43,7 @@ public class DockerExtensionsContainer extends AbstractStreamPipesDockerContaine
.add("SP_HOST", NodeConfiguration.getNodeHost())
.add("SP_PORT", "8090")
.build())
- .withLabels(ContainerLabels.with(StreamPipesDockerServiceID.SP_SVC_EXTENSIONS_ID, retrieveNodeType(),
- ContainerType.EXTENSIONS))
+ .withLabels(ContainerLabels.with(SP_SVC_EXTENSIONS_ID, retrieveNodeType(), ContainerType.EXTENSIONS))
.build();
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerKafkaContainer.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerKafkaContainer.java
index 16ba52e..1812055 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerKafkaContainer.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerKafkaContainer.java
@@ -24,14 +24,16 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Abs
public class DockerKafkaContainer extends AbstractStreamPipesDockerContainer {
+ public static String SP_SVC_KAFKA_ID = "svc/org.apache.streampipes.node.broker.kafka";
+
@Override
public DockerContainer declareDockerContainer() {
- return DockerContainerBuilder.create(StreamPipesDockerServiceID.SP_SVC_KAFKA_ID)
+ return DockerContainerBuilder.create(SP_SVC_KAFKA_ID)
.withImage("fogsyio/kafka:2.2.0")
.supportedArchitectures(
- SupportedArchitectures.amd(),
- SupportedArchitectures.arm32v7(),
- SupportedArchitectures.arm64v8())
+ SupportedArchitectures.amd64(),
+ SupportedArchitectures.arm32(),
+ SupportedArchitectures.aarch64())
.supportedOperatingSystemTypes(
SupportedOsType.linux(),
SupportedOsType.darwin())
@@ -51,8 +53,7 @@ public class DockerKafkaContainer extends AbstractStreamPipesDockerContainer {
.add("KAFKA_FETCH_MESSAGE_MAX_BYTES", "5000012")
.add("KAFKA_REPLICA_FETCH_MAX_BYTES", "10000000")
.build())
- .withLabels(ContainerLabels.with(StreamPipesDockerServiceID.SP_SVC_KAFKA_ID, retrieveNodeType(),
- ContainerType.BROKER))
+ .withLabels(ContainerLabels.with(SP_SVC_KAFKA_ID, retrieveNodeType(), ContainerType.BROKER))
.withVolumes(ContainerVolumesBuilder.create()
.add("streampipes-kafka-vol", "/kafka", false)
.build())
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerMosquittoContainer.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerMosquittoContainer.java
index b3fff80..4f80b19 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerMosquittoContainer.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerMosquittoContainer.java
@@ -22,14 +22,16 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Abs
public class DockerMosquittoContainer extends AbstractStreamPipesDockerContainer {
+ public static String SP_SVC_MOSQUITTO_ID = "svc/org.apache.streampipes.node.broker.mosquitto";
+
@Override
public DockerContainer declareDockerContainer() {
- return DockerContainerBuilder.create(StreamPipesDockerServiceID.SP_SVC_MOSQUITTO_ID)
+ return DockerContainerBuilder.create(SP_SVC_MOSQUITTO_ID)
.withImage("eclipse-mosquitto:1.6.12")
.supportedArchitectures(
- SupportedArchitectures.amd(),
- SupportedArchitectures.arm32v7(),
- SupportedArchitectures.arm64v8())
+ SupportedArchitectures.amd64(),
+ SupportedArchitectures.arm32(),
+ SupportedArchitectures.aarch64())
.supportedOperatingSystemTypes(
SupportedOsType.linux(),
SupportedOsType.darwin())
@@ -38,8 +40,7 @@ public class DockerMosquittoContainer extends AbstractStreamPipesDockerContainer
.withEnvironmentVariables(ContainerEnvBuilder.create()
.addNodeEnvs(generateStreamPipesNodeEnvs())
.build())
- .withLabels(ContainerLabels.with(StreamPipesDockerServiceID.SP_SVC_MOSQUITTO_ID, retrieveNodeType(),
- ContainerType.BROKER))
+ .withLabels(ContainerLabels.with(SP_SVC_MOSQUITTO_ID, retrieveNodeType(), ContainerType.BROKER))
.build();
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerZookeeperContainer.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerZookeeperContainer.java
index b861084..bdb5ee8 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerZookeeperContainer.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/DockerZookeeperContainer.java
@@ -22,14 +22,16 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Abs
public class DockerZookeeperContainer extends AbstractStreamPipesDockerContainer {
+ public static String SP_SVC_ZOOKEEPER_ID = "svc/org.apache.streampipes.node.broker.zookeeper";
+
@Override
public DockerContainer declareDockerContainer() {
- return DockerContainerBuilder.create(StreamPipesDockerServiceID.SP_SVC_ZOOKEEPER_ID)
+ return DockerContainerBuilder.create(SP_SVC_ZOOKEEPER_ID)
.withImage("fogsyio/zookeeper:3.4.13")
.supportedArchitectures(
- SupportedArchitectures.amd(),
- SupportedArchitectures.arm32v7(),
- SupportedArchitectures.arm64v8())
+ SupportedArchitectures.amd64(),
+ SupportedArchitectures.arm32(),
+ SupportedArchitectures.aarch64())
.supportedOperatingSystemTypes(
SupportedOsType.linux(),
SupportedOsType.darwin())
@@ -38,8 +40,7 @@ public class DockerZookeeperContainer extends AbstractStreamPipesDockerContainer
.withEnvironmentVariables(ContainerEnvBuilder.create()
.addNodeEnvs(generateStreamPipesNodeEnvs())
.build())
- .withLabels(ContainerLabels.with(StreamPipesDockerServiceID.SP_SVC_ZOOKEEPER_ID, retrieveNodeType(),
- ContainerType.BROKER))
+ .withLabels(ContainerLabels.with(SP_SVC_ZOOKEEPER_ID, retrieveNodeType(), ContainerType.BROKER))
.build();
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/StreamPipesDockerServiceID.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/StreamPipesDockerServiceID.java
deleted file mode 100644
index a6bc540..0000000
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/container/StreamPipesDockerServiceID.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.node.controller.container;
-
-public class StreamPipesDockerServiceID {
- public static String SP_SVC_EXTENSIONS_ID = "svc/org.apache.streampipes.pe.extensions";
- public static String SP_SVC_MOSQUITTO_ID = "svc/org.apache.streampipes.node.broker.mosquitto";
- public static String SP_SVC_KAFKA_ID = "svc/org.apache.streampipes.node.broker.kafka";
- public static String SP_SVC_ZOOKEEPER_ID = "svc/org.apache.streampipes.node.broker.zookeeper";
-}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/IHandler.java
similarity index 85%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/IHandler.java
index ec31bfc..53fbc88 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/IHandler.java
@@ -15,9 +15,8 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management;
-public enum Comparator {
- SMALLER,
- GREATER
+public interface IHandler<T> {
+ T handle();
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/NodeControllerSubmitter.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/NodeControllerSubmitter.java
index 399ebe9..9f4a669 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/NodeControllerSubmitter.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/NodeControllerSubmitter.java
@@ -58,8 +58,8 @@ public abstract class NodeControllerSubmitter {
LOG.info("Load node info description");
NodeManager.getInstance().init();
- LOG.info("Register node controller at StreamPipes cluster management");
- boolean success = NodeManager.getInstance().register();
+ LOG.info("Register node controller at StreamPipes node management");
+ boolean success = NodeManager.getInstance().registerNodeAtNodeManagement();
if (success) {
LOG.info("Start resource manager");
@@ -80,7 +80,7 @@ public abstract class NodeControllerSubmitter {
LOG.info("Start janitor manager");
JanitorManager.getInstance().run();
}
- } else throw new SpRuntimeException("Could not register node controller at backend");
+ } else throw new SpRuntimeException("Could not register node controller at StreamPipes node management");
}
@PreDestroy
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectInteractionType.java
similarity index 81%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectInteractionType.java
index ec31bfc..a3204e8 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectInteractionType.java
@@ -15,9 +15,12 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.connect;
-public enum Comparator {
- SMALLER,
- GREATER
+public enum ConnectInteractionType {
+ INVOKE,
+ DETACH,
+ GUESS_SCHEMA,
+ FETCH_CONFIGS,
+ FETCH_ASSETS
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectManager.java
deleted file mode 100644
index da2f6d7..0000000
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectManager.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.node.controller.management.connect;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.fluent.Request;
-import org.apache.http.client.fluent.Response;
-import org.apache.http.entity.ContentType;
-import org.apache.http.util.EntityUtils;
-import org.apache.streampipes.commons.exceptions.SpRuntimeException;
-import org.apache.streampipes.model.base.UnnamedStreamPipesEntity;
-import org.apache.streampipes.model.connect.adapter.AdapterDescription;
-import org.apache.streampipes.model.connect.adapter.AdapterSetDescription;
-import org.apache.streampipes.model.connect.adapter.AdapterStreamDescription;
-import org.apache.streampipes.model.connect.guess.GuessSchema;
-import org.apache.streampipes.model.connect.worker.ConnectWorkerContainer;
-import org.apache.streampipes.model.runtime.RuntimeOptionsRequest;
-import org.apache.streampipes.node.controller.config.NodeConfiguration;
-import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class ConnectManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(ConnectManager.class.getCanonicalName());
-
- private static final String HTTP_PROTOCOL = "http://";
- private static final String COLON = ":";
- private static final String SLASH = "/";
- private static final String BACKEND_ADMINISTRATION_ROUTE = "/streampipes-backend/api/v2/connect/{username}/master" +
- "/administration";
-
- // Connect adapter base route
- // TODO: get from registered extensions or connect adapater config
- private static final String CONNECT_WORKER_HOST = "streampipes-extensions";
- private static final int CONNECT_WORKER_PORT = 8090;
- private static final String CONNECT_WORKER_BASE_ROUTE = "/api/v1/{username}/worker";
- private static final String STREAM_ROUTE = "/stream";
- private static final String SET_ROUTE = "/set";
- private static final String INVOKE_ROUTE = "/invoke";
- private static final String STOP_ROUTE ="/stop";
- private static final String GUESS_ROUTE = "/guess/schema";
- private static final String RESOLVABLE_ROUTE = "/resolvable/{id}/configurations";
- private static final String ADAPTER_ROUTE = "/adapters/{id}/assets";
- private static final String PROTOCOL_ROUTE = "/protocols/{id}/assets";
-
- private static final Integer CONNECT_TIMEOUT = 10000;
- private static final Integer SOCKET_TIMEOUT = 100000;
- private static ConnectManager instance = null;
-
- private ConnectManager() {}
-
- public static ConnectManager getInstance() {
- if (instance == null) {
- synchronized (ConnectManager.class) {
- if (instance == null)
- instance = new ConnectManager();
- }
- }
- return instance;
- }
-
- // adapter -> backend communication: registration
-
- // MasterRestClient
- public String register(String username, ConnectWorkerContainer wc) {
- String endpoint = (NodeConfiguration.getBackendUrl() + BACKEND_ADMINISTRATION_ROUTE.replace("{username}", username));
- LOG.info("Trying to register connect worker at backend: " + endpoint);
- return post(endpoint , jackson(wc)).toString();
- }
-
- // backend -> adapter communication
-
- // WorkerResource
- public <T extends AdapterDescription> String invoke(String username, T ad) {
- LOG.info("Invoke adapter: appId=" + ad.getAppId() + ", name=" + ad.getName());
- if (ad instanceof AdapterStreamDescription) {
- return post(endpointFromDescription(username, ad, STREAM_ROUTE + INVOKE_ROUTE), jackson(ad)).toString();
- } else if (ad instanceof AdapterSetDescription) {
- return post(endpointFromDescription(username, ad, SET_ROUTE + INVOKE_ROUTE), jackson(ad)).toString();
- }
- throw new SpRuntimeException("Could not invoke adapter: " + ad.getAppId());
- }
-
- public <T extends AdapterDescription> String stop(String username, T ad) {
- LOG.info("Stop adapter: appId=" + ad.getAppId() + ", name=" + ad.getName());
- if (ad instanceof AdapterStreamDescription) {
- return post(endpointFromDescription(username, ad, STREAM_ROUTE + STOP_ROUTE), jackson(ad)).toString();
- } else if (ad instanceof AdapterSetDescription) {
- return post(endpointFromDescription(username, ad, SET_ROUTE + STOP_ROUTE), jackson(ad)).toString();
- }
- throw new SpRuntimeException("Could not stop adapter: " + ad.getAppId());
- }
-
- // GuessResource
- public GuessSchema guess(String username, AdapterDescription ad) {
- try {
- LOG.info("Trying to guess schema: " + ad.getAppId());
-
- Response resp = post(endpointFromDescription(username, ad, GUESS_ROUTE), jackson(ad));
- HttpResponse httpResponse = resp.returnResponse();
- String responseString = EntityUtils.toString(httpResponse.getEntity());
- return JacksonSerializer.getObjectMapper().readValue(responseString, GuessSchema.class);
- } catch (IOException e) {
- e.printStackTrace();
- }
- throw new SpRuntimeException("Could not deserialize object");
- }
-
- // RuntimeResolvableResource
- public String fetchConfigurations(String username, String appId, RuntimeOptionsRequest runtimeOptions) {
- String endpoint = endpointFromStringRoute(username, RESOLVABLE_ROUTE.replace("{id}", appId));
- LOG.info("Trying to fetch configurations at: " + endpoint);
- return post(endpoint, jackson(runtimeOptions)).toString();
- }
-
- // AdapterResource
- public byte[] assets(String username, String appId, String assetType, String subroute) {
- String endpoint = "";
- if ("adapter".equals(assetType)) {
- if (subroute.isEmpty()) {
- endpoint = endpointFromStringRoute(username, ADAPTER_ROUTE.replace("{id}", appId));
- } else {
- endpoint = endpointFromStringRoute(username, (ADAPTER_ROUTE.replace("{id}", appId) + subroute));
- }
- } else if ("protocol".equals(assetType)) {
- if (subroute.isEmpty()) {
- endpoint = endpointFromStringRoute(username, PROTOCOL_ROUTE.replace("{id}", appId));
- } else {
- endpoint = endpointFromStringRoute(username, (PROTOCOL_ROUTE.replace("{id}", appId) + subroute));
- }
- }
- return get(endpoint);
- }
-
- // Helper methods
- private Response post(String endpoint, String payload) {
- try {
- return Request.Post(endpoint)
- .bodyString(payload, ContentType.APPLICATION_JSON)
- .connectTimeout(CONNECT_TIMEOUT)
- .socketTimeout(SOCKET_TIMEOUT)
- .execute();
- } catch (IOException e) {
- e.printStackTrace();
- }
- throw new SpRuntimeException("Post request was not successful");
- }
-
- private byte[] get(String endpoint) {
- try {
- return Request.Get(endpoint)
- .connectTimeout(CONNECT_TIMEOUT)
- .socketTimeout(SOCKET_TIMEOUT)
- .execute().returnContent().asBytes();
- } catch (IOException e) {
- e.printStackTrace();
- }
- throw new SpRuntimeException("Get request was not successful");
- }
-
- private <T extends AdapterDescription> String jackson(T ad) {
- try {
- return JacksonSerializer.getObjectMapper().writeValueAsString(ad);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- throw new SpRuntimeException("Could not serialize object");
- }
-
- private <T extends UnnamedStreamPipesEntity> String jackson(T ad) {
- try {
- return JacksonSerializer.getObjectMapper().writeValueAsString(ad);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- throw new SpRuntimeException("Could not serialize object");
- }
-
- private <T extends AdapterDescription> String endpointFromDescription(String username, T ad, String subroute) {
- return workerUrl(ad) + addUserToBaseRoute(username) + subroute;
- }
-
- private String endpointFromStringRoute(String username, String subroute) {
- return workerUrl() + addUserToBaseRoute(username) + subroute;
- }
-
- private String addUserToBaseRoute(String username) {
- return CONNECT_WORKER_BASE_ROUTE.replace("{username}", username);
- }
-
- private String workerUrl() {
- if("true".equals(System.getenv("SP_DEBUG"))) {
- return HTTP_PROTOCOL + "localhost" + COLON + "7024";
- }
- return HTTP_PROTOCOL + CONNECT_WORKER_HOST + COLON + CONNECT_WORKER_PORT;
- }
-
- private <T extends AdapterDescription> String workerUrl(T ad) {
- return HTTP_PROTOCOL + ad.getElementEndpointHostname() + COLON + ad.getElementEndpointPort();
- }
-}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectWorkerManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectWorkerManager.java
new file mode 100644
index 0000000..0e6d95f
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/ConnectWorkerManager.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect;
+
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.connect.guess.GuessSchema;
+import org.apache.streampipes.model.connect.worker.ConnectWorkerContainer;
+import org.apache.streampipes.model.runtime.RuntimeOptionsRequest;
+import org.apache.streampipes.node.controller.management.connect.handler.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ConnectWorkerManager implements IConnectManager {
+ private static final Logger LOG = LoggerFactory.getLogger(ConnectWorkerManager.class.getCanonicalName());
+
+ private static ConnectWorkerManager instance = null;
+
+ private ConnectWorkerManager() {}
+
+ public static ConnectWorkerManager getInstance() {
+ if (instance == null) {
+ synchronized (ConnectWorkerManager.class) {
+ if (instance == null)
+ instance = new ConnectWorkerManager();
+ }
+ }
+ return instance;
+ }
+
+ @Override
+ public Response register(String username, ConnectWorkerContainer worker) {
+ return new MasterRegistrationHandler(username, worker).handle();
+ }
+
+ @Override
+ public <T extends AdapterDescription> String invoke(String username, T adapterDescription) {
+ return new WorkerInteractionHandler<T>(username, adapterDescription, ConnectInteractionType.INVOKE).handle();
+ }
+
+ @Override
+ public <T extends AdapterDescription> String detach(String username, T adapterDescription) {
+ return new WorkerInteractionHandler<T>(username, adapterDescription, ConnectInteractionType.DETACH).handle();
+ }
+
+ @Override
+ public GuessSchema guessSchema(String username, AdapterDescription adapterDescription) {
+ return new WorkerGuessSchemaHandler(username, adapterDescription).handle();
+ }
+
+ @Override
+ public String fetchConfigurations(String username, String appId, RuntimeOptionsRequest options) {
+ return new WorkerFetchConfigurationsHandler(username, appId, options).handle();
+ }
+
+ @Override
+ public byte[] fetchAssets(String username, String appId, String assetType, String subroute) {
+ return new WorkerFetchAssetsHandler(username, appId, assetType, subroute).handle();
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/IConnectManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/IConnectManager.java
new file mode 100644
index 0000000..0eed989
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/IConnectManager.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect;
+
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.connect.guess.GuessSchema;
+import org.apache.streampipes.model.connect.worker.ConnectWorkerContainer;
+import org.apache.streampipes.model.runtime.RuntimeOptionsRequest;
+
+public interface IConnectManager {
+ Response register(String username, ConnectWorkerContainer worker);
+ <T extends AdapterDescription> String invoke(String username, T adapterDescription);
+ <T extends AdapterDescription> String detach(String username, T adapterDescription);
+ GuessSchema guessSchema(String username, AdapterDescription adapterDescription);
+ String fetchConfigurations(String username, String appId, RuntimeOptionsRequest options);
+ byte[] fetchAssets(String username, String appId, String assetType, String subroute);
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/MasterRegistrationHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/MasterRegistrationHandler.java
new file mode 100644
index 0000000..b9737c9
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/MasterRegistrationHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler;
+
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.connect.worker.ConnectWorkerContainer;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.utils.ConnectEndpointUtils;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MasterRegistrationHandler implements IHandler<Response> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MasterRegistrationHandler.class.getCanonicalName());
+
+ private final String username;
+ private final ConnectWorkerContainer worker;
+
+ public MasterRegistrationHandler(String username, ConnectWorkerContainer worker) {
+ this.username = username;
+ this.worker = worker;
+ }
+
+ @Override
+ public Response handle() {
+ String url = ConnectEndpointUtils.generateMasterAdministrationEndpoint(username);
+ LOG.info("Trying to register connect worker at backend: " + url);
+ return HttpUtils.post(url, worker);
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchAssetsHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchAssetsHandler.java
new file mode 100644
index 0000000..804bfb3
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchAssetsHandler.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler;
+
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.utils.ConnectEndpointUtils;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+public class WorkerFetchAssetsHandler implements IHandler<byte[]> {
+
+ private final String username;
+ private final String appId;
+ private final String assetType;
+ private final String subroute;
+
+ public WorkerFetchAssetsHandler(String username, String appId, String assetType, String subroute) {
+ this.username = username;
+ this.appId = appId;
+ this.assetType = assetType;
+ this.subroute = subroute;
+ }
+
+ @Override
+ public byte[] handle() {
+ String url = null;
+ if ("adapter".equals(assetType)) {
+ url = ConnectEndpointUtils.generateFetchAdapterAssetsEndpoint(username, appId, subroute);
+ } else if ("protocol".equals(assetType)) {
+ url = ConnectEndpointUtils.generateFetchProtocolAssetsEndpoint(username, appId, subroute);
+ }
+
+ return HttpUtils.get(url, byte[].class);
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchConfigurationsHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchConfigurationsHandler.java
new file mode 100644
index 0000000..bdd6db1
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerFetchConfigurationsHandler.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler;
+
+import org.apache.streampipes.model.runtime.RuntimeOptionsRequest;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+public class WorkerFetchConfigurationsHandler implements IHandler<String> {
+
+ private static final String HTTP_PROTOCOL = "http://";
+ private static final String COLON = ":";
+
+ private static final String CONNECT_WORKER_HOST = "streampipes-extensions";
+ private static final int CONNECT_WORKER_PORT = 8090;
+ private static final String CONNECT_WORKER_BASE_ROUTE = "/api/v1/{username}/worker";
+ private static final String RESOLVABLE_ROUTE = "/resolvable/{id}/configurations";
+
+ private final String username;
+ private final String appId;
+ private final RuntimeOptionsRequest options;
+
+ public WorkerFetchConfigurationsHandler(String username, String appId, RuntimeOptionsRequest options) {
+ this.username = username;
+ this.appId = appId;
+ this.options = options;
+ }
+
+ @Override
+ public String handle() {
+ String url = generateFetchConfigurationsEndpoint();
+ String body = HttpUtils.serialize(options);
+
+ return HttpUtils.post(url, body);
+ }
+
+ // Helpers
+
+ private String generateFetchConfigurationsEndpoint() {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + RESOLVABLE_ROUTE.replace("{id}", appId);
+ }
+
+
+ private String workerUrl() {
+ if("true".equals(System.getenv("SP_DEBUG"))) {
+ return HTTP_PROTOCOL + "localhost" + COLON + "7024";
+ }
+ return HTTP_PROTOCOL + CONNECT_WORKER_HOST + COLON + CONNECT_WORKER_PORT;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerGuessSchemaHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerGuessSchemaHandler.java
new file mode 100644
index 0000000..22ad44f
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerGuessSchemaHandler.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler;
+
+
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.connect.guess.GuessSchema;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.utils.ConnectEndpointUtils;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+public class WorkerGuessSchemaHandler implements IHandler<GuessSchema> {
+
+ private final String username;
+ private final AdapterDescription adapterDescription;
+
+ public WorkerGuessSchemaHandler(String username, AdapterDescription adapterDescription) {
+ this.username = username;
+ this.adapterDescription = adapterDescription;
+ }
+
+ @Override
+ public GuessSchema handle() {
+ String url = ConnectEndpointUtils.generateGuessSchemaEndpoint(username, adapterDescription);
+ return HttpUtils.post(url, adapterDescription, GuessSchema.class);
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerInteractionHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerInteractionHandler.java
new file mode 100644
index 0000000..c63733c
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/WorkerInteractionHandler.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.connect.adapter.AdapterSetDescription;
+import org.apache.streampipes.model.connect.adapter.AdapterStreamDescription;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.connect.ConnectInteractionType;
+import org.apache.streampipes.node.controller.management.connect.handler.utils.ConnectEndpointUtils;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WorkerInteractionHandler<T extends AdapterDescription> implements IHandler<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(WorkerInteractionHandler.class.getCanonicalName());
+
+ private static final String INVOKE_ROUTE = "/invoke";
+ private static final String STOP_ROUTE ="/stop";
+
+ private final String username;
+ private final T adapterDescription;
+ private final ConnectInteractionType connectType;
+
+ public WorkerInteractionHandler(String username, T adapterDescription, ConnectInteractionType connectType) {
+ this.username = username;
+ this.adapterDescription = adapterDescription;
+ this.connectType = connectType;
+ }
+
+ @Override
+ public String handle() {
+ switch(connectType) {
+ case INVOKE:
+ return invoke();
+ case DETACH:
+ return detach();
+ default:
+ throw new SpRuntimeException("Unsupported interaction scheme: " + connectType);
+ }
+ }
+
+ private String invoke() {
+ LOG.info("Invoke adapter: appId=" + adapterDescription.getAppId() + ", name=" + adapterDescription.getName());
+
+ String url;
+ if (adapterDescription instanceof AdapterStreamDescription) {
+ url = ConnectEndpointUtils.generateAdapterStreamEndpoint(username, adapterDescription, INVOKE_ROUTE);
+ return HttpUtils.post(url, adapterDescription, String.class);
+
+ } else if (adapterDescription instanceof AdapterSetDescription) {
+ url = ConnectEndpointUtils.generateAdapterSetEndpoint(username, adapterDescription, INVOKE_ROUTE);
+ return HttpUtils.post(url, adapterDescription, String.class);
+ }
+ throw new SpRuntimeException("Could not invoke adapter: " + adapterDescription.getAppId());
+ }
+
+ private String detach() {
+ LOG.info("Stop adapter: appId=" + adapterDescription.getAppId() + ", name=" + adapterDescription.getName());
+ String url;
+ if (adapterDescription instanceof AdapterStreamDescription) {
+ url = ConnectEndpointUtils.generateAdapterStreamEndpoint(username, adapterDescription, STOP_ROUTE);
+ return HttpUtils.post(url, adapterDescription, String.class);
+
+ } else if (adapterDescription instanceof AdapterSetDescription) {
+ url = ConnectEndpointUtils.generateAdapterSetEndpoint(username, adapterDescription, STOP_ROUTE);
+ return HttpUtils.post(url, adapterDescription, String.class);
+ }
+ throw new SpRuntimeException("Could not stop adapter: " + adapterDescription.getAppId());
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/utils/ConnectEndpointUtils.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/utils/ConnectEndpointUtils.java
new file mode 100644
index 0000000..775f3d4
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/connect/handler/utils/ConnectEndpointUtils.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.connect.handler.utils;
+
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.apache.streampipes.node.controller.management.connect.handler.MasterRegistrationHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.WorkerFetchAssetsHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.WorkerGuessSchemaHandler;
+import org.apache.streampipes.node.controller.management.connect.handler.WorkerInteractionHandler;
+
+public class ConnectEndpointUtils {
+
+ private static final String HTTP_PROTOCOL = "http://";
+ private static final String COLON = ":";
+ private static final String CONNECT_WORKER_HOST = "streampipes-extensions";
+ private static final int CONNECT_WORKER_PORT = 8090;
+ private static final String BASE_ROUTE = "/streampipes-backend/api/v2";
+ private static final String BACKEND_ADMINISTRATION_ROUTE = "/connect/{username}/master/administration";
+ private static final String CONNECT_WORKER_BASE_ROUTE = "/api/v1/{username}/worker";
+ private static final String ADAPTER_ROUTE = "/adapters/{id}/assets";
+ private static final String PROTOCOL_ROUTE = "/protocols/{id}/assets";
+ private static final String GUESS_ROUTE = "/guess/schema";
+ private static final String STREAM_ROUTE = "/stream";
+ private static final String SET_ROUTE = "/set";
+
+ /**
+ * Endpoint for {@link MasterRegistrationHandler}
+ *
+ * @param username
+ * @return
+ */
+ public static String generateMasterAdministrationEndpoint(String username) {
+ return NodeConfiguration.getBackendUrl()
+ + BASE_ROUTE
+ + BACKEND_ADMINISTRATION_ROUTE.replace("{username}", username);
+ }
+
+ /**
+ * Endpoint for {@link WorkerFetchAssetsHandler}
+ *
+ * @param username
+ * @param appId
+ * @param subroute
+ * @return
+ */
+ public static String generateFetchAdapterAssetsEndpoint(String username, String appId, String subroute) {
+ if (subroute.isEmpty()) {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + ADAPTER_ROUTE.replace("{id}", appId);
+ } else {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + ADAPTER_ROUTE.replace("{id}", appId)
+ + subroute;
+ }
+ }
+
+ /**
+ * Endpoint for {@link WorkerFetchAssetsHandler}
+ *
+ * @param username
+ * @param appId
+ * @param subroute
+ * @return
+ */
+ public static String generateFetchProtocolAssetsEndpoint(String username, String appId, String subroute) {
+ if (subroute.isEmpty()) {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + PROTOCOL_ROUTE.replace("{id}", appId);
+ } else {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + PROTOCOL_ROUTE.replace("{id}", appId)
+ + subroute;
+ }
+ }
+
+ /**
+ * Endpoint for {@link WorkerGuessSchemaHandler}
+ *
+ * @param username
+ * @param adapterDescription
+ * @return
+ */
+ public static String generateGuessSchemaEndpoint(String username, AdapterDescription adapterDescription) {
+ return workerUrl(adapterDescription)
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + GUESS_ROUTE;
+ }
+
+
+ /**
+ * Endpoint for {@link WorkerInteractionHandler}
+ *
+ * @param username
+ * @param adapterDescription
+ * @param subroute
+ * @return
+ */
+ public static String generateAdapterStreamEndpoint(String username, AdapterDescription adapterDescription,
+ String subroute) {
+ return workerUrl(adapterDescription)
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + STREAM_ROUTE
+ + subroute;
+ }
+
+ /**
+ * Endpoint for {@link WorkerInteractionHandler}
+ *
+ * @param username
+ * @param adapterDescription
+ * @param subroute
+ * @return
+ */
+ public static String generateAdapterSetEndpoint(String username, AdapterDescription adapterDescription,
+ String subroute) {
+ return workerUrl()
+ + CONNECT_WORKER_BASE_ROUTE.replace("{username}", username)
+ + SET_ROUTE
+ + subroute;
+ }
+
+ // Helpers
+
+ private static String workerUrl(AdapterDescription adapterDescription) {
+ return HTTP_PROTOCOL
+ + adapterDescription.getElementEndpointHostname()
+ + COLON
+ + adapterDescription.getElementEndpointPort();
+ }
+
+
+ private static String workerUrl() {
+ if("true".equals(System.getenv("SP_DEBUG"))) {
+ return HTTP_PROTOCOL + "localhost" + COLON + "7024";
+ }
+ return HTTP_PROTOCOL + CONNECT_WORKER_HOST + COLON + CONNECT_WORKER_PORT;
+ }
+
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java
index e5d799d..103f777 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java
@@ -24,11 +24,12 @@ import org.apache.streampipes.model.node.container.DockerContainer;
public interface INodeManager {
void init();
void addNode(NodeInfoDescription node);
- NodeInfoDescription getNode();
+ NodeInfoDescription getNodeInfoDescription();
Message updateNode(NodeInfoDescription node);
Message activateNode();
Message deactivateNode();
- boolean registerNode();
- void registerDeployedContainer(DockerContainer container);
- void deregisterDeployContainer(DockerContainer container);
+ boolean synchronizeNodeWithNodeManagement();
+ boolean registerNodeAtNodeManagement();
+ void registerContainerWhenDeployed(DockerContainer container);
+ void unregisterContainerWhenRemoved(DockerContainer container);
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpFactory.java
similarity index 59%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpFactory.java
index e5d799d..cd5f553 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/INodeManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpFactory.java
@@ -17,18 +17,21 @@
*/
package org.apache.streampipes.node.controller.management.node;
-import org.apache.streampipes.model.message.Message;
import org.apache.streampipes.model.node.NodeInfoDescription;
-import org.apache.streampipes.model.node.container.DockerContainer;
+import org.apache.streampipes.node.controller.utils.HttpRequest;
-public interface INodeManager {
- void init();
- void addNode(NodeInfoDescription node);
- NodeInfoDescription getNode();
- Message updateNode(NodeInfoDescription node);
- Message activateNode();
- Message deactivateNode();
- boolean registerNode();
- void registerDeployedContainer(DockerContainer container);
- void deregisterDeployContainer(DockerContainer container);
+public class NodeClientHttpFactory {
+
+ private static final String BASE_ROUTE = "/streampipes-backend/api/v2/users/admin@streampipes.org/nodes";
+
+ public static boolean execute(NodeInfoDescription node, NodeLifeCycleType type) {
+ switch (type) {
+ case REGISTER:
+ return new NodeClientHttpHandler(node, BASE_ROUTE, HttpRequest.POST).execute();
+ case UPDATE:
+ return new NodeClientHttpHandler(node, BASE_ROUTE + "/sync", HttpRequest.POST).execute();
+ default:
+ return false;
+ }
+ }
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpHandler.java
new file mode 100644
index 0000000..4e16024
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeClientHttpHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.node;
+
+import org.apache.streampipes.model.node.NodeInfoDescription;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.apache.streampipes.node.controller.utils.HttpRequest;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NodeClientHttpHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(NodeClientHttpHandler.class.getCanonicalName());
+
+ private static final long RETRY_INTERVAL_MS = 5000;
+ private final NodeInfoDescription node;
+ private final String route;
+ private final HttpRequest httpRequest;
+
+ public NodeClientHttpHandler(NodeInfoDescription node, String route, HttpRequest httpRequest) {
+ this.node = node;
+ this.route = route;
+ this.httpRequest = httpRequest;
+ }
+
+ public boolean execute() {
+
+ String host = NodeConfiguration.getBackendHost();
+ int port = NodeConfiguration.getBackendPort();
+
+ String url = HttpUtils.generateEndpoint(host, port, route);
+ LOG.info("Trying to sync with StreamPipes node management=" + url);
+
+ boolean connected = false;
+ while (!connected) {
+ // call REST endpoints of StreamPipes node management
+ if (httpRequest == HttpRequest.POST) {
+ connected = HttpUtils.post(url, node, Boolean.class);
+ }
+
+ if (!connected) {
+ LOG.debug("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOG.info("Successfully synced with StreamPipes node management=" + url);
+ return connected;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeLifeCycleType.java
similarity index 85%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeLifeCycleType.java
index ec31bfc..df412bd 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeLifeCycleType.java
@@ -15,9 +15,9 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.node;
-public enum Comparator {
- SMALLER,
- GREATER
+public enum NodeLifeCycleType {
+ REGISTER,
+ UPDATE
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeManager.java
index 8d5fe95..df205a0 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/node/NodeManager.java
@@ -17,7 +17,6 @@
*/
package org.apache.streampipes.node.controller.management.node;
-import org.apache.streampipes.model.client.version.VersionInfo;
import org.apache.streampipes.model.message.Message;
import org.apache.streampipes.model.message.NotificationType;
import org.apache.streampipes.model.message.Notifications;
@@ -25,29 +24,15 @@ import org.apache.streampipes.model.node.*;
import org.apache.streampipes.model.node.container.DeploymentContainer;
import org.apache.streampipes.model.node.container.DockerContainer;
import org.apache.streampipes.model.node.meta.GeoLocation;
-import org.apache.streampipes.node.controller.config.NodeConfiguration;
-import org.apache.streampipes.node.controller.utils.HttpUtils;
-import org.apache.streampipes.node.controller.utils.SocketUtils;
-import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
import java.util.List;
-public class NodeManager {
+public class NodeManager implements INodeManager {
private static final Logger LOG = LoggerFactory.getLogger(NodeManager.class.getCanonicalName());
- private static final String BACKEND_BASE_ROUTE = "/streampipes-backend/api/v2";
- private static final String BACKEND_VERSION_ROUTE = BACKEND_BASE_ROUTE + "/info/versions";
- private static final String NODE_REGISTRATION_ROUTE = BACKEND_BASE_ROUTE + "/users/admin@streampipes.org/nodes";
- private static final String NODE_SYNC_UPDATE_ROUTE = BACKEND_BASE_ROUTE + "/users/admin@streampipes.org/nodes/sync";
- private static final long RETRY_INTERVAL_MS = 5000;
-
private NodeInfoDescription nodeInfo = new NodeInfoDescription();
-
private static NodeManager instance = null;
private NodeManager() {}
@@ -58,14 +43,7 @@ public class NodeManager {
return instance;
}
- public void add(NodeInfoDescription n) {
- nodeInfo = n;
- }
-
- public NodeInfoDescription getNode() {
- return nodeInfo;
- }
-
+ @Override
public void init() {
NodeInfoDescription nodeInfoDescription =
NodeInfoDescriptionBuilder.create(NodeConstants.NODE_CONTROLLER_ID)
@@ -78,6 +56,7 @@ public class NodeManager {
.staticNodeMetadata(
NodeConstants.NODE_TYPE,
NodeConstants.NODE_MODEL,
+ // TODO: get actual geolocation in case of mobile edge node
new GeoLocation(),
NodeConstants.NODE_LOCATION_TAGS)
.withSupportedElements(NodeConstants.SUPPORTED_PIPELINE_ELEMENTS)
@@ -96,129 +75,66 @@ public class NodeManager {
.build())
.build();
- add(nodeInfoDescription);
+ addNode(nodeInfoDescription);
}
+ @Override
+ public void addNode(NodeInfoDescription node) {
+ nodeInfo = node;
+ }
+
+ @Override
+ public NodeInfoDescription getNodeInfoDescription() {
+ return nodeInfo;
+ }
+
+ @Override
public Message updateNode(NodeInfoDescription desc) {
LOG.info("Update node description for node controller: {}", desc.getNodeControllerId());
this.nodeInfo = desc;
return Notifications.success(NotificationType.OPERATION_SUCCESS);
}
+ @Override
public Message activateNode() {
LOG.info("Activate node controller");
this.nodeInfo.setActive(true);
return Notifications.success(NotificationType.OPERATION_SUCCESS);
}
+ @Override
public Message deactivateNode() {
LOG.info("Deactivate node controller");
this.nodeInfo.setActive(false);
return Notifications.success(NotificationType.OPERATION_SUCCESS);
}
- public void registerContainer(DockerContainer container) {
+ @Override
+ public void registerContainerWhenDeployed(DockerContainer container) {
if(!registered(container)) {
this.nodeInfo.addRegisteredContainer(container);
- syncWithNodeClusterManager();
+ synchronizeNodeWithNodeManagement();
}
}
- public void deregisterContainer(DockerContainer container) {
+ @Override
+ public void unregisterContainerWhenRemoved(DockerContainer container) {
if(registered(container)) {
this.nodeInfo.removeRegisteredContainer(container);
- syncWithNodeClusterManager();
+ synchronizeNodeWithNodeManagement();
}
}
- public List<DeploymentContainer> getRegisteredContainer() {
- return this.nodeInfo.getRegisteredContainers();
- }
-
// Interactions with core
- public boolean register() {
- boolean connected = false;
- boolean registered;
- String host = NodeConfiguration.getBackendHost();
- int port = NodeConfiguration.getBackendPort();
- String endpoint = HttpUtils.generateEndpoint(host, port, NODE_REGISTRATION_ROUTE);
-
- while (!connected) {
- LOG.info("Trying to register node controller at StreamPipes cluster management: " + endpoint);
- connected = SocketUtils.isReady(NodeConfiguration.getBackendHost(), NodeConfiguration.getBackendPort());
- if (!connected) {
- LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
- try {
- Thread.sleep(RETRY_INTERVAL_MS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-
- registered = HttpUtils.post(endpoint, this.nodeInfo);
- if (registered) {
- LOG.info("Successfully registered node controller at StreamPipes cluster management");
- }
-
- return registered;
+ @Override
+ public boolean registerNodeAtNodeManagement() {
+ return NodeClientHttpFactory.execute(this.nodeInfo, NodeLifeCycleType.REGISTER);
}
-
- private boolean syncWithNodeClusterManager() {
- boolean connected = false;
- String host = NodeConfiguration.getBackendHost();
- int port = NodeConfiguration.getBackendPort();
-
- String endpoint = HttpUtils.generateEndpoint(host, port, NODE_SYNC_UPDATE_ROUTE);
-
- LOG.info("Trying to sync node updates with StreamPipes cluster management: " + endpoint);
-
- while (!connected) {
- connected = HttpUtils.post(endpoint, this.nodeInfo);
- if (!connected) {
- LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
- try {
- Thread.sleep(RETRY_INTERVAL_MS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- LOG.info("Successfully synced node controller with StreamPipes cluster management");
- return connected;
- }
-
- public String getStreamPipesVersion() {
- boolean connected = false;
- VersionInfo versionInfo = new VersionInfo();
- String host = NodeConfiguration.getBackendHost();
- int port = NodeConfiguration.getBackendPort();
-
- String endpoint = HttpUtils.generateEndpoint(host, port, BACKEND_VERSION_ROUTE);
- String bearerToken = NodeConfiguration.getNodeApiKey();
-
- LOG.info("Trying to retrieve StreamPipes version from backend: " + endpoint);
-
- while (!connected) {
- versionInfo = HttpUtils.get(endpoint, bearerToken, VersionInfo.class);
-
- if (versionInfo.getBackendVersion() != null) {
- connected = true;
- }
-
- if (!connected) {
- LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
- try {
- Thread.sleep(RETRY_INTERVAL_MS);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- LOG.info("Successfully retrieved StreamPipes version from backend");
- return versionInfo.getBackendVersion();
+ @Override
+ public boolean synchronizeNodeWithNodeManagement() {
+ return NodeClientHttpFactory.execute(this.nodeInfo, NodeLifeCycleType.UPDATE);
}
// Helpers
@@ -227,4 +143,8 @@ public class NodeManager {
return this.nodeInfo.getRegisteredContainers().contains(container);
}
+ public List<DeploymentContainer> getRegisteredContainer() {
+ return this.nodeInfo.getRegisteredContainers();
+ }
+
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
index 6f49266..a17ab5c 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/OffloadingPolicyManager.java
@@ -18,22 +18,18 @@
package org.apache.streampipes.node.controller.management.offloading;
-import org.apache.http.client.fluent.Request;
-import org.apache.http.entity.ContentType;
-import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.streampipes.model.Response;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.node.NodeInfoDescription;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
-import org.apache.streampipes.model.pipeline.Pipeline;
import org.apache.streampipes.node.controller.config.NodeConfiguration;
-import org.apache.streampipes.node.controller.management.offloading.model.OffloadingStrategy;
-import org.apache.streampipes.node.controller.management.pe.InvocableElementManager;
-import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.node.controller.management.offloading.strategies.OffloadingStrategy;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementManager;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -42,6 +38,8 @@ public class OffloadingPolicyManager {
private static final String HTTP_PROTOCOL = "http://";
private static final String COLON = ":";
private static final String SLASH = "/";
+ private static final String BACKEND_BASE_ROUTE = "streampipes-backend/api/v2/users/admin@streampipes.org";
+ private static final String NODE_MANAGEMENT_ONLINE_ENDPOINT = BACKEND_BASE_ROUTE + "/nodes/online";
private final List<OffloadingStrategy<?>> offloadingStrategies = new ArrayList<>();
private static OffloadingPolicyManager instance;
@@ -55,18 +53,18 @@ public class OffloadingPolicyManager {
}
public void checkPolicies(ResourceMetrics rm){
- for(OffloadingStrategy strategy:offloadingStrategies){
+ for(OffloadingStrategy strategy : offloadingStrategies){
strategy.getOffloadingPolicy().addValue(strategy.getResourceProperty().getProperty(rm));
if(strategy.getOffloadingPolicy().isViolated()){
- InvocableStreamPipesEntity offloadEntity = strategy.getSelectionStrategy().selectEntity();
+ InvocableStreamPipesEntity offloadEntity = strategy.getSelectionStrategy().select();
if(offloadEntity != null){
- Response resp = InvocableElementManager.getInstance().postOffloadRequest(offloadEntity);
+ Response resp = PipelineElementManager.getInstance().offload(offloadEntity);
if(resp.isSuccess())
LOG.info("Successfully offloaded: " + offloadEntity.getAppId()
- + " from Pipeline: " + offloadEntity.getCorrespondingPipeline());
+ + " of pipeline: " + offloadEntity.getCorrespondingPipeline());
else LOG.info("Failed to offload: " + offloadEntity.getAppId()
- + " from Pipeline: " + offloadEntity.getCorrespondingPipeline());
- }else LOG.info("No entity to offload found");
+ + " of pipeline: " + offloadEntity.getCorrespondingPipeline());
+ } else LOG.info("No pipeline element found to offload");
}
}
}
@@ -76,16 +74,8 @@ public class OffloadingPolicyManager {
}
public List<NodeInfoDescription> getOnlineNodes(){
- try {
- String url = generateNodeManagementOnlineNodesEndpoint();
- org.apache.http.client.fluent.Response resp = Request.Get(url).execute();
- ArrayList<NodeInfoDescription> onlineNodes =
- JacksonSerializer.getObjectMapper().readValue(resp.returnContent().asString(),
- ArrayList.class);
- return onlineNodes;
- } catch (IOException e) {
- throw new SpRuntimeException(e);
- }
+ String endpoint = generateNodeManagementOnlineNodesEndpoint();
+ return HttpUtils.get(endpoint, new TypeReference<List<NodeInfoDescription>>(){});
}
private String generateNodeManagementOnlineNodesEndpoint() {
@@ -94,7 +84,7 @@ public class OffloadingPolicyManager {
+ COLON
+ NodeConfiguration.getBackendPort()
+ SLASH
- + "streampipes-backend/api/v2/users/admin@streampipes.org/nodes/online";
+ + NODE_MANAGEMENT_ONLINE_ENDPOINT;
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategyFactory.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategyFactory.java
deleted file mode 100644
index 22068c5..0000000
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategyFactory.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.node.controller.management.offloading.model;
-
-import org.apache.streampipes.node.controller.management.offloading.model.policies.Comparator;
-import org.apache.streampipes.node.controller.management.offloading.model.policies.OffloadingPolicy;
-import org.apache.streampipes.node.controller.management.offloading.model.policies.ThresholdViolationOffloadingPolicy;
-import org.apache.streampipes.node.controller.management.offloading.model.property.CPULoadResourceProperty;
-import org.apache.streampipes.node.controller.management.offloading.model.property.FreeDiskSpaceResourceProperty;
-import org.apache.streampipes.node.controller.management.offloading.model.property.FreeMemoryResourceProperty;
-import org.apache.streampipes.node.controller.management.offloading.model.selection.CPULoadSelectionStrategy;
-import org.apache.streampipes.node.controller.management.offloading.model.selection.PrioritySelectionStrategy;
-import org.apache.streampipes.node.controller.management.offloading.model.selection.RandomSelectionStrategy;
-import org.apache.streampipes.node.controller.management.offloading.model.selection.SelectionStrategy;
-
-public class OffloadingStrategyFactory {
-
- public OffloadingStrategy getFromEnv(){
- if(System.getenv("SP_AUTO_OFFLOADING_POLICY") == null || System.getenv("SP_AUTO_OFFLOADING_POLICY").isEmpty()){
- return new OffloadingStrategy<Float>(new ThresholdViolationOffloadingPolicy<>(5, Comparator.GREATER,90f, 4),
- new CPULoadResourceProperty(), new RandomSelectionStrategy());
- }else{
- switch (System.getenv("SP_AUTO_OFFLOADING_POLICY")){
- case "CPU":
- return new OffloadingStrategy<Float>(new ThresholdViolationOffloadingPolicy<>(5, Comparator.GREATER,90f, 4),
- new CPULoadResourceProperty(), new RandomSelectionStrategy());
- case "debug":
- return new OffloadingStrategy<Float>(new ThresholdViolationOffloadingPolicy<>(5,
- Comparator.GREATER,0.5f, 1),
- new CPULoadResourceProperty(), new PrioritySelectionStrategy());
- case "memory":
- return new OffloadingStrategy<Long>(new ThresholdViolationOffloadingPolicy<>(5, Comparator.SMALLER,
- 549755813888l, 5),
- new FreeMemoryResourceProperty(), new RandomSelectionStrategy());
- case "disk space":
- return new OffloadingStrategy<Long>(new ThresholdViolationOffloadingPolicy<>(5, Comparator.SMALLER,
- 549755813888l, 5),
- new FreeDiskSpaceResourceProperty(), new RandomSelectionStrategy());
- }
- }
- return null;
- }
-
-}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategy.java
similarity index 94%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategy.java
index d359508..0c58788 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/OffloadingStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategy.java
@@ -16,11 +16,11 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model;
+package org.apache.streampipes.node.controller.management.offloading.strategies;
-import org.apache.streampipes.node.controller.management.offloading.model.policies.OffloadingPolicy;
-import org.apache.streampipes.node.controller.management.offloading.model.property.ResourceProperty;
-import org.apache.streampipes.node.controller.management.offloading.model.selection.SelectionStrategy;
+import org.apache.streampipes.node.controller.management.offloading.strategies.policies.OffloadingPolicy;
+import org.apache.streampipes.node.controller.management.offloading.strategies.property.ResourceProperty;
+import org.apache.streampipes.node.controller.management.offloading.strategies.selection.SelectionStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyFactory.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyFactory.java
new file mode 100644
index 0000000..cbe5b63
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.node.controller.management.offloading.strategies;
+
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.apache.streampipes.node.controller.management.offloading.strategies.policies.Comparator;
+import org.apache.streampipes.node.controller.management.offloading.strategies.policies.ThresholdViolationOffloadingPolicy;
+import org.apache.streampipes.node.controller.management.offloading.strategies.property.CPULoadResourceProperty;
+import org.apache.streampipes.node.controller.management.offloading.strategies.property.FreeDiskSpaceResourceProperty;
+import org.apache.streampipes.node.controller.management.offloading.strategies.property.FreeMemoryResourceProperty;
+import org.apache.streampipes.node.controller.management.offloading.strategies.selection.PrioritySelectionStrategy;
+import org.apache.streampipes.node.controller.management.offloading.strategies.selection.RandomSelectionStrategy;
+
+public class OffloadingStrategyFactory {
+
+ public OffloadingStrategy select(){
+
+ switch (NodeConfiguration.getAutoOffloadingStrategy()) {
+ case CPU:
+ return new OffloadingStrategy<Float>(
+ new ThresholdViolationOffloadingPolicy<>(5, Comparator.GREATER, 90f, 5),
+ new CPULoadResourceProperty(), new RandomSelectionStrategy());
+ case MEM:
+ return new OffloadingStrategy<Long>(
+ new ThresholdViolationOffloadingPolicy<>(5, Comparator.SMALLER,
+ 549755813888L, 5),
+ new FreeMemoryResourceProperty(), new RandomSelectionStrategy());
+ case DISK:
+ return new OffloadingStrategy<Long>(
+ new ThresholdViolationOffloadingPolicy<>(5, Comparator.SMALLER,
+ 549755813888L, 5),
+ new FreeDiskSpaceResourceProperty(), new RandomSelectionStrategy());
+ case DEBUG:
+ return new OffloadingStrategy<Float>(
+ new ThresholdViolationOffloadingPolicy<>(5,
+ Comparator.GREATER, 0.5f, 1),
+ new CPULoadResourceProperty(), new PrioritySelectionStrategy());
+ default:
+ return new OffloadingStrategy<Float>(
+ new ThresholdViolationOffloadingPolicy<>(5, Comparator.GREATER, 90f, 4),
+ new CPULoadResourceProperty(), new RandomSelectionStrategy());
+ }
+ }
+
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyType.java
similarity index 64%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyType.java
index 1022e64..8fbe677 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/OffloadingStrategyType.java
@@ -15,19 +15,27 @@
* limitations under the License.
*
*/
+package org.apache.streampipes.node.controller.management.offloading.strategies;
-package org.apache.streampipes.node.controller.management.offloading.model.selection;
+public enum OffloadingStrategyType {
+ DEFAULT("default"),
+ DEBUG("debug"),
+ CPU("cpu"),
+ MEM("mem"),
+ DISK("disk");
-import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+ String name;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
+ OffloadingStrategyType(String name) {
+ this.name = name;
+ }
-public class CPULoadSelectionStrategy implements SelectionStrategy{
- @Override
- public InvocableStreamPipesEntity selectEntity() {
- //TODO: Migrate the PE with the highest load on CPU (or possibly other resource)
- ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+ public static OffloadingStrategyType fromString(String name) {
+ for (OffloadingStrategyType o : OffloadingStrategyType.values()) {
+ if (o.name.equalsIgnoreCase(name.toLowerCase())) {
+ return o;
+ }
+ }
return null;
}
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/Comparator.java
similarity index 97%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/Comparator.java
index ec31bfc..ea2ec82 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/Comparator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.offloading.strategies.policies;
public enum Comparator {
SMALLER,
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/OffloadingPolicy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/OffloadingPolicy.java
similarity index 97%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/OffloadingPolicy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/OffloadingPolicy.java
index 429d4c9..36abcb1 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/OffloadingPolicy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/OffloadingPolicy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.offloading.strategies.policies;
public interface OffloadingPolicy<T> {
void addValue(T value);
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/ThresholdViolationOffloadingPolicy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/ThresholdViolationOffloadingPolicy.java
similarity index 99%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/ThresholdViolationOffloadingPolicy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/ThresholdViolationOffloadingPolicy.java
index e91fcda..aa30b3e 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/ThresholdViolationOffloadingPolicy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/policies/ThresholdViolationOffloadingPolicy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.offloading.strategies.policies;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/CPULoadResourceProperty.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/CPULoadResourceProperty.java
similarity index 97%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/CPULoadResourceProperty.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/CPULoadResourceProperty.java
index 3ded047..fc0c084 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/CPULoadResourceProperty.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/CPULoadResourceProperty.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.property;
+package org.apache.streampipes.node.controller.management.offloading.strategies.property;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeDiskSpaceResourceProperty.java
similarity index 97%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeDiskSpaceResourceProperty.java
index 7057ac9..ab66603 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeDiskSpaceResourceProperty.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.property;
+package org.apache.streampipes.node.controller.management.offloading.strategies.property;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeMemoryResourceProperty.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeMemoryResourceProperty.java
similarity index 97%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeMemoryResourceProperty.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeMemoryResourceProperty.java
index 6189d53..8113009 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeMemoryResourceProperty.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/FreeMemoryResourceProperty.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.property;
+package org.apache.streampipes.node.controller.management.offloading.strategies.property;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/ResourceProperty.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/ResourceProperty.java
similarity index 97%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/ResourceProperty.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/ResourceProperty.java
index d461330..cf4a0b4 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/ResourceProperty.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/property/ResourceProperty.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.property;
+package org.apache.streampipes.node.controller.management.offloading.strategies.property;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/CPULoadSelectionStrategy.java
similarity index 94%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/CPULoadSelectionStrategy.java
index 1022e64..8658ae3 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/CPULoadSelectionStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/CPULoadSelectionStrategy.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.selection;
+package org.apache.streampipes.node.controller.management.offloading.strategies.selection;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
@@ -25,7 +25,7 @@ import java.lang.management.ThreadMXBean;
public class CPULoadSelectionStrategy implements SelectionStrategy{
@Override
- public InvocableStreamPipesEntity selectEntity() {
+ public InvocableStreamPipesEntity select() {
//TODO: Migrate the PE with the highest load on CPU (or possibly other resource)
ThreadMXBean bean = ManagementFactory.getThreadMXBean();
return null;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/PrioritySelectionStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/PrioritySelectionStrategy.java
similarity index 61%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/PrioritySelectionStrategy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/PrioritySelectionStrategy.java
index 000a93b..44ad591 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/PrioritySelectionStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/PrioritySelectionStrategy.java
@@ -16,11 +16,11 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.selection;
+package org.apache.streampipes.node.controller.management.offloading.strategies.selection;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.model.node.NodeInfoDescription;
-import org.apache.streampipes.node.controller.management.node.NodeManager;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
import org.apache.streampipes.node.controller.management.offloading.OffloadingPolicyManager;
import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
@@ -32,33 +32,43 @@ import java.util.stream.Collectors;
public class PrioritySelectionStrategy implements SelectionStrategy{
@Override
- public InvocableStreamPipesEntity selectEntity() {
- List<InvocableStreamPipesEntity> instances = RunningInvocableInstances.INSTANCE.getAll();
+ public InvocableStreamPipesEntity select() {
+ List<InvocableStreamPipesEntity> runningPipelineElements = RunningInvocableInstances.INSTANCE.getAll();
+
//List all other nodes that are online
+ // TODO: this involves request to central backend
List<NodeInfoDescription> onlineNodes = getNodeInfos().stream().filter(desc ->
- !desc.getNodeControllerId().equals(NodeManager.getInstance().retrieveNodeInfoDescription().getNodeControllerId()))
+ !desc.getNodeControllerId().equals(NodeConfiguration.getNodeControllerId()))
.collect(Collectors.toList());
- List<InvocableStreamPipesEntity> candidateEntities =
- instances.stream()
+ List<InvocableStreamPipesEntity> candidatesForOffloading = runningPipelineElements.stream()
.filter(InvocableStreamPipesEntity::isPreemption)
- .filter(entity -> checkIfSupported(entity, onlineNodes)).collect(Collectors.toList());
- if(candidateEntities.isEmpty()) return null;
- return lowestPriority(candidateEntities);
+ .filter(entity -> verifyIfSupportedOnOtherNodes(entity, onlineNodes))
+ .collect(Collectors.toList());
+
+ if (candidatesForOffloading.isEmpty()) {
+ // TODO what to do when null?
+ return null;
+ } else {
+ return selectPipelineElementWithLowestPriorityScore(candidatesForOffloading);
+ }
}
private List<NodeInfoDescription> getNodeInfos(){
return OffloadingPolicyManager.getInstance().getOnlineNodes();
}
- private InvocableStreamPipesEntity lowestPriority(List<InvocableStreamPipesEntity> entities){
- List<InvocableStreamPipesEntity> sortedEntities = entities.stream()
+ private InvocableStreamPipesEntity selectPipelineElementWithLowestPriorityScore(List<InvocableStreamPipesEntity> entities){
+ // sort pipeline element priorityScores ascending
+ List<InvocableStreamPipesEntity> sortedPipelineElements = entities.stream()
.sorted(Comparator.comparingInt(InvocableStreamPipesEntity::getPriorityScore))
.collect(Collectors.toList());
- return sortedEntities.get(0);
+
+ // return first element with lowest score
+ return sortedPipelineElements.get(0);
}
- private boolean checkIfSupported(InvocableStreamPipesEntity entity, List<NodeInfoDescription> nodeInfos){
+ private boolean verifyIfSupportedOnOtherNodes(InvocableStreamPipesEntity entity, List<NodeInfoDescription> nodeInfos){
List<NodeInfoDescription> candidateNodes = new ArrayList<>();
for(NodeInfoDescription nodeInfo : nodeInfos){
if (nodeInfo.getSupportedElements().contains(entity.getAppId()))
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/RandomSelectionStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/RandomSelectionStrategy.java
similarity index 94%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/RandomSelectionStrategy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/RandomSelectionStrategy.java
index 6f214df..a37289c 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/RandomSelectionStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/RandomSelectionStrategy.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.selection;
+package org.apache.streampipes.node.controller.management.offloading.strategies.selection;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
@@ -27,7 +27,7 @@ import java.util.Random;
public class RandomSelectionStrategy implements SelectionStrategy{
@Override
- public InvocableStreamPipesEntity selectEntity() {
+ public InvocableStreamPipesEntity select() {
List<InvocableStreamPipesEntity> instances = RunningInvocableInstances.INSTANCE.getAll();
if(instances.size() == 0)
return null;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/SelectionStrategy.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/SelectionStrategy.java
similarity index 93%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/SelectionStrategy.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/SelectionStrategy.java
index 23f1931..1964198 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/selection/SelectionStrategy.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/strategies/selection/SelectionStrategy.java
@@ -16,10 +16,10 @@
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.selection;
+package org.apache.streampipes.node.controller.management.offloading.strategies.selection;
import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
public interface SelectionStrategy {
- InvocableStreamPipesEntity selectEntity();
+ InvocableStreamPipesEntity select();
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/DockerEngineManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/DockerEngineManager.java
index af9b163..d3318b4 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/DockerEngineManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/DockerEngineManager.java
@@ -29,6 +29,7 @@ import org.apache.streampipes.node.controller.management.orchestrator.docker.Doc
import org.apache.streampipes.node.controller.management.orchestrator.docker.model.ContainerStatus;
import org.apache.streampipes.node.controller.management.orchestrator.docker.utils.DockerUtils;
import org.apache.streampipes.node.controller.management.orchestrator.status.ContainerDeploymentStatus;
+import org.apache.streampipes.node.controller.management.orchestrator.storage.RunningContainerInstances;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,7 +62,8 @@ public class DockerEngineManager implements IContainerEngine {
ContainerDeploymentStatus status = deploy(container);
if (status.getStatus() == ContainerStatus.DEPLOYED) {
- NodeManager.getInstance().registerContainer(status.getContainer());
+ NodeManager.getInstance().registerContainerWhenDeployed(status.getContainer());
+ RunningContainerInstances.INSTANCE.add(status.getContainerId(), container);
}
});
}
@@ -69,30 +71,31 @@ public class DockerEngineManager implements IContainerEngine {
@Override
public ContainerDeploymentStatus deploy(DockerContainer container) {
- LOG.info("Pull image and deploy pipeline element container {}", container.getImageTag());
- Optional<Container> containerOptional = docker.getContainer(container.getContainerName());
- if (!containerOptional.isPresent()) {
- LOG.info("Deploy pipeline element container \"" + container.getImageTag() + "\"");
- String containerId = "";
+ Optional<Container> runningContainer = docker.getContainer(container.getContainerName());
+
+ if (!runningContainer.isPresent()) {
+ String containerId;
try {
+ LOG.info("Deploy container for image={}", container.getImageTag());
containerId = deployPipelineElementContainer(container);
+
} catch (Exception e) {
- LOG.error("Could not deploy pipeline element. {}", e.toString());
- return generateContainerStatus(containerId, container, ContainerStatus.FAILED);
+ LOG.error("Failed to deploy container", e);
+ return generateContainerStatus(null, container, ContainerStatus.FAILED);
}
- LOG.info("Finished pull image and deployed pipeline element container");
+ LOG.debug("Finished pulling image and deployed container");
return generateContainerStatus(containerId, container, ContainerStatus.DEPLOYED);
}
- LOG.info("Container already running {}", container.getContainerName());
- return generateContainerStatus(containerOptional.get().id(), container, ContainerStatus.RUNNING);
+ LOG.debug("Container already running {}", container.getContainerName());
+ return generateContainerStatus(runningContainer.get().id(), container, ContainerStatus.RUNNING);
}
@Override
public ContainerDeploymentStatus remove(DockerContainer container) {
LOG.info("Remove pipeline element container: {}", container.getImageTag());
- Optional<com.spotify.docker.client.messages.Container> containerOptional = docker.getContainer(container.getContainerName());
- if(containerOptional.isPresent()) {
+ Optional<Container> runningContainer = docker.getContainer(container.getContainerName());
+ if(runningContainer.isPresent()) {
docker.forceRemove(container.getContainerName());
@@ -100,10 +103,10 @@ public class DockerEngineManager implements IContainerEngine {
ConsulUtil.deregisterService(container.getServiceId());
ConsulUtil.deleteConfig(container.getServiceId());
- return generateContainerStatus(containerOptional.get().id(), container, ContainerStatus.REMOVED);
+ return generateContainerStatus(runningContainer.get().id(), container, ContainerStatus.REMOVED);
}
- return generateContainerStatus(containerOptional.get().id(), container, ContainerStatus.UNKNOWN);
+ return generateContainerStatus(runningContainer.get().id(), container, ContainerStatus.UNKNOWN);
}
@Override
@@ -133,7 +136,7 @@ public class DockerEngineManager implements IContainerEngine {
return deployPipelineElementContainer(container, true);
}
- private String deployPipelineElementContainer(DockerContainer container, boolean pullImage) throws Exception {
+ private String deployPipelineElementContainer(DockerContainer container, boolean pullImage) throws NotFoundException {
if (pullImage) {
try {
docker.pullImage(container.getImageTag(), false);
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/IContainerEngine.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/IContainerEngine.java
index 18a09f2..3f006c1 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/IContainerEngine.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/IContainerEngine.java
@@ -21,13 +21,8 @@ import org.apache.streampipes.model.node.container.DockerContainer;
import org.apache.streampipes.node.controller.management.orchestrator.status.ContainerDeploymentStatus;
public interface IContainerEngine {
-
void init();
-
ContainerDeploymentStatus deploy(DockerContainer p);
-
ContainerDeploymentStatus remove(DockerContainer p);
-
String list();
-
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/AbstractStreamPipesDockerContainer.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/AbstractStreamPipesDockerContainer.java
index e5fb8fa..9b76e35 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/AbstractStreamPipesDockerContainer.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/AbstractStreamPipesDockerContainer.java
@@ -21,6 +21,7 @@ import org.apache.streampipes.model.node.container.DockerContainer;
import org.apache.streampipes.node.controller.config.EnvConfigParam;
import org.apache.streampipes.node.controller.config.NodeConfiguration;
import org.apache.streampipes.node.controller.management.node.NodeManager;
+import org.apache.streampipes.node.controller.utils.VersionUtils;
import java.util.*;
@@ -31,7 +32,7 @@ public abstract class AbstractStreamPipesDockerContainer {
public static String getStreamPipesVersion() {
return !NodeConfiguration.getStreampipesVersion().isEmpty() ?
- NodeConfiguration.getStreampipesVersion() : NodeManager.getInstance().getStreamPipesVersion();
+ NodeConfiguration.getStreampipesVersion() : VersionUtils.getStreamPipesVersion();
}
public List<String> generateStreamPipesNodeEnvs() {
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/DockerContainerDeclarerSingleton.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/DockerContainerDeclarerSingleton.java
index b113cd4..7bd526c 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/DockerContainerDeclarerSingleton.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/DockerContainerDeclarerSingleton.java
@@ -20,7 +20,10 @@ package org.apache.streampipes.node.controller.management.orchestrator.docker;
import org.apache.streampipes.model.node.container.DockerContainer;
import org.apache.streampipes.node.controller.config.NodeConfiguration;
-import org.apache.streampipes.node.controller.container.StreamPipesDockerServiceID;
+import org.apache.streampipes.node.controller.container.DockerKafkaContainer;
+import org.apache.streampipes.node.controller.container.DockerMosquittoContainer;
+import org.apache.streampipes.node.controller.container.DockerZookeeperContainer;
+import org.apache.streampipes.node.controller.management.node.NodeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,12 +65,12 @@ public class DockerContainerDeclarerSingleton {
public List<DockerContainer> getAutoDeploymentDockerContainers() {
if ("kafka".equals(NodeConfiguration.getNodeBrokerProtocol())) {
- remove(StreamPipesDockerServiceID.SP_SVC_MOSQUITTO_ID);
+ remove(DockerMosquittoContainer.SP_SVC_MOSQUITTO_ID);
} else {
- remove(StreamPipesDockerServiceID.SP_SVC_KAFKA_ID, StreamPipesDockerServiceID.SP_SVC_ZOOKEEPER_ID);
+ remove(DockerKafkaContainer.SP_SVC_KAFKA_ID, DockerZookeeperContainer.SP_SVC_ZOOKEEPER_ID);
}
- LinkedHashMap<String, DockerContainer> sorted = sort();
- return new ArrayList<>(sorted.values());
+ LinkedHashMap<String, DockerContainer> topologicallySortedContainers = sort();
+ return new ArrayList<>(topologicallySortedContainers.values());
}
// Helpers
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/model/DockerInfo.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/model/DockerInfo.java
index d362395..42fb369 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/model/DockerInfo.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/orchestrator/docker/model/DockerInfo.java
@@ -31,7 +31,8 @@ public class DockerInfo {
public DockerInfo(){}
- public DockerInfo(String serverVersion, String apiVersion, Long memTotal, int cpus, String os, String osType, String kernelVersion, String arch, boolean hasNvidiaRuntime) {
+ public DockerInfo(String serverVersion, String apiVersion, Long memTotal, int cpus, String os, String osType,
+ String kernelVersion, String arch, boolean hasNvidiaRuntime) {
this.serverVersion = serverVersion;
this.apiVersion = apiVersion;
this.memTotal = memTotal;
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/IPipelineElementLifeCycle.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/IPipelineElementLifeCycle.java
index f8d35e9..3daa469 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/IPipelineElementLifeCycle.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/IPipelineElementLifeCycle.java
@@ -28,10 +28,12 @@ public interface IPipelineElementLifeCycle {
Response invoke(InvocableStreamPipesEntity graph);
- Response detach(String runningInstanceId);
+ Response detach(InvocableStreamPipesEntity graph, String runningInstanceId);
- Response adapt(InvocableStreamPipesEntity graph, PipelineElementReconfigurationEntity reconfigurationEvent);
+ Response reconfigure(InvocableStreamPipesEntity graph, PipelineElementReconfigurationEntity reconfigurationEvent);
- void unregister();
+ Response offload(InvocableStreamPipesEntity graph);
+
+ void deregister();
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/InvocableElementManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/InvocableElementManager.java
deleted file mode 100644
index 883c36c..0000000
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/InvocableElementManager.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.streampipes.node.controller.management.pe;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.gson.JsonSyntaxException;
-import org.apache.http.client.fluent.Request;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.streampipes.commons.exceptions.SpRuntimeException;
-import org.apache.streampipes.container.model.node.InvocableRegistration;
-import org.apache.streampipes.messaging.EventProducer;
-import org.apache.streampipes.messaging.jms.ActiveMQPublisher;
-import org.apache.streampipes.messaging.kafka.SpKafkaProducer;
-import org.apache.streampipes.messaging.mqtt.MqttPublisher;
-import org.apache.streampipes.model.Response;
-import org.apache.streampipes.model.base.ConsumableStreamPipesEntity;
-import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
-import org.apache.streampipes.model.base.NamedStreamPipesEntity;
-import org.apache.streampipes.model.grounding.*;
-import org.apache.streampipes.model.node.NodeInfoDescription;
-import org.apache.streampipes.model.pipeline.PipelineElementReconfigurationEntity;
-import org.apache.streampipes.model.resource.Hardware;
-import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
-import org.apache.streampipes.model.staticproperty.StaticProperty;
-import org.apache.streampipes.node.controller.config.NodeConfiguration;
-import org.apache.streampipes.node.controller.management.node.NodeManager;
-import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
-import org.apache.streampipes.node.controller.management.resource.utils.ResourceChecker;
-import org.apache.streampipes.serializers.json.JacksonSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.Socket;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.stream.Collectors;
-
-public class InvocableElementManager implements IPipelineElementLifeCycle {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(InvocableElementManager.class.getCanonicalName());
-
- private static final String HTTP_PROTOCOL = "http://";
- private static final String COLON = ":";
- private static final String SLASH = "/";
- private static final String DOT = ".";
- private static final String SP_URL = "SP_URL";
- private static final String CONSUL_LOCATION = "CONSUL_LOCATION";
- private static final String CONSUL_REGISTRATION_ROUTE = "v1/agent/service/register";
- private static final String RECONFIGURATION_TOPIC = "org.apache.streampipes.control.event.reconfigure";
- private static final int CONSUL_DEFAULT_PORT = 8500;
- private static final Integer CONNECT_TIMEOUT_MS = 10000;
- private static final long RETRY_INTERVAL_MS = 5000;
- private static InvocableElementManager instance = null;
-
- private InvocableElementManager() {}
-
- public static InvocableElementManager getInstance() {
- if (instance == null) {
- synchronized (InvocableElementManager.class) {
- if (instance == null)
- instance = new InvocableElementManager();
- }
- }
- return instance;
- }
-
- @Override
- public void register(InvocableRegistration registration) {
- registerAtConsul(registration);
- updateAndSyncNodeInfoDescription(registration);
- invokePipelineElementsOnSystemRebootOrRestart();
- LOG.info("Successfully registered pipeline element container");
- }
-
- @Override
- public Response invoke(InvocableStreamPipesEntity graph) {
-
- Response response = new Response();
- String endpoint = graph.getBelongsTo();
- String body = toJson(graph);
- LOG.info("Trying to invoke pipeline element: {}", endpoint);
-
- boolean connected = false;
- while (!connected) {
-
- connected = post(endpoint, body, response);
-
- if (!connected) {
- LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
- try {
- Thread.sleep(RETRY_INTERVAL_MS);
- } catch (InterruptedException e) {
- throw new RuntimeException("Failed to invoke pipeline element: " + endpoint);
- }
- }
- }
- LOG.info("Successfully invoked pipeline element {}", endpoint);
- return response;
- }
-
- @Override
- public Response detach(String endpoint) {
- LOG.info("Detach pipeline element: {}", endpoint);
- try {
- org.apache.http.client.fluent.Response httpResp = Request
- .Delete(endpoint)
- .connectTimeout(CONNECT_TIMEOUT_MS)
- .execute();
- return handleResponse(httpResp);
- } catch (Exception e) {
- LOG.error(e.getMessage());
- }
- throw new IllegalArgumentException("Failed to detach pipeline element: " + endpoint);
- }
-
- @Override
- public void unregister(){
- // TODO: unregister element from Consul and
- setSupportedPipelineElements(Collections.emptyList());
- try {
- String url = generateNodeManagementUpdateEndpoint();
- String desc = toJson(getNodeInfoDescription());
- Request.Put(url)
- .bodyString(desc, ContentType.APPLICATION_JSON)
- .execute();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Response adapt(InvocableStreamPipesEntity graph, PipelineElementReconfigurationEntity reconfigurationEntity) {
-
- Response response = new Response();
- response.setElementId(graph.getElementId());
- response.setSuccess(false);
-
- EventProducer pub = getReconfigurationEventProducerFromInvocable(graph);
-
- byte [] reconfigurationEvent = reconfigurationToByteArray(reconfigurationEntity);
- pub.publish(reconfigurationEvent);
- pub.disconnect();
-
- adaptPipelineDescription(graph, reconfigurationEntity);
-
- response.setSuccess(true);
- return response;
- }
-
- public Response postOffloadRequest(InvocableStreamPipesEntity instanceToOffload){
- Response resp = new Response();
- post(generatePipelineManagementOffloadEndpoint(), toJson(instanceToOffload), resp);
- return resp;
- }
-
- private void adaptPipelineDescription(InvocableStreamPipesEntity graph, PipelineElementReconfigurationEntity reconfigurationEntity){
- List<StaticProperty> staticProperties = new ArrayList<>();
- graph.getStaticProperties().forEach(sp -> {
- int ind = graph.getStaticProperties().indexOf(sp);
- staticProperties.add(ind, sp);
- reconfigurationEntity.getReconfiguredStaticProperties().forEach(rp -> {
- if(sp.getInternalName().equals(rp.getInternalName())){
- staticProperties.remove(ind);
- staticProperties.add(ind, rp);
- }
- });
- });
- graph.setStaticProperties(staticProperties);
- RunningInvocableInstances.INSTANCE.remove(reconfigurationEntity.getDeploymentRunningInstanceId());
- RunningInvocableInstances.INSTANCE.add(reconfigurationEntity.getDeploymentRunningInstanceId(), graph);
- }
-
- private boolean post(String endpoint, String body, Response response) {
- try {
- org.apache.http.client.fluent.Response httpResp = Request
- .Post(endpoint)
- .bodyString(body, ContentType.APPLICATION_JSON)
- .connectTimeout(CONNECT_TIMEOUT_MS)
- .execute();
- Response resp = handleResponse(httpResp);
-
- response.setElementId(resp.getElementId());
- response.setSuccess(resp.isSuccess());
- response.setOptionalMessage(resp.getOptionalMessage());
-
- return true;
- } catch (IOException e) {
- LOG.error("Could not invoke pipeline element - is the pipeline element container ready?");
- }
- return false;
- }
-
- public void invokePipelineElementsOnSystemRebootOrRestart() {
- getAllInvocables()
- .forEach(graph -> {
- Response status = InvocableElementManager.getInstance().invoke(graph);
- if (status.isSuccess()) {
- if (status.getOptionalMessage().isEmpty()) {
- LOG.info("Pipeline element successfully restarted {}", status.getElementId());
- } else {
- LOG.info("Pipeline element already running {}", status.getElementId());
- }
- } else {
- LOG.info("Pipeline element could not be restarted - are the pipeline element containers " +
- "running? {}", status.getElementId());
- }
- });
- }
-
- private EventProducer getReconfigurationEventProducerFromInvocable(InvocableStreamPipesEntity graph) {
- TransportProtocol tp = getReconfigurationTransportProtocol(graph);
- EventProducer pub;
- if(tp instanceof KafkaTransportProtocol){
- pub = new SpKafkaProducer();
- pub.connect(tp);
- } else if (tp instanceof JmsTransportProtocol){
- pub = new ActiveMQPublisher();
- pub.connect(tp);
- } else{
- pub = new MqttPublisher();
- pub.connect(tp);
- }
- return pub;
- }
-
- private byte[] reconfigurationToByteArray(PipelineElementReconfigurationEntity entity) {
- Map<String, String> reconfigurationEventMap = new HashMap<>();
- entity.getReconfiguredStaticProperties().forEach(staticProperty -> {
- if (staticProperty instanceof FreeTextStaticProperty) {
- reconfigurationEventMap.put(staticProperty.getInternalName(),
- ((FreeTextStaticProperty) staticProperty).getValue());
- }
- });
- return toJson(reconfigurationEventMap).getBytes(StandardCharsets.UTF_8);
- }
-
- private void updateAndSyncNodeInfoDescription(InvocableRegistration registration) {
- setSupportedPipelineElements(getSupportedEntities(registration).stream()
- .map(NamedStreamPipesEntity::getAppId).collect(Collectors.toList()));
- try {
- String url = generateNodeManagementUpdateEndpoint();
- String desc = toJson(getNodeInfoDescription());
- Request.Put(url)
- .bodyString(desc, ContentType.APPLICATION_JSON)
- .execute();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private List<ConsumableStreamPipesEntity> getSupportedEntities(InvocableRegistration registration){
- //Check if the node supports the entity (atm only hardware requirements; could be expanded to include
- // software requirements)
- ResourceChecker resourceChecker = new ResourceChecker(getNodeInfoDescription());
- return registration.getSupportedPipelineElements().stream()
- .filter(resourceChecker::checkResources).collect(Collectors.toList());
- }
-
- private List<InvocableStreamPipesEntity> getAllInvocables() {
- return RunningInvocableInstances.INSTANCE.getAll();
- }
-
- private NodeInfoDescription getNodeInfoDescription() {
- return NodeManager.getInstance().getNode();
- }
-
- private void setSupportedPipelineElements(List<String> supportedPipelineElements) {
- NodeManager.getInstance()
- .getNode()
- .setSupportedElements(supportedPipelineElements);
- }
-
- private TransportProtocol getReconfigurationTransportProtocol(InvocableStreamPipesEntity graph) {
- TransportProtocol tp = graph.getInputStreams().get(0).getEventGrounding().getTransportProtocol();
- tp.setTopicDefinition(makeReconfigurationTopic(graph.getDeploymentRunningInstanceId()));
- return tp;
- }
-
- private TopicDefinition makeReconfigurationTopic(String runningInstanceId) {
- return new SimpleTopicDefinition( RECONFIGURATION_TOPIC + DOT + runningInstanceId);
- }
-
- private String generateNodeManagementUpdateEndpoint() {
- return HTTP_PROTOCOL
- + NodeConfiguration.getBackendHost()
- + COLON
- + NodeConfiguration.getBackendPort()
- + SLASH
- + "streampipes-backend/api/v2/users/admin@streampipes.org/nodes"
- + SLASH
- + NodeConfiguration.getNodeControllerId();
- }
-
- private String generatePipelineManagementOffloadEndpoint() {
- return HTTP_PROTOCOL
- + NodeConfiguration.getBackendHost()
- + COLON
- + NodeConfiguration.getBackendPort()
- + SLASH
- + "streampipes-backend/api/v2/users/admin@streampipes.org/pipelines/offload";
- }
-
- private void registerAtConsul(InvocableRegistration registration) {
- try {
- String endpoint = consulURL().toString() + SLASH + CONSUL_REGISTRATION_ROUTE;
- Request.Put(endpoint)
- .addHeader("accept", "application/json")
- .body(new StringEntity(toJson(registration.getConsulServiceRegistrationBody())))
- .execute();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static boolean isAvailable(String host, int port) {
- try {
- InetSocketAddress sa = new InetSocketAddress(host, port);
- Socket ss = new Socket();
- ss.connect(sa, 1000);
- ss.close();
- } catch(Exception e) {
- return false;
- }
- return true;
- }
-
- private Response handleResponse(org.apache.http.client.fluent.Response httpResp) throws JsonSyntaxException,
- IOException {
- String resp = httpResp.returnContent().asString();
- return JacksonSerializer
- .getObjectMapper()
- .readValue(resp, Response.class);
- }
-
- private static URL consulURL() {
- Map<String, String> env = System.getenv();
- URL url = null;
-
- if (env.containsKey(SP_URL)) {
- try {
- URL coreServerUrl = new URL(env.get(SP_URL));
- url = new URL("http", coreServerUrl.getHost(), CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- } else if (env.containsKey(CONSUL_LOCATION)) {
- try {
- url = new URL("http", env.get(CONSUL_LOCATION), CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- } else {
- try {
- url = new URL("http", "localhost", CONSUL_DEFAULT_PORT, "");
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
- return url;
- }
-
- private <T> String toJson(T element) {
- try {
- return JacksonSerializer.getObjectMapper().writeValueAsString(element);
- } catch (JsonProcessingException e) {
- throw new SpRuntimeException("Could not serialize object: " + element, e);
- }
- }
-
-}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementLifeCycleState.java
similarity index 81%
copy from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
copy to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementLifeCycleState.java
index ec31bfc..bad4511 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementLifeCycleState.java
@@ -15,9 +15,13 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.management.pe;
-public enum Comparator {
- SMALLER,
- GREATER
+public enum PipelineElementLifeCycleState {
+ REGISTER,
+ INVOKE,
+ DETACH,
+ RECONFIGURE,
+ OFFLOAD,
+ DEREGISTER
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementManager.java
new file mode 100644
index 0000000..7927814
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/PipelineElementManager.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe;
+
+import org.apache.streampipes.container.model.node.InvocableRegistration;
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.model.node.NodeInfoDescription;
+import org.apache.streampipes.model.pipeline.PipelineElementReconfigurationEntity;
+import org.apache.streampipes.node.controller.management.node.NodeManager;
+import org.apache.streampipes.node.controller.management.pe.handler.*;
+import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class PipelineElementManager implements IPipelineElementLifeCycle {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PipelineElementManager.class.getCanonicalName());
+
+ private static PipelineElementManager instance = null;
+
+ private PipelineElementManager() {}
+
+ public static PipelineElementManager getInstance() {
+ if (instance == null) {
+ synchronized (PipelineElementManager.class) {
+ if (instance == null)
+ instance = new PipelineElementManager();
+ }
+ }
+ return instance;
+ }
+
+ @Override
+ public void register(InvocableRegistration registration) {
+ boolean consulRegistration = new ConsulInteractionHandler(registration, PipelineElementLifeCycleState.REGISTER).handle();
+ if (consulRegistration) {
+ LOG.info("Successfully registered pipeline element at consul");
+ }
+
+ NodeInfoDescription node = getNodeInfoDescription();
+ boolean nodeRegistration =
+ new PipelineElementNodeInteractionHandler(node, registration, PipelineElementLifeCycleState.REGISTER).handle();
+ if (nodeRegistration) {
+ LOG.info("Successfully registered and synchronized supported pipeline elements");
+ }
+
+ List<InvocableStreamPipesEntity> invokedPipelineElements = getAllInvocables();
+ boolean invokedPipelineElementsOnReboot = new PipelineElementRestartOnReboot(invokedPipelineElements).handle();
+ if (invokedPipelineElementsOnReboot) {
+ LOG.info("Successfully restarted pipeline elements");
+ }
+ }
+
+ @Override
+ public Response invoke(InvocableStreamPipesEntity graph) {
+ return new PipelineElementInteractionHandler(graph, PipelineElementLifeCycleState.INVOKE).handle();
+ }
+
+ @Override
+ public Response detach(InvocableStreamPipesEntity graph, String runningInstanceId) {
+ return new PipelineElementInteractionHandler(graph, PipelineElementLifeCycleState.DETACH, runningInstanceId).handle();
+ }
+
+ @Override
+ public void deregister(){
+ // TODO: unregister element from Consul
+
+ NodeInfoDescription node = getNodeInfoDescription();
+ boolean nodeRegistration = new PipelineElementNodeInteractionHandler(node, PipelineElementLifeCycleState.DEREGISTER).handle();
+ if (nodeRegistration) {
+ LOG.info("Successfully deregistered and synchronized pipeline elements");
+ }
+ }
+
+ @Override
+ public Response reconfigure(InvocableStreamPipesEntity graph,
+ PipelineElementReconfigurationEntity reconfigurationEntity) {
+ return new PipelineElementReconfigurationHandler(graph, reconfigurationEntity).handle();
+ }
+
+ @Override
+ public Response offload(InvocableStreamPipesEntity graph) {
+ return new PipelineElementOffloadingHandler(graph).handle();
+ }
+
+
+ private List<InvocableStreamPipesEntity> getAllInvocables() {
+ return RunningInvocableInstances.INSTANCE.getAll();
+ }
+
+ private NodeInfoDescription getNodeInfoDescription() {
+ return NodeManager.getInstance().getNodeInfoDescription();
+ }
+
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/ConsulInteractionHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/ConsulInteractionHandler.java
new file mode 100644
index 0000000..67a4fc1
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/ConsulInteractionHandler.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+import org.apache.streampipes.container.model.node.InvocableRegistration;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementLifeCycleState;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+
+public class ConsulInteractionHandler implements IHandler<Boolean> {
+
+ private static final String HTTP_PROTOCOL = "http";
+ private static final String SLASH = "/";
+ private static final String SP_URL = "SP_URL";
+ private static final String CONSUL_LOCATION = "CONSUL_LOCATION";
+ private static final String CONSUL_REGISTRATION_ROUTE = "v1/agent/service/register";
+ private static final String CONSUL_DEREGISTRATION_ROUTE = "v1/agent/service/deregister";
+ private static final int CONSUL_DEFAULT_PORT = 8500;
+
+ private final InvocableRegistration registration;
+ private final PipelineElementLifeCycleState type;
+
+ public ConsulInteractionHandler(InvocableRegistration registration, PipelineElementLifeCycleState type) {
+ this.registration = registration;
+ this.type = type;
+ }
+
+ @Override
+ public Boolean handle() {
+ String endpoint = consulURL().toString();
+ switch(type) {
+ case REGISTER:
+ endpoint = endpoint + SLASH + CONSUL_REGISTRATION_ROUTE;
+ return HttpUtils.put(endpoint, registration.getConsulServiceRegistrationBody());
+ case DEREGISTER:
+ String svdId = registration.getConsulServiceRegistrationBody().getID();
+ endpoint = endpoint + SLASH + CONSUL_DEREGISTRATION_ROUTE + SLASH + svdId;
+ return HttpUtils.put(endpoint);
+ default:
+ return false;
+ }
+ }
+
+ private static URL consulURL() {
+ Map<String, String> env = System.getenv();
+ URL url = null;
+
+ if (env.containsKey(SP_URL)) {
+ try {
+ URL coreServerUrl = new URL(env.get(SP_URL));
+ url = new URL(HTTP_PROTOCOL, coreServerUrl.getHost(), CONSUL_DEFAULT_PORT, "");
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ } else if (env.containsKey(CONSUL_LOCATION)) {
+ try {
+ url = new URL(HTTP_PROTOCOL, env.get(CONSUL_LOCATION), CONSUL_DEFAULT_PORT, "");
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ } else {
+ try {
+ url = new URL(HTTP_PROTOCOL, "localhost", CONSUL_DEFAULT_PORT, "");
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ }
+ return url;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementInteractionHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementInteractionHandler.java
new file mode 100644
index 0000000..b0a5f72
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementInteractionHandler.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementLifeCycleState;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PipelineElementInteractionHandler implements IHandler<Response> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PipelineElementInteractionHandler.class.getCanonicalName());
+ private static final String SLASH = "/";
+ private static final long RETRY_INTERVAL_MS = 5000;
+
+ private final InvocableStreamPipesEntity graph;
+ private final PipelineElementLifeCycleState type;
+ private final String runningInstanceId;
+
+ public PipelineElementInteractionHandler(InvocableStreamPipesEntity graph, PipelineElementLifeCycleState type) {
+ this(graph, type, "");
+ }
+
+ public PipelineElementInteractionHandler(InvocableStreamPipesEntity graph, PipelineElementLifeCycleState type, String runningInstanceId) {
+ this.graph = graph;
+ this.type = type;
+ this.runningInstanceId = runningInstanceId;
+ }
+
+ @Override
+ public Response handle() {
+ switch(type) {
+ case INVOKE:
+ return invoke();
+ case DETACH:
+ return detach();
+ default:
+ throw new SpRuntimeException("Life cycle step not supported" + type);
+ }
+ }
+
+ private Response invoke() {
+ Response response = new Response();
+ String url = graph.getBelongsTo();
+
+ LOG.info("Trying to invoke pipeline element: {}", url);
+ boolean connected = false;
+ while (!connected) {
+
+ response = HttpUtils.post(url, graph);
+ connected = response.isSuccess();
+
+ if (!connected) {
+ LOG.info("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Failed to invoke pipeline element: " + url);
+ }
+ }
+ }
+ LOG.info("Successfully invoked pipeline element {}", url);
+ return response;
+ }
+
+ private Response detach() {
+ Response response = new Response();
+ String url = graph.getBelongsTo() + SLASH + runningInstanceId;
+
+ LOG.info("Trying to detach pipeline element: {}", url);
+ boolean connected = false;
+ while (!connected) {
+
+ response = HttpUtils.delete(url);
+ connected = response.isSuccess();
+
+ if (!connected) {
+ LOG.debug("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Failed to detach pipeline element: " + url, e);
+ }
+ }
+ }
+ LOG.info("Successfully detached pipeline element {}", url);
+ return response;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementNodeInteractionHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementNodeInteractionHandler.java
new file mode 100644
index 0000000..693e010
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementNodeInteractionHandler.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+import org.apache.streampipes.commons.exceptions.SpRuntimeException;
+import org.apache.streampipes.container.model.node.InvocableRegistration;
+import org.apache.streampipes.model.base.ConsumableStreamPipesEntity;
+import org.apache.streampipes.model.base.NamedStreamPipesEntity;
+import org.apache.streampipes.model.node.NodeInfoDescription;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.node.NodeManager;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementLifeCycleState;
+import org.apache.streampipes.node.controller.management.pe.validator.RequirementsResourceValidator;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class PipelineElementNodeInteractionHandler implements IHandler<Boolean> {
+
+ private static final String HTTP_PROTOCOL = "http://";
+ private static final String COLON = ":";
+ private static final String SLASH = "/";
+ private static final String BASE_ROUTE = "streampipes-backend/api/v2/users/admin@streampipes.org/nodes";
+
+ private final NodeInfoDescription node;
+ private final InvocableRegistration registration;
+ private final PipelineElementLifeCycleState type;
+
+ public PipelineElementNodeInteractionHandler(NodeInfoDescription node, PipelineElementLifeCycleState type) {
+ this(node, null, type);
+ }
+
+ public PipelineElementNodeInteractionHandler(NodeInfoDescription node, InvocableRegistration registration,
+ PipelineElementLifeCycleState type) {
+ this.node = node;
+ this.registration = registration;
+ this.type = type;
+ }
+
+ @Override
+ public Boolean handle() {
+ switch(type) {
+ case REGISTER:
+ return register();
+ case DEREGISTER:
+ return deregister();
+ default:
+ throw new SpRuntimeException("Life cycle step not supported" + type);
+ }
+ }
+
+ private boolean register() {
+ List<String> supportedPipelineElementAppIds = validate();
+
+ NodeManager.getInstance()
+ .getNodeInfoDescription()
+ .setSupportedElements(supportedPipelineElementAppIds);
+
+ String url = generateNodeManagementUpdateEndpoint();
+ return HttpUtils.put(url, node);
+ }
+
+ private boolean deregister() {
+ NodeManager.getInstance()
+ .getNodeInfoDescription()
+ .setSupportedElements(Collections.emptyList());
+
+ String url = generateNodeManagementUpdateEndpoint();
+ return HttpUtils.put(url, node);
+ }
+
+
+ private List<String> validate(){
+ //Check if the node supports the entity (atm only hardware requirements; could be expanded to include
+ // software requirements)
+ List<ConsumableStreamPipesEntity> allPipelineElements = registration.getSupportedPipelineElements();
+ List<ConsumableStreamPipesEntity> validPipelineElements =
+ new RequirementsResourceValidator(node, allPipelineElements).validate();
+
+ return validPipelineElements.stream()
+ .map(NamedStreamPipesEntity::getAppId)
+ .collect(Collectors.toList());
+ }
+
+ private String generateNodeManagementUpdateEndpoint() {
+ return HTTP_PROTOCOL
+ + NodeConfiguration.getBackendHost()
+ + COLON
+ + NodeConfiguration.getBackendPort()
+ + SLASH
+ + BASE_ROUTE
+ + SLASH
+ + NodeConfiguration.getNodeControllerId();
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementOffloadingHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementOffloadingHandler.java
new file mode 100644
index 0000000..59e6e34
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementOffloadingHandler.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+
+public class PipelineElementOffloadingHandler implements IHandler<Response> {
+
+ private static final String HTTP_PROTOCOL = "http://";
+ private static final String COLON = ":";
+ private static final String SLASH = "/";
+ private static final String BASE_ROUTE = "streampipes-backend/api/v2/users/admin@streampipes.org";
+ private static final String OFFLOADING_ROUTE = BASE_ROUTE + "/pipelines/offload";
+
+ private final InvocableStreamPipesEntity graph;
+
+ public PipelineElementOffloadingHandler(InvocableStreamPipesEntity graph) {
+ this.graph = graph;
+ }
+
+ @Override
+ public Response handle() {
+ String url = generatePipelineManagementOffloadEndpoint();
+ return HttpUtils.post(url, graph);
+ }
+
+ private String generatePipelineManagementOffloadEndpoint() {
+ return HTTP_PROTOCOL
+ + NodeConfiguration.getBackendHost()
+ + COLON
+ + NodeConfiguration.getBackendPort()
+ + SLASH
+ + OFFLOADING_ROUTE;
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementReconfigurationHandler.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementReconfigurationHandler.java
new file mode 100644
index 0000000..fe1486c
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementReconfigurationHandler.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+
+import org.apache.streampipes.messaging.EventProducer;
+import org.apache.streampipes.messaging.jms.ActiveMQPublisher;
+import org.apache.streampipes.messaging.kafka.SpKafkaProducer;
+import org.apache.streampipes.messaging.mqtt.MqttPublisher;
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.model.grounding.*;
+import org.apache.streampipes.model.pipeline.PipelineElementReconfigurationEntity;
+import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty;
+import org.apache.streampipes.model.staticproperty.StaticProperty;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.pe.storage.RunningInvocableInstances;
+import org.apache.streampipes.node.controller.utils.HttpUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PipelineElementReconfigurationHandler implements IHandler<Response> {
+
+ private static final String DOT = ".";
+ private static final String RECONFIGURATION_TOPIC = "org.apache.streampipes.control.event.reconfigure";
+
+ private final InvocableStreamPipesEntity graph;
+ private final PipelineElementReconfigurationEntity reconfigurationEntity;
+
+ public PipelineElementReconfigurationHandler(InvocableStreamPipesEntity graph,
+ PipelineElementReconfigurationEntity reconfigurationEntity) {
+ this.graph = graph;
+ this.reconfigurationEntity = reconfigurationEntity;
+ }
+
+ @Override
+ public Response handle() {
+
+ Response response = new Response();
+ response.setElementId(graph.getElementId());
+ response.setSuccess(false);
+
+ EventProducer pub = getReconfigurationEventProducer();
+
+ byte [] reconfigurationEvent = reconfigurationToByteArray();
+ pub.publish(reconfigurationEvent);
+ pub.disconnect();
+
+ adaptPipelineDescription();
+
+ response.setSuccess(true);
+ return response;
+ }
+
+ private EventProducer getReconfigurationEventProducer() {
+ TransportProtocol tp = getReconfigurationTransportProtocol();
+ EventProducer pub;
+ if(tp instanceof KafkaTransportProtocol){
+ pub = new SpKafkaProducer();
+ pub.connect(tp);
+ } else if (tp instanceof JmsTransportProtocol){
+ pub = new ActiveMQPublisher();
+ pub.connect(tp);
+ } else{
+ pub = new MqttPublisher();
+ pub.connect(tp);
+ }
+ return pub;
+ }
+
+ // TODO: currently only FreeTextProperty is supported - need to updated in case other StaticProperty types will
+ // be supported in the future
+ private byte[] reconfigurationToByteArray() {
+ Map<String, String> reconfigurationEventMap = new HashMap<>();
+ reconfigurationEntity.getReconfiguredStaticProperties().forEach(staticProperty -> {
+ if (staticProperty instanceof FreeTextStaticProperty) {
+ reconfigurationEventMap.put(staticProperty.getInternalName(),
+ ((FreeTextStaticProperty) staticProperty).getValue());
+ }
+ });
+ return HttpUtils.serialize(reconfigurationEventMap).getBytes(StandardCharsets.UTF_8);
+ }
+
+ private void adaptPipelineDescription(){
+ List<StaticProperty> staticProperties = new ArrayList<>();
+ graph.getStaticProperties().forEach(sp -> {
+ int ind = graph.getStaticProperties().indexOf(sp);
+ staticProperties.add(ind, sp);
+ reconfigurationEntity.getReconfiguredStaticProperties().forEach(rp -> {
+ if(sp.getInternalName().equals(rp.getInternalName())){
+ staticProperties.remove(ind);
+ staticProperties.add(ind, rp);
+ }
+ });
+ });
+ graph.setStaticProperties(staticProperties);
+ RunningInvocableInstances.INSTANCE.remove(reconfigurationEntity.getDeploymentRunningInstanceId());
+ RunningInvocableInstances.INSTANCE.add(reconfigurationEntity.getDeploymentRunningInstanceId(), graph);
+ }
+
+ private TransportProtocol getReconfigurationTransportProtocol() {
+ TransportProtocol tp = graph.getInputStreams().get(0).getEventGrounding().getTransportProtocol();
+ TopicDefinition topic = generateReconfigurationTopicFromId(graph.getDeploymentRunningInstanceId());
+ tp.setTopicDefinition(topic);
+ return tp;
+ }
+
+ private TopicDefinition generateReconfigurationTopicFromId(String runningInstanceId) {
+ return new SimpleTopicDefinition( RECONFIGURATION_TOPIC + DOT + runningInstanceId);
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementRestartOnReboot.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementRestartOnReboot.java
new file mode 100644
index 0000000..92ca3a3
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/handler/PipelineElementRestartOnReboot.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.management.pe.handler;
+
+import org.apache.streampipes.model.Response;
+import org.apache.streampipes.model.base.InvocableStreamPipesEntity;
+import org.apache.streampipes.node.controller.management.IHandler;
+import org.apache.streampipes.node.controller.management.pe.PipelineElementManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PipelineElementRestartOnReboot implements IHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PipelineElementRestartOnReboot.class.getCanonicalName());
+
+ private final List<InvocableStreamPipesEntity> pipelineElements;
+
+ public PipelineElementRestartOnReboot(List<InvocableStreamPipesEntity> pipelineElements) {
+ this.pipelineElements = pipelineElements;
+ }
+
+ @Override
+ public Boolean handle() {
+ List<Response> allStatus = new ArrayList<>();
+ pipelineElements.forEach(graph -> {
+ Response status = PipelineElementManager.getInstance().invoke(graph);
+ allStatus.add(status);
+
+ if (status.isSuccess()) {
+ if (status.getOptionalMessage().isEmpty()) {
+ LOG.info("Pipeline element successfully restarted {}", status.getElementId());
+ } else {
+ LOG.info("Pipeline element already running {}", status.getElementId());
+ }
+ } else {
+ LOG.info("Pipeline element could not be restarted - are the pipeline element containers " +
+ "running? {}", status.getElementId());
+ }
+ });
+
+ return allStatus.stream().allMatch(Response::isSuccess);
+ }
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/RequirementsResourceValidator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/RequirementsResourceValidator.java
new file mode 100644
index 0000000..b17b3d9
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/RequirementsResourceValidator.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.node.controller.management.pe.validator;
+
+import org.apache.streampipes.model.base.ConsumableStreamPipesEntity;
+import org.apache.streampipes.model.node.NodeInfoDescription;
+import org.apache.streampipes.model.node.resources.hardware.HardwareResource;
+import org.apache.streampipes.model.resource.Hardware;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class RequirementsResourceValidator implements Validator {
+
+ private final NodeInfoDescription node;
+ private final List<ConsumableStreamPipesEntity> registeredPipelineElements;
+
+ public RequirementsResourceValidator(NodeInfoDescription node, List<ConsumableStreamPipesEntity> registeredPipelineElements) {
+ this.node = node;
+ this.registeredPipelineElements = registeredPipelineElements;
+ }
+
+ @Override
+ public List<ConsumableStreamPipesEntity> validate() {
+
+ return registeredPipelineElements.stream()
+ .filter(this::validateCpuRequirement)
+ .filter(this::validateMemoryRequirement)
+ .filter(this::validateDiskRequirement)
+ .filter(this::validateGpuRequirement)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public boolean validateGpuRequirement(ConsumableStreamPipesEntity entity) {
+ Optional<Hardware> hardwareRequirements = extractHardwareRequirements(entity);
+ HardwareResource nodeHardwareResource = getNodeHardwareResource();
+ boolean entityRequiresGpu = false;
+ if(hardwareRequirements.isPresent())
+ entityRequiresGpu = hardwareRequirements.get().isGpu();
+ boolean nodeHasGPU = (nodeHardwareResource.getGpu().getCores()>0);
+ return (!entityRequiresGpu || nodeHasGPU);
+ }
+
+ @Override
+ public boolean validateDiskRequirement(ConsumableStreamPipesEntity entity) {
+ Optional<Hardware> hardwareRequirements = extractHardwareRequirements(entity);
+ HardwareResource nodeHardwareResource = getNodeHardwareResource();
+ long requiredDiskSpace = 0L;
+ if(hardwareRequirements.isPresent())
+ requiredDiskSpace = hardwareRequirements.get().getDisk();
+ //Looks at total diskspace (could be adjusted to consider currently available diskspace)
+ long actualNodeDiskSpace = nodeHardwareResource.getDisk().getDiskTotal();
+ return (requiredDiskSpace <= actualNodeDiskSpace);
+ }
+
+ @Override
+ public boolean validateCpuRequirement(ConsumableStreamPipesEntity entity) {
+ Optional<Hardware> hardwareRequirements = extractHardwareRequirements(entity);
+ HardwareResource nodeHardwareResource = getNodeHardwareResource();
+ int requiredCPUCores = 0;
+ if(hardwareRequirements.isPresent())
+ requiredCPUCores = hardwareRequirements.get().getCpuCores();
+ int nodeCPUCores = nodeHardwareResource.getCpu().getCores();
+ return (requiredCPUCores <= nodeCPUCores);
+ }
+
+ @Override
+ public boolean validateMemoryRequirement(ConsumableStreamPipesEntity entity) {
+ Optional<Hardware> hardwareRequirements = extractHardwareRequirements(entity);
+ HardwareResource nodeHardwareResource = getNodeHardwareResource();
+ long requiredMemory = 0L;
+ if(hardwareRequirements.isPresent())
+ requiredMemory = hardwareRequirements.get().getMemory();
+ //Looks at total memory (could be adjusted to consider currently available Memory)
+ long actualNodeMemory = nodeHardwareResource.getMemory().getMemTotal();
+ return (requiredMemory <= actualNodeMemory);
+ }
+
+ // Helpers
+
+ private Optional<Hardware> extractHardwareRequirements(ConsumableStreamPipesEntity entity){
+ return entity.getResourceRequirements().stream()
+ .filter(req -> req instanceof Hardware)
+ .map(req -> (Hardware) req)
+ .findFirst();
+ }
+
+ private HardwareResource getNodeHardwareResource() {
+ return node.getNodeResources().getHardwareResource();
+ }
+
+}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/Validator.java
similarity index 60%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/Validator.java
index 7057ac9..4fe5566 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/property/FreeDiskSpaceResourceProperty.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/pe/validator/Validator.java
@@ -15,16 +15,16 @@
* limitations under the License.
*
*/
+package org.apache.streampipes.node.controller.management.pe.validator;
-package org.apache.streampipes.node.controller.management.offloading.model.property;
+import org.apache.streampipes.model.base.ConsumableStreamPipesEntity;
+import java.util.List;
-import org.apache.streampipes.model.node.monitor.ResourceMetrics;
-
-public class FreeDiskSpaceResourceProperty implements ResourceProperty<Long>{
-
- @Override
- public Long getProperty(ResourceMetrics resourceMetrics) {
- return resourceMetrics.getFreeDiskSpaceInBytes();
- }
+public interface Validator {
+ List<ConsumableStreamPipesEntity> validate();
+ boolean validateGpuRequirement(ConsumableStreamPipesEntity entity);
+ boolean validateDiskRequirement(ConsumableStreamPipesEntity entity);
+ boolean validateCpuRequirement(ConsumableStreamPipesEntity entity);
+ boolean validateMemoryRequirement(ConsumableStreamPipesEntity entity);
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/ResourceManager.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/ResourceManager.java
index fb3b20b..0107388 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/ResourceManager.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/ResourceManager.java
@@ -21,7 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
import org.apache.streampipes.node.controller.config.NodeConfiguration;
import org.apache.streampipes.node.controller.management.offloading.OffloadingPolicyManager;
-import org.apache.streampipes.node.controller.management.offloading.model.OffloadingStrategyFactory;
+import org.apache.streampipes.node.controller.management.offloading.strategies.OffloadingStrategyFactory;
import org.apache.streampipes.model.node.monitor.ResourceMetrics;
import org.apache.streampipes.node.controller.management.resource.utils.DiskSpace;
import org.apache.streampipes.node.controller.management.resource.utils.ResourceUtils;
@@ -45,11 +45,11 @@ public class ResourceManager {
private final HardwareAbstractionLayer hal = si.getHardware();
private final OperatingSystem os = si.getOperatingSystem();
private final Calendar cal = Calendar.getInstance();
- private final ResourceMetrics resourceMetrics = new ResourceMetrics();
+ private final ResourceMetrics metrics = new ResourceMetrics();
private ResourceManager() {
//Offloading Policy
- OffloadingPolicyManager.getInstance().addOffloadingStrategy(new OffloadingStrategyFactory().getFromEnv());
+ OffloadingPolicyManager.getInstance().addOffloadingStrategy(new OffloadingStrategyFactory().select());
}
public static ResourceManager getInstance() {
@@ -70,9 +70,9 @@ public class ResourceManager {
while(true) {
try {
// get current node resource metrics
- Thread.sleep( NodeConfiguration.getResourceMonitorFreqSecs() * 1000);
retrieveResources();
checkOffloadingPolicy();
+ Thread.sleep( NodeConfiguration.getResourceMonitorFreqSecs() * 1000);
} catch (InterruptedException e) {
LOG.error("Thread interrupted. {}", e.toString());
}
@@ -80,7 +80,7 @@ public class ResourceManager {
};
private void checkOffloadingPolicy() {
- OffloadingPolicyManager.getInstance().checkPolicies(resourceMetrics);
+ OffloadingPolicyManager.getInstance().checkPolicies(metrics);
}
private void retrieveResources() {
@@ -88,20 +88,19 @@ public class ResourceManager {
SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss a");
// Monitor node resources
- resourceMetrics.setSystemTime(dateFormat.format(cal.getTime()));
- resourceMetrics.setBooted(ResourceUtils.getBooted(this.os));
- resourceMetrics.setUptime(ResourceUtils.getUptime(this.os));
- resourceMetrics.setCpuLoad(String.format("%.1f%%", ResourceUtils.getCpuLoad(this.hal.getProcessor())));
- resourceMetrics.setCpuLoadInPercent(ResourceUtils.getCpuLoad(this.hal.getProcessor()));
- resourceMetrics.setCpuTemperature(String.format("%.2f°C",
- ResourceUtils.getCpuTemperature(this.hal.getSensors())));
- resourceMetrics.setCpuTemperatureCelcius(ResourceUtils.getCpuTemperature(this.hal.getSensors()));
- resourceMetrics.setFreeMemoryInBytes(ResourceUtils.getAvailableMemory(this.hal.getMemory()));
- resourceMetrics.setUsedMemoryInBytes(this.hal.getMemory().getTotal() - ResourceUtils.getAvailableMemory(this.hal.getMemory()));
- resourceMetrics.setTotalMemoryInBytes(this.hal.getMemory().getTotal());
+ metrics.setSystemTime(dateFormat.format(cal.getTime()));
+ metrics.setBooted(ResourceUtils.getBooted(this.os));
+ metrics.setUptime(ResourceUtils.getUptime(this.os));
+ metrics.setCpuLoad(String.format("%.1f%%", ResourceUtils.getCpuLoad(this.hal.getProcessor())));
+ metrics.setCpuLoadInPercent(ResourceUtils.getCpuLoad(this.hal.getProcessor()));
+ metrics.setCpuTemperature(String.format("%.2f°C", ResourceUtils.getCpuTemperature(this.hal.getSensors())));
+ metrics.setCpuTemperatureCelcius(ResourceUtils.getCpuTemperature(this.hal.getSensors()));
+ metrics.setFreeMemoryInBytes(ResourceUtils.getAvailableMemory(this.hal.getMemory()));
+ metrics.setUsedMemoryInBytes(this.hal.getMemory().getTotal() - ResourceUtils.getAvailableMemory(this.hal.getMemory()));
+ metrics.setTotalMemoryInBytes(this.hal.getMemory().getTotal());
ResourceUtils.getDiskUsage(this.os.getFileSystem()).forEach((key, value) -> {
- resourceMetrics.setFreeDiskSpaceInBytes(value.get(DiskSpace.USABLE.getName()));
+ metrics.setFreeDiskSpaceInBytes(value.get(DiskSpace.USABLE.getName()));
});
}
@@ -110,7 +109,7 @@ public class ResourceManager {
return JacksonSerializer
.getObjectMapper()
.writerWithDefaultPrettyPrinter()
- .writeValueAsString(resourceMetrics);
+ .writeValueAsString(metrics);
} catch (JsonProcessingException e) {
throw new SpRuntimeException("Could not serialize node resources " + e.getMessage(), e);
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/utils/ResourceChecker.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/utils/ResourceChecker.java
deleted file mode 100644
index 9af923b..0000000
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/resource/utils/ResourceChecker.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.node.controller.management.resource.utils;
-
-import org.apache.streampipes.model.base.ConsumableStreamPipesEntity;
-import org.apache.streampipes.model.node.NodeInfoDescription;
-import org.apache.streampipes.model.resource.Hardware;
-
-import java.util.Optional;
-
-public class ResourceChecker {
- private final NodeInfoDescription nodeInfo;
-
- public ResourceChecker(NodeInfoDescription nodeInfo){
- this.nodeInfo = nodeInfo;
- }
-
- public boolean checkResources(ConsumableStreamPipesEntity entity){
- return (checkCpuRequirement(entity) && checkGpuRequirement(entity)
- && checkMemoryRequirement(entity) && checkDiskSpaceRequirement(entity));
- }
-
- private boolean checkGpuRequirement(ConsumableStreamPipesEntity spEntity){
- Optional<Hardware> hw = extractHardware(spEntity);
- boolean entityRequiresGpu = false;
- if(hw.isPresent())
- entityRequiresGpu = hw.get().isGpu();
- boolean nodeHasGPU = (nodeInfo.getNodeResources().getHardwareResource().getGpu().getCores()>0);
- return (!entityRequiresGpu || nodeHasGPU);
- }
-
- private boolean checkCpuRequirement(ConsumableStreamPipesEntity spEntity){
- Optional<Hardware> hw = extractHardware(spEntity);
- int requiredCPUCores = 0;
- if(hw.isPresent())
- requiredCPUCores = hw.get().getCpuCores();
- int nodeCPUCores = nodeInfo.getNodeResources().getHardwareResource().getCpu().getCores();
- return (requiredCPUCores <= nodeCPUCores);
- }
-
- private boolean checkMemoryRequirement(ConsumableStreamPipesEntity spEntity){
- Optional<Hardware> hw = extractHardware(spEntity);
- long requiredMemory = 0L;
- if(hw.isPresent())
- requiredMemory = hw.get().getMemory();
- //Looks at total memory (could be adjusted to consider currently available Memory)
- long actualNodeMemory = nodeInfo.getNodeResources().getHardwareResource().getMemory().getMemTotal();
- return (requiredMemory <= actualNodeMemory);
- }
-
- private boolean checkDiskSpaceRequirement(ConsumableStreamPipesEntity spEntity){
- Optional<Hardware> hw = extractHardware(spEntity);
- long requiredDiskSpace = 0L;
- if(hw.isPresent())
- requiredDiskSpace = hw.get().getDisk();
- //Looks at total diskspace (could be adjusted to consider currently available diskspace)
- long actualNodeDiskSpace = nodeInfo.getNodeResources().getHardwareResource().getDisk().getDiskTotal();
- return (requiredDiskSpace <= actualNodeDiskSpace);
- }
-
- private Optional<Hardware> extractHardware(ConsumableStreamPipesEntity spEntity){
- return spEntity.getResourceRequirements().stream()
- .filter(req -> req instanceof Hardware).map(req -> (Hardware) req).findFirst();
- }
-
-}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpRequest.java
similarity index 85%
rename from streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
rename to streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpRequest.java
index ec31bfc..92224d6 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/management/offloading/model/policies/Comparator.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpRequest.java
@@ -15,9 +15,8 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.node.controller.management.offloading.model.policies;
+package org.apache.streampipes.node.controller.utils;
-public enum Comparator {
- SMALLER,
- GREATER
+public enum HttpRequest {
+ GET,POST,PUT,DELETE
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpUtils.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpUtils.java
index b97386c..71beb71 100644
--- a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpUtils.java
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/HttpUtils.java
@@ -18,13 +18,11 @@
package org.apache.streampipes.node.controller.utils;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.http.client.fluent.Request;
import org.apache.http.client.fluent.Response;
import org.apache.http.entity.ContentType;
import org.apache.streampipes.commons.exceptions.SpRuntimeException;
-import org.apache.streampipes.model.eventrelay.SpDataStreamRelayContainer;
-import org.apache.streampipes.model.node.NodeInfoDescription;
-import org.apache.streampipes.node.controller.config.NodeConfiguration;
import org.apache.streampipes.serializers.json.JacksonSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,12 +45,31 @@ public class HttpUtils {
}
}
- public static <T>T get(String url, Class<T> clazz) {
+ public static <T> T get(String url, Class<T> clazz) {
+ try {
+ Response response = Request.Get(url)
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute();
+ if (clazz.isInstance(Response.class)) {
+ return (T) response;
+ } else if (clazz.isInstance(String.class)) {
+ return (T) response.returnContent().asString();
+ } else if (clazz.isAssignableFrom(byte[].class)) {
+ return (T) response.returnContent().asBytes();
+ } else {
+ return deserialize(response, clazz);
+ }
+ } catch (IOException e) {
+ throw new SpRuntimeException("Something went wrong during GET request", e);
+ }
+ }
+
+ public static <T>T get(String url, final TypeReference<T> type) {
try {
return deserialize(Request.Get(url)
- .connectTimeout(CONNECT_TIMEOUT)
- .execute()
- .returnContent().asString(), clazz);
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute()
+ .returnContent().asString(), type);
} catch (IOException e) {
throw new SpRuntimeException("Something went wrong during GET request", e);
}
@@ -83,33 +100,115 @@ public class HttpUtils {
}
}
- public static boolean post(String url, String body) {
+ public static boolean put(String url) {
try {
- Request.Post(url)
- .bodyString(body, ContentType.APPLICATION_JSON)
+ Request.Put(url)
.connectTimeout(CONNECT_TIMEOUT)
.execute();
return true;
} catch (IOException e) {
- LOG.error("Something went wrong during POST request", e);
+ LOG.error("Something went wrong during PUT request", e);
return false;
}
}
- public static <T> boolean post(String url, T object) {
+ public static <T> boolean put(String url, T object) {
String body = serialize(object);
try {
- Request.Post(url)
+ Request.Put(url)
.bodyString(body, ContentType.APPLICATION_JSON)
.connectTimeout(CONNECT_TIMEOUT)
.execute();
return true;
} catch (IOException e) {
- LOG.error("Something went wrong during POST request", e);
+ LOG.error("Something went wrong during PUT request", e);
return false;
}
}
+ public static String post(String url, String body) {
+ try {
+ return Request.Post(url)
+ .bodyString(body, ContentType.APPLICATION_JSON)
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute().returnContent().asString();
+ } catch (IOException e) {
+ LOG.error("Something went wrong during POST request", e);
+ return null;
+ }
+ }
+
+ public static <T1, T2> T2 post(String url, T1 object, Class<T2> clazz) {
+ String body = serialize(object);
+ try {
+ Response response = Request.Post(url)
+ .bodyString(body, ContentType.APPLICATION_JSON)
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute();
+
+ if (clazz.isInstance(Response.class)) {
+ return (T2) response;
+ } else if (clazz.isInstance(String.class)) {
+ return (T2) response.returnContent().asString();
+ } else if (clazz.isAssignableFrom(byte[].class)) {
+ return (T2) response.returnContent().asBytes();
+ } else if (clazz.isAssignableFrom(Boolean.class)) {
+ return (T2) Boolean.TRUE;
+ } else {
+ return deserialize(response, clazz);
+ }
+ } catch (IOException e) {
+ if (clazz.isAssignableFrom(Boolean.class)) {
+ return (T2) Boolean.FALSE;
+ }
+
+ throw new SpRuntimeException("Something went wrong during POST request", e);
+ }
+ }
+
+ public static <T> org.apache.streampipes.model.Response post(String url, T object) {
+ String body = serialize(object);
+ org.apache.streampipes.model.Response response = new org.apache.streampipes.model.Response();
+ response.setSuccess(false);
+ try {
+ response = deserialize(Request.Post(url)
+ .bodyString(body, ContentType.APPLICATION_JSON)
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute(), org.apache.streampipes.model.Response.class);
+ return response;
+ } catch (IOException e) {
+ LOG.error("Something went wrong during POST request", e);
+ return response;
+ }
+ }
+
+// public static <T1, T2> T2 post(String url, T1 object, Class<T2> clazz) {
+// String body = serialize(object);
+// try {
+// return deserialize(Request.Post(url)
+// .bodyString(body, ContentType.APPLICATION_JSON)
+// .connectTimeout(CONNECT_TIMEOUT)
+// .execute()
+// .returnContent().asString(), clazz);
+// } catch (IOException e) {
+// throw new SpRuntimeException("Something went wrong during POST request", e);
+// }
+// }
+
+ public static org.apache.streampipes.model.Response delete(String url) {
+ org.apache.streampipes.model.Response response = new org.apache.streampipes.model.Response();
+ response.setSuccess(false);
+ try {
+ response = deserialize(Request.Delete(url)
+ .connectTimeout(CONNECT_TIMEOUT)
+ .execute(), org.apache.streampipes.model.Response.class);
+ return response;
+ } catch (Exception e) {
+ LOG.error("Something went wrong during DELETE request", e);
+ return response;
+ }
+ }
+
public static String generateEndpoint(String host, int port, String route) {
if (route.startsWith("/")) {
@@ -140,4 +239,27 @@ public class HttpUtils {
throw new SpRuntimeException("Could not deserialize object", e);
}
}
+
+ public static <T>T deserialize(Response response, Class<T> clazz) {
+ try {
+ String responseString = response.returnContent().asString();
+ return JacksonSerializer
+ .getObjectMapper()
+ .readValue(responseString, clazz);
+ } catch (JsonProcessingException e) {
+ throw new SpRuntimeException("Could not deserialize object", e);
+ } catch (IOException e) {
+ throw new SpRuntimeException("Could not get return content as string", e);
+ }
+ }
+
+ public static <T>T deserialize(String objectString, final TypeReference<T> type) {
+ try {
+ return JacksonSerializer
+ .getObjectMapper()
+ .readValue(objectString, type);
+ } catch (JsonProcessingException e) {
+ throw new SpRuntimeException("Could not deserialize object", e);
+ }
+ }
}
diff --git a/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/VersionUtils.java b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/VersionUtils.java
new file mode 100644
index 0000000..f018795
--- /dev/null
+++ b/streampipes-node-controller/src/main/java/org/apache/streampipes/node/controller/utils/VersionUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.node.controller.utils;
+
+import org.apache.streampipes.model.client.version.VersionInfo;
+import org.apache.streampipes.node.controller.config.NodeConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class VersionUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(VersionUtils.class.getCanonicalName());
+
+ private static final long RETRY_INTERVAL_MS = 5000;
+ private static final String ROUTE = "/streampipes-backend/api/v2/info/versions";
+
+ public static String getStreamPipesVersion() {
+ boolean connected = false;
+ VersionInfo versionInfo = new VersionInfo();
+ String host = NodeConfiguration.getBackendHost();
+ int port = NodeConfiguration.getBackendPort();
+
+ String endpoint = HttpUtils.generateEndpoint(host, port, ROUTE);
+ String bearerToken = NodeConfiguration.getNodeApiKey();
+
+ LOG.info("Trying to retrieve StreamPipes version from backend: " + endpoint);
+
+ while (!connected) {
+ versionInfo = HttpUtils.get(endpoint, bearerToken, VersionInfo.class);
+
+ if (versionInfo.getBackendVersion() != null) {
+ connected = true;
+ }
+
+ if (!connected) {
+ LOG.debug("Retrying in {} seconds", (RETRY_INTERVAL_MS / 1000));
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOG.info("Successfully retrieved StreamPipes version from backend");
+ return versionInfo.getBackendVersion();
+ }
+}
diff --git a/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/health/ClusterHealthCheckMonitor.java b/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/health/ClusterHealthCheckMonitor.java
index 6fa823d..4107801 100644
--- a/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/health/ClusterHealthCheckMonitor.java
+++ b/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/health/ClusterHealthCheckMonitor.java
@@ -82,14 +82,15 @@ public class ClusterHealthCheckMonitor {
if (failedChecks > 0) {
LOG.info(String.format("Healthcheck successful - %s is back online", endpoint));
nodeLiveness.resetNumFailedLivenessChecks();
+ persistNodeUpdate(node, true);
+ } else {
+ LOG.debug(String.format("Healthcheck successful - %s is still online", endpoint));
+ // set timestamp from healthcheck
+ node.setLastHeartBeatTime(healthCheck.getTimestamp());
+ // update node description in DB
+ persistNodeUpdate(node, false);
}
- LOG.debug(String.format("Healthcheck successful - %s is still online", endpoint));
- // set timestamp from healthcheck
- node.setLastHeartBeatTime(healthCheck.getTimestamp());
- // update node description in DB
- persistNodeUpdate(node, false);
-
} else {
// node was online and had unsuccessful healthcheck
// check if failedChecks exceeds allowed limit
@@ -171,7 +172,8 @@ public class ClusterHealthCheckMonitor {
private void addNodeToInMemoryStorage(NodeInfoDescription node) {
String nodeControllerId = node.getNodeControllerId();
if (inMemoryNodeLivenessStore.get(nodeControllerId) == null) {
- inMemoryNodeLivenessStore.put(nodeControllerId, new NodeLiveness(nodeControllerId, NodeCondition.CREATED,3));
+ inMemoryNodeLivenessStore.put(nodeControllerId, new NodeLiveness(nodeControllerId,
+ NodeCondition.CREATED, 3));
}
}
diff --git a/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/resource/ClusterResourceMonitor.java b/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/resource/ClusterResourceMonitor.java
index 47b7deb..4efd2d2 100644
--- a/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/resource/ClusterResourceMonitor.java
+++ b/streampipes-node-management/src/main/java/org/apache/streampipes/node/management/operation/monitor/resource/ClusterResourceMonitor.java
@@ -38,12 +38,9 @@ public class ClusterResourceMonitor {
private static final String RESOURCE_COLLECTOR_ROUTE = "/api/v2/node/info/resources";
private static final int RESOURCE_COLLECTOR_INTERVAL_SECS = 60;
- private static ClusterResourceMonitor instance = null;
- private static Map<String, Queue<ResourceMetrics>> resourceMetricsMap = new HashMap<>();
+ private static final Map<String, Queue<ResourceMetrics>> resourceMetricsMap = new HashMap<>();
- public static Map<String, Queue<ResourceMetrics>> getResourceMetricsMap(){
- return resourceMetricsMap;
- }
+ private static ClusterResourceMonitor instance = null;
private ClusterResourceMonitor() {}
@@ -69,7 +66,6 @@ public class ClusterResourceMonitor {
String endpoint = HttpUtils.generateEndpoint(node, RESOURCE_COLLECTOR_ROUTE);
ResourceMetrics resourceMetrics = new NodeResourceCollector(endpoint).execute();
- // TODO: implement
addResourceMetrics(node, resourceMetrics);
});
};
@@ -82,4 +78,8 @@ public class ClusterResourceMonitor {
resourceMetricsMap.get(node.getNodeControllerId()).offer(rm);
}
}
+
+ public static Queue<ResourceMetrics> getNodeResourceMetricsById(String nodeControllerId){
+ return resourceMetricsMap.get(nodeControllerId);
+ }
}
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/ReconfigurationEndpointUrlGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/ReconfigurationEndpointUrlGenerator.java
index c7b0dae..2150f7f 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/ReconfigurationEndpointUrlGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/ReconfigurationEndpointUrlGenerator.java
@@ -26,7 +26,7 @@ public class ReconfigurationEndpointUrlGenerator {
protected static final String SLASH = "/";
private static final String BASE_ROUTE = "/api/v2/node/element";
- private static final String RECONFIGURE_ROUTE = "/adapt";
+ private static final String RECONFIGURE_ROUTE = "/reconfigure";
private final PipelineElementReconfigurationEntity reconfigurationEntity;
diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationPipelineGenerator.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationPipelineGenerator.java
index b73c479..2d01aad 100644
--- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationPipelineGenerator.java
+++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/MigrationPipelineGenerator.java
@@ -33,24 +33,24 @@ import java.util.stream.Collectors;
public class MigrationPipelineGenerator {
- private InvocableStreamPipesEntity entityToMigrate;
- private Pipeline correspondingPipeline;
- private final float memoryMultiplier = 0.9F;
- private final float diskSpaceMultiplier = 0.9F;
+ private static final float MEM_MULTIPLICATION_FACTOR = 0.9F;
+ private static final float DISK_MULTIPLICATION_FACTOR = 0.9F;
+
+ private final InvocableStreamPipesEntity entityToMigrate;
+ private final Pipeline correspondingPipeline;
public MigrationPipelineGenerator(InvocableStreamPipesEntity entityToMigrate, Pipeline correspondingPipeline){
this.entityToMigrate = entityToMigrate;
this.correspondingPipeline = correspondingPipeline;
}
-
public Pipeline generateMigrationPipeline(){
- List<NodeInfoDescription> possibleTargetNodes = getNodeInfos();
+ List<NodeInfoDescription> eligibleTargetNodes = findEligibleTargetNodes();
switch(correspondingPipeline.getExecutionPolicy()){
case "custom": //TODO: Enum class
- possibleTargetNodes = filterLocationTags(possibleTargetNodes);
+ eligibleTargetNodes = filterLocationTags(eligibleTargetNodes);
case "locality-aware":
//TODO: incorporate strategy for locality-aware deployment
case "default":
@@ -58,28 +58,37 @@ public class MigrationPipelineGenerator {
}
//Check current resource utilization on node
- possibleTargetNodes = filterResourceUtilization(possibleTargetNodes);
-
+ eligibleTargetNodes = filterResourceUtilization(eligibleTargetNodes);
//Different strategies possible (atm cancel offloading)
- if(possibleTargetNodes == null || possibleTargetNodes.isEmpty())
+ if(eligibleTargetNodes == null || eligibleTargetNodes.isEmpty())
return null;
- //Random Selection of new Node within the remaining possible nodes
- changeEntityDescriptionToMatchRandomNode(possibleTargetNodes);
+ //Random Selection of new node within the remaining eligibile nodes
+ randomSelectionAndUpdate(eligibleTargetNodes);
return generateTargetPipeline();
}
- private List<NodeInfoDescription> getNodeInfos(){
- List<NodeInfoDescription> possibleTargetNodes = new ArrayList<>();
- List<NodeInfoDescription> nodeInfo = NodeManagement.getInstance().getOnlineNodes();
- nodeInfo.forEach(desc ->{
- if(desc.getSupportedElements().stream().anyMatch(element -> element.equals(entityToMigrate.getAppId()))
- && !desc.getNodeControllerId().equals(entityToMigrate.getDeploymentTargetNodeId()))
- possibleTargetNodes.add(desc);
- });
- return possibleTargetNodes;
+ private List<NodeInfoDescription> findEligibleTargetNodes(){
+
+ List<NodeInfoDescription> eligibileTargetNodes = new ArrayList<>();
+ List<NodeInfoDescription> onlineNodes = NodeManagement.getInstance().getOnlineNodes();
+
+ onlineNodes.stream()
+ .filter(this::matchAndVerify)
+ .map(eligibileTargetNodes::add);
+
+ return eligibileTargetNodes;
+ }
+
+ private boolean matchAndVerify(NodeInfoDescription node) {
+ boolean matchingAppIds = node.getSupportedElements().stream()
+ .anyMatch(pipelineElement -> pipelineElement.equals(entityToMigrate.getAppId()));
+
+ boolean dissimilarNodes = !node.getNodeControllerId().equals(entityToMigrate.getDeploymentTargetNodeId());
+
+ return matchingAppIds && dissimilarNodes;
}
private List<NodeInfoDescription> filterLocationTags(List<NodeInfoDescription> possibleTargetNodes){
@@ -89,25 +98,33 @@ public class MigrationPipelineGenerator {
}
private boolean nodeTagsContainElementTag(List<String> pipelineNodeTags,
- NodeInfoDescription desc){
- return desc.getStaticNodeMetadata().getLocationTags().stream().anyMatch(pipelineNodeTags::contains);
+ NodeInfoDescription node){
+ return node.getStaticNodeMetadata().getLocationTags().stream().anyMatch(pipelineNodeTags::contains);
}
private List<NodeInfoDescription> filterResourceUtilization(List<NodeInfoDescription> possibleTargetNodes){
//Currently only checking for free disk space and memory
List<NodeInfoDescription> filteredTargetNodes = new ArrayList<>();
+
for(NodeInfoDescription nodeInfo : possibleTargetNodes){
- Queue<ResourceMetrics> rmHistory = ClusterResourceMonitor.getResourceMetricsMap()
- .get(nodeInfo.getNodeControllerId());
- if(rmHistory == null) return null;
+
+ String nodeControllerId = nodeInfo.getNodeControllerId();
+ Queue<ResourceMetrics> rmHistory = ClusterResourceMonitor.getNodeResourceMetricsById(nodeControllerId);
+
+ if(rmHistory == null)
+ return null;
+
Hardware hardware = entityToMigrate.getResourceRequirements().stream()
- .filter(nodeRR -> nodeRR instanceof Hardware).map(nodeRR -> (Hardware)nodeRR).findFirst().
- orElse(null);
+ .filter(nodeRR -> nodeRR instanceof Hardware)
+ .map(nodeRR -> (Hardware)nodeRR)
+ .findFirst()
+ .orElse(null);
+
if(hardware != null){ //TODO: Map CPU load ()
//Does produce empty list if no hardware requirements are defined
if (rmHistory.peek() != null
- && hardware.getDisk() <= diskSpaceMultiplier * rmHistory.peek().getFreeDiskSpaceInBytes()
- && hardware.getMemory() <= memoryMultiplier * rmHistory.peek().getFreeMemoryInBytes()) {
+ && hardware.getDisk() <= DISK_MULTIPLICATION_FACTOR * rmHistory.peek().getFreeDiskSpaceInBytes()
+ && hardware.getMemory() <= MEM_MULTIPLICATION_FACTOR * rmHistory.peek().getFreeMemoryInBytes()) {
filteredTargetNodes.add(nodeInfo);
}
}
@@ -116,10 +133,10 @@ public class MigrationPipelineGenerator {
}
private Pipeline generateTargetPipeline(){
- Optional<DataProcessorInvocation> originalInvocation =
- correspondingPipeline.getSepas().stream().filter(dp ->
- dp.getDeploymentRunningInstanceId().equals(entityToMigrate.getDeploymentRunningInstanceId()))
- .findFirst();
+ Optional<DataProcessorInvocation> originalInvocation = correspondingPipeline.getSepas().stream()
+ .filter(dp -> dp.getDeploymentRunningInstanceId().equals(entityToMigrate.getDeploymentRunningInstanceId()))
+ .findFirst();
+
int index = correspondingPipeline.getSepas().indexOf(originalInvocation.get());
Pipeline targetPipeline;
@@ -136,7 +153,7 @@ public class MigrationPipelineGenerator {
return targetPipeline;
}
- private void changeEntityDescriptionToMatchRandomNode(List<NodeInfoDescription> nodes){
+ private void randomSelectionAndUpdate(List<NodeInfoDescription> nodes){
NodeInfoDescription targetNode = nodes.get(new Random().nextInt(nodes.size()));
entityToMigrate.setDeploymentTargetNodeHostname(targetNode.getHostname());
diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
index f40a753..31cc55a 100644
--- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
+++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.scss
@@ -18,9 +18,9 @@
@import '../../../../scss/sp/sp-dialog.scss';
-//.sp-dialog-container {
-// width: 520px;
-//}
+.sp-dialog-container {
+ width: 500px;
+}
.customize-section {
display:flex;
diff --git a/ui/src/scss/sp/sp-dialog.scss b/ui/src/scss/sp/sp-dialog.scss
index e79a646..f74432c 100644
--- a/ui/src/scss/sp/sp-dialog.scss
+++ b/ui/src/scss/sp/sp-dialog.scss
@@ -20,7 +20,6 @@
display: grid;
height: 100%;
grid-template-rows: auto 10px 60px;
-
}
.sp-dialog-content {