You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/08/19 13:53:06 UTC
(hop) branch main updated: Issue #8016 : Group GuiWidgetElement fields into tabs (#8018)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 77200d6c5c Issue #8016 : Group GuiWidgetElement fields into tabs (#8018)
77200d6c5c is described below
commit 77200d6c5c59836f1b4ab0a65e1e8db54c4817a2
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Wed Aug 19 15:52:59 2026 +0200
Issue #8016 : Group GuiWidgetElement fields into tabs (#8018)
* Issue #7992 : Add load-balancing pipeline and workflow engines
Introduce Hop Server pool run configurations that assign each pipeline
or workflow execution to one server from a group (even-load or pack),
with live status probing, server-side max_concurrent admission, and
client retry when a server is at capacity.
Also switch Pipeline/Workflow Executor run-configuration widgets to
MetaSelectionLine, add finished execution counts to Get Server Status,
and add a load-balance integration-test project.
* Issue #7992 : Put load-balancing run configuration options on three tabs
Split the engine-specific widgets into Management, Load-balancing, and
Hop Servers so the metadata editor no longer overflows a single form.
* Issue #8016 : Group GuiWidgetElement fields into tabs
Add optional group, groupOrder, groupImage, and groupType on
GuiWidgetElement. GuiCompositeWidgets lays grouped fields out as a
CTabFolder. LIST and BOXES are reserved and currently fall back to
tabs.
Migrate the load-balancing pipeline and workflow run configuration
editors onto this API so they no longer build a custom tab folder.
The Hop Servers table is an extra group registered by the editor.
---
.../apache/hop/core/gui/plugin/GuiElements.java | 51 ++++
.../hop/core/gui/plugin/GuiWidgetElement.java | 19 ++
.../hop/core/gui/plugin/GuiWidgetGroupType.java | 29 ++
.../hop/core/gui/plugin/GuiWidgetGroups.java | 142 ++++++++++
.../hop/core/gui/plugin/GuiWidgetGroupsTest.java | 84 ++++++
.../integration-tests-load-balance.yaml | 72 +++++
...d-balance-pipeline-run-configuration-editor.png | Bin 0 -> 238356 bytes
...d-balance-workflow-run-configuration-editor.png | Bin 0 -> 226382 bytes
docs/hop-user-manual/modules/ROOT/nav.adoc | 4 +
.../modules/ROOT/pages/hop-server/index.adoc | 2 +-
.../modules/ROOT/pages/hop-server/rest-api.adoc | 2 +
.../pages/metadata-types/pipeline-run-config.adoc | 1 +
.../pages/metadata-types/workflow-run-config.adoc | 2 +-
.../native-load-balancing-pipeline-engine.adoc | 130 +++++++++
.../pipeline-run-configurations.adoc | 1 +
.../pages/pipeline/transforms/serverstatus.adoc | 2 +
.../native-load-balancing-workflow-engine.adoc | 44 ++++
.../workflow-run-configurations.adoc | 1 +
.../loadbalance/LoadBalancingPipelineEngine.java | 289 ++++++++++++++++++++
.../LoadBalancingPipelineRunConfiguration.java | 290 ++++++++++++++++++++
.../engines/remote/RemotePipelineEngine.java | 41 ++-
.../remote/RemotePipelineRunConfiguration.java | 3 +-
.../hop/server/IRemoteCapableRunConfiguration.java | 44 ++++
.../hop/server/loadbalance/HopServerProbe.java | 162 ++++++++++++
.../ILoadBalancingRunConfiguration.java | 57 ++++
.../server/loadbalance/LoadBalancingAlgorithm.java | 58 ++++
.../loadbalance/LoadBalancingAssignment.java | 49 ++++
.../loadbalance/LoadBalancingAssignmentStore.java | 102 +++++++
.../loadbalance/LoadBalancingCoordinator.java | 292 +++++++++++++++++++++
.../loadbalance/LoadBalancingRetryPolicy.java | 86 ++++++
.../server/loadbalance/LoadBalancingSelector.java | 64 +++++
.../loadbalance/LoadBalancingServerEntry.java | 52 ++++
.../server/loadbalance/ServerHealthSnapshot.java | 100 +++++++
.../loadbalance/LoadBalancingWorkflowEngine.java | 264 +++++++++++++++++++
.../LoadBalancingWorkflowRunConfiguration.java | 289 ++++++++++++++++++++
.../engines/remote/RemoteWorkflowEngine.java | 143 ++++++----
.../remote/RemoteWorkflowRunConfiguration.java | 3 +-
.../org/apache/hop/www/BaseWorkflowServlet.java | 30 ++-
.../org/apache/hop/www/HopServerAdmission.java | 136 ++++++++++
.../hop/www/HopServerAtCapacityException.java | 45 ++++
.../org/apache/hop/www/RegisterPackageServlet.java | 81 +++---
.../apache/hop/www/RegisterPipelineServlet.java | 19 +-
.../apache/hop/www/RegisterWorkflowServlet.java | 19 +-
.../java/org/apache/hop/www/RemoteHopServer.java | 16 +-
.../LoadBalancingPipelineEngineTest.java | 91 +++++++
.../hop/server/loadbalance/HopServerProbeTest.java | 70 +++++
.../LoadBalancingAssignmentStoreTest.java | 64 +++++
.../loadbalance/LoadBalancingCoordinatorTest.java | 78 ++++++
.../loadbalance/LoadBalancingRetryPolicyTest.java | 56 ++++
.../loadbalance/LoadBalancingSelectorTest.java | 88 +++++++
.../org/apache/hop/www/HopServerAdmissionTest.java | 103 ++++++++
integration-tests/load-balance/.gitignore | 2 +
.../load-balance/0001-child-delay.hpl | 108 ++++++++
.../load-balance/0001-child-delay.hwf | 99 +++++++
.../load-balance/0001-drive-100-pipelines.hpl | 182 +++++++++++++
.../load-balance/0001-server-status.hpl | 244 +++++++++++++++++
.../load-balance/0002-drive-100-workflows.hpl | 181 +++++++++++++
integration-tests/load-balance/dev-env-config.json | 3 +
integration-tests/load-balance/hop-config.json | 33 +++
.../main-0001-load-balance-pipeline-executor.hwf | 261 ++++++++++++++++++
.../main-0002-load-balance-workflow-executor.hwf | 261 ++++++++++++++++++
.../pipeline-run-configuration/load-balance.json | 37 +++
.../metadata/pipeline-run-configuration/local.json | 25 ++
.../load-balance/metadata/server/hop-server-a.json | 13 +
.../load-balance/metadata/server/hop-server-b.json | 13 +
.../workflow-run-configuration/load-balance.json | 35 +++
.../metadata/workflow-run-configuration/local.json | 11 +
integration-tests/load-balance/project-config.json | 14 +
integration-tests/scripts/run-tests-docker.sh | 10 +-
.../pipelineexecutor/PipelineExecutorDialog.java | 30 +--
.../messages/messages_en_US.properties | 1 +
.../transforms/serverstatus/GetServerStatus.java | 80 ++----
.../serverstatus/GetServerStatusDialog.java | 42 +++
.../serverstatus/GetServerStatusMeta.java | 48 ++++
.../messages/messages_en_US.properties | 2 +
.../workflowexecutor/WorkflowExecutorDialog.java | 32 +--
.../messages/messages_en_US.properties | 1 +
.../ui/core/gui/GuiCompositeWidgetsGroupTest.java | 271 +++++++++++++++++++
.../hop/ui/core/gui/GuiCompositeWidgets.java | 257 ++++++++++++++++--
.../apache/hop/ui/hopgui/PaletteEngineFilter.java | 20 +-
.../config/PipelineRunConfigurationEditor.java | 25 ++
.../LoadBalancingRunConfigurationWidgets.java | 173 ++++++++++++
.../config/WorkflowRunConfigurationEditor.java | 35 ++-
.../ui/core/gui/messages/messages_en_US.properties | 18 ++
.../config/messages/messages_en_US.properties | 14 +
.../loadbalance/messages/messages_en_US.properties | 25 ++
.../config/messages/messages_en_US.properties | 14 +
.../hop/ui/hopgui/PaletteEngineFilterTest.java | 11 +-
88 files changed, 6238 insertions(+), 263 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiElements.java b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiElements.java
index 0c773a3c6e..abcd042192 100644
--- a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiElements.java
+++ b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiElements.java
@@ -79,8 +79,14 @@ public class GuiElements extends BaseGuiElements implements Comparable<GuiElemen
private Class<? extends IHopMetadata> metadata;
private Method buttonMethod;
+ private String group;
+ private String groupOrder;
+ private String groupImage;
+ private GuiWidgetGroupType groupType;
+
public GuiElements() {
children = new ArrayList<>();
+ groupType = GuiWidgetGroupType.NONE;
}
public GuiElements(GuiWidgetElement guiElement, Field field) {
@@ -116,6 +122,7 @@ public class GuiElements extends BaseGuiElements implements Comparable<GuiElemen
this.typeFilename = guiElement.typeFilename();
this.metadata = guiElement.metadata();
this.buttonMethod = null;
+ copyGroup(guiElement, fieldPackageName, field.getDeclaringClass());
}
/**
@@ -157,6 +164,14 @@ public class GuiElements extends BaseGuiElements implements Comparable<GuiElemen
this.metadata = guiElement.metadata();
this.classLoader = classLoader;
this.buttonMethod = method;
+ copyGroup(guiElement, methodPackageName, method.getDeclaringClass());
+ }
+
+ private void copyGroup(GuiWidgetElement guiElement, String i18nPackage, Class<?> resourceClass) {
+ this.group = getTranslation(guiElement.group(), i18nPackage, resourceClass);
+ this.groupOrder = guiElement.groupOrder();
+ this.groupImage = guiElement.groupImage();
+ this.groupType = guiElement.groupType();
}
/** Sort the children using the sort order. If no sort field is available we use the ID */
@@ -616,4 +631,40 @@ public class GuiElements extends BaseGuiElements implements Comparable<GuiElemen
public void setButtonMethod(Method buttonMethod) {
this.buttonMethod = buttonMethod;
}
+
+ public String getGroup() {
+ return group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public String getGroupOrder() {
+ return groupOrder;
+ }
+
+ public void setGroupOrder(String groupOrder) {
+ this.groupOrder = groupOrder;
+ }
+
+ public String getGroupImage() {
+ return groupImage;
+ }
+
+ public void setGroupImage(String groupImage) {
+ this.groupImage = groupImage;
+ }
+
+ public GuiWidgetGroupType getGroupType() {
+ return groupType == null ? GuiWidgetGroupType.NONE : groupType;
+ }
+
+ public void setGroupType(GuiWidgetGroupType groupType) {
+ this.groupType = groupType;
+ }
+
+ public boolean hasGroup() {
+ return StringUtils.isNotEmpty(group);
+ }
}
diff --git a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetElement.java b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetElement.java
index 3d9adda3e9..15bf4ed8e7 100644
--- a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetElement.java
+++ b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetElement.java
@@ -144,4 +144,23 @@ public @interface GuiWidgetElement {
* @return
*/
Class<? extends IHopMetadata> metadata() default IHopMetadata.class;
+
+ /**
+ * Layout bucket inside the {@link #parentId()} tree. Empty (the default) keeps the widget on the
+ * flat form. When any sibling has a group, widgets are shown in that container ({@link
+ * #groupType()}).
+ */
+ String group() default "";
+
+ /** Sort key for the group among other groups. Compared as a string, same as {@link #order()}. */
+ String groupOrder() default "";
+
+ /** Optional image filename for the group (tab icon, list item, …). */
+ String groupImage() default "";
+
+ /**
+ * How groups on this {@link #parentId()} are shown. Repeat on every field; the first non-{@link
+ * GuiWidgetGroupType#NONE} value wins. Mixed types log a warning and fall back to tabs.
+ */
+ GuiWidgetGroupType groupType() default GuiWidgetGroupType.NONE;
}
diff --git a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupType.java b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupType.java
new file mode 100644
index 0000000000..b56836b12b
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupType.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.gui.plugin;
+
+/**
+ * How {@link GuiWidgetElement} fields that share a {@link GuiWidgetElement#group()} are laid out
+ * inside one {@code parentId} tree. {@link #NONE} keeps the current single-form layout.
+ */
+public enum GuiWidgetGroupType {
+ NONE,
+ TABS,
+ LIST,
+ BOXES
+}
diff --git a/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroups.java b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroups.java
new file mode 100644
index 0000000000..f8433b044b
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/core/gui/plugin/GuiWidgetGroups.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.gui.plugin;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+
+/** Partitions {@link GuiElements} into layout groups. SWT-free so it can be unit-tested in core. */
+public final class GuiWidgetGroups {
+
+ public static final String UNGROUPED_KEY = "";
+
+ private GuiWidgetGroups() {}
+
+ public static boolean hasGroups(List<GuiElements> children) {
+ if (children == null) {
+ return false;
+ }
+ for (GuiElements child : children) {
+ if (include(child) && child.hasGroup()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * True when two included children declare different non-{@link GuiWidgetGroupType#NONE} types.
+ * Mixed types fall back to {@link GuiWidgetGroupType#TABS}.
+ */
+ public static boolean hasMixedTypes(List<GuiElements> children) {
+ GuiWidgetGroupType type = GuiWidgetGroupType.NONE;
+ if (children == null) {
+ return false;
+ }
+ for (GuiElements child : children) {
+ if (!include(child) || child.getGroupType() == GuiWidgetGroupType.NONE) {
+ continue;
+ }
+ if (type == GuiWidgetGroupType.NONE) {
+ type = child.getGroupType();
+ } else if (type != child.getGroupType()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static GuiWidgetGroupType typeOf(List<GuiElements> children) {
+ if (hasMixedTypes(children)) {
+ return GuiWidgetGroupType.TABS;
+ }
+ GuiWidgetGroupType type = GuiWidgetGroupType.NONE;
+ if (children != null) {
+ for (GuiElements child : children) {
+ if (!include(child) || child.getGroupType() == GuiWidgetGroupType.NONE) {
+ continue;
+ }
+ type = child.getGroupType();
+ break;
+ }
+ }
+ return type == GuiWidgetGroupType.NONE ? GuiWidgetGroupType.TABS : type;
+ }
+
+ public static List<Bucket> from(List<GuiElements> children, String ungroupedLabel) {
+ Map<String, Bucket> byKey = new LinkedHashMap<>();
+ List<GuiElements> sorted = new ArrayList<>();
+ if (children != null) {
+ sorted.addAll(children);
+ }
+ Collections.sort(sorted);
+
+ for (GuiElements child : sorted) {
+ if (!include(child)) {
+ continue;
+ }
+ String key = child.hasGroup() ? child.getGroup() : UNGROUPED_KEY;
+ String label = child.hasGroup() ? child.getGroup() : ungroupedLabel;
+ String order = child.hasGroup() ? Const.NVL(child.getGroupOrder(), "") : "0";
+ Bucket bucket =
+ byKey.computeIfAbsent(key, k -> new Bucket(key, label, order, child.getGroupImage()));
+ if (StringUtils.isEmpty(bucket.getOrder()) && StringUtils.isNotEmpty(order)) {
+ bucket.setOrder(order);
+ }
+ if (StringUtils.isEmpty(bucket.getImage()) && StringUtils.isNotEmpty(child.getGroupImage())) {
+ bucket.setImage(child.getGroupImage());
+ }
+ bucket.getElements().add(child);
+ }
+
+ List<Bucket> buckets = new ArrayList<>(byKey.values());
+ buckets.sort(
+ Comparator.comparing((Bucket b) -> Const.NVL(b.getOrder(), ""))
+ .thenComparing(b -> Const.NVL(b.getLabel(), "")));
+ return buckets;
+ }
+
+ private static boolean include(GuiElements child) {
+ return child != null && !child.isIgnored() && child.getId() != null;
+ }
+
+ @Getter
+ @Setter
+ public static final class Bucket {
+ private final String key;
+ private final String label;
+ private String order;
+ private String image;
+ private final List<GuiElements> elements = new ArrayList<>();
+
+ Bucket(String key, String label, String order, String image) {
+ this.key = key;
+ this.label = label;
+ this.order = order;
+ this.image = image;
+ }
+ }
+}
diff --git a/core/src/test/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupsTest.java b/core/src/test/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupsTest.java
new file mode 100644
index 0000000000..89b5e7e97c
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/gui/plugin/GuiWidgetGroupsTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.gui.plugin;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.junit.jupiter.api.Test;
+
+class GuiWidgetGroupsTest {
+
+ @Test
+ void noGroupsWhenNothingIsLabeled() {
+ GuiElements a = element("a", "", GuiWidgetGroupType.NONE);
+ GuiElements b = element("b", "", GuiWidgetGroupType.NONE);
+ assertFalse(GuiWidgetGroups.hasGroups(List.of(a, b)));
+ }
+
+ @Test
+ void partitionsByGroupAndKeepsOrder() {
+ GuiElements second = element("second", "Two", "20", GuiWidgetGroupType.TABS);
+ GuiElements first = element("first", "One", "10", GuiWidgetGroupType.TABS);
+ assertTrue(GuiWidgetGroups.hasGroups(List.of(second, first)));
+
+ List<GuiWidgetGroups.Bucket> buckets = GuiWidgetGroups.from(List.of(second, first), "General");
+ assertEquals(2, buckets.size());
+ assertEquals("One", buckets.get(0).getLabel());
+ assertEquals("first", buckets.get(0).getElements().get(0).getId());
+ assertEquals("Two", buckets.get(1).getLabel());
+ assertEquals(GuiWidgetGroupType.TABS, GuiWidgetGroups.typeOf(List.of(second, first)));
+ assertFalse(GuiWidgetGroups.hasMixedTypes(List.of(second, first)));
+ }
+
+ @Test
+ void ungroupedSiblingsLandInGeneralWhenAnyGroupExists() {
+ GuiElements grouped = element("g", "Tab", "10", GuiWidgetGroupType.TABS);
+ GuiElements loose = element("l", "", GuiWidgetGroupType.TABS);
+ List<GuiWidgetGroups.Bucket> buckets = GuiWidgetGroups.from(List.of(grouped, loose), "General");
+ assertEquals(2, buckets.size());
+ assertEquals("General", buckets.get(0).getLabel());
+ assertEquals("l", buckets.get(0).getElements().get(0).getId());
+ assertEquals("Tab", buckets.get(1).getLabel());
+ }
+
+ @Test
+ void mixedTypesFallBackToTabs() {
+ GuiElements tabs = element("a", "A", "10", GuiWidgetGroupType.TABS);
+ GuiElements list = element("b", "B", "20", GuiWidgetGroupType.LIST);
+ assertTrue(GuiWidgetGroups.hasMixedTypes(List.of(tabs, list)));
+ assertEquals(GuiWidgetGroupType.TABS, GuiWidgetGroups.typeOf(List.of(tabs, list)));
+ }
+
+ private static GuiElements element(String id, String group, GuiWidgetGroupType type) {
+ return element(id, group, "", type);
+ }
+
+ private static GuiElements element(
+ String id, String group, String groupOrder, GuiWidgetGroupType type) {
+ GuiElements element = new GuiElements();
+ element.setId(id);
+ element.setOrder(id);
+ element.setGroup(group);
+ element.setGroupOrder(groupOrder);
+ element.setGroupType(type);
+ return element;
+ }
+}
diff --git a/docker/integration-tests/integration-tests-load-balance.yaml b/docker/integration-tests/integration-tests-load-balance.yaml
new file mode 100644
index 0000000000..e447b87b28
--- /dev/null
+++ b/docker/integration-tests/integration-tests-load-balance.yaml
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+ integration_test_load_balance:
+ extends:
+ file: integration-tests-base.yaml
+ service: integration_test
+ depends_on:
+ hop_server_a:
+ condition: service_healthy
+ hop_server_b:
+ condition: service_healthy
+ links:
+ - hop_server_a
+ - hop_server_b
+
+ hop_server_a:
+ image: apache/hop:Development
+ build:
+ context: ../../.
+ dockerfile: docker/Dockerfile
+ pull_policy: never
+ hostname: hop-server-a
+ networks:
+ default:
+ aliases:
+ - hop-server-a
+ environment:
+ - HOP_SERVER_PORT=8181
+ - HOP_SERVER_HOSTNAME=hop-server-a
+ healthcheck:
+ test: curl -u cluster:cluster --fail http://hop-server-a:8181 || exit 1
+ interval: 10s
+ timeout: 5s
+ retries: 12
+ start_period: 120s
+
+ hop_server_b:
+ image: apache/hop:Development
+ build:
+ context: ../../.
+ dockerfile: docker/Dockerfile
+ pull_policy: never
+ hostname: hop-server-b
+ networks:
+ default:
+ aliases:
+ - hop-server-b
+ environment:
+ - HOP_SERVER_PORT=8181
+ - HOP_SERVER_HOSTNAME=hop-server-b
+ healthcheck:
+ test: curl -u cluster:cluster --fail http://hop-server-b:8181 || exit 1
+ interval: 10s
+ timeout: 5s
+ retries: 12
+ start_period: 120s
diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-pipeline-run-configuration-editor.png b/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-pipeline-run-configuration-editor.png
new file mode 100644
index 0000000000..8b49857883
Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-pipeline-run-configuration-editor.png differ
diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-workflow-run-configuration-editor.png b/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-workflow-run-configuration-editor.png
new file mode 100644
index 0000000000..af5518a7c6
Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/run-configuration/load-balance-workflow-run-configuration-editor.png differ
diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc
index a262ebd627..f4bbb31125 100644
--- a/docs/hop-user-manual/modules/ROOT/nav.adoc
+++ b/docs/hop-user-manual/modules/ROOT/nav.adoc
@@ -64,6 +64,7 @@ under the License.
*** xref:pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc[Beam Spark]
*** xref:pipeline/pipeline-run-configurations/native-local-pipeline-engine.adoc[Native Local]
*** xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Native Remote]
+*** xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[Native Load-balancing]
*** xref:pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc[Native Spark]
** xref:pipeline/pipeline-unit-testing.adoc[Pipeline Unit Tests]
** xref:pipeline/metadata-injection.adoc[Metadata Injection]
@@ -319,6 +320,9 @@ under the License.
** xref:workflow/create-workflow.adoc[Create a Workflow]
** xref:workflow/run-debug-workflow.adoc[Run and Debug a Workflow]
** xref:workflow/workflow-run-configurations/workflow-run-configurations.adoc[Workflow Run Configurations]
+*** xref:workflow/workflow-run-configurations/native-local-workflow-engine.adoc[Native Local]
+*** xref:workflow/workflow-run-configurations/native-remote-workflow-engine.adoc[Native Remote]
+*** xref:workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc[Native Load-balancing]
** xref:workflow/actions.adoc[Actions]
//::=START AUTO GENERATED LINKS ACTIONS
*** xref:workflow/actions/abort.adoc[Abort workflow]
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-server/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-server/index.adoc
index e36f777530..b182275f0b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-server/index.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-server/index.adoc
@@ -20,7 +20,7 @@ under the License.
= Hop Server
-Hop Server is a lightweight server to run workflows and pipelines with the xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Remote pipeline] or xref:workflow/workflow-run-configurations/native-remote-workflow-engine.adoc[Remote workflow] run configurations.
+Hop Server is a lightweight server to run workflows and pipelines with the xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Remote pipeline], xref:workflow/workflow-run-configurations/native-remote-workflow-engine.adoc[Remote workflow], or xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[load-balancing] run configurations.
How you get a project onto a Hop Server is covered in xref:hop-server/deploying.adoc[Deploying Hop Server].
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-server/rest-api.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-server/rest-api.adoc
index 197541c636..037b687e4a 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-server/rest-api.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-server/rest-api.adoc
@@ -377,6 +377,7 @@ GET `hop/registerPipeline`
parameters::
* xml (optional): return the result as xml (default HTML) use `&xml=Y`
* json (optional): return the result as json (default HTML) use `&json=Y`
+* max_concurrent (optional): refuse the registration with result `SERVER_AT_CAPACITY` when the number of unfinished pipelines plus workflows on this server is already at this value. Used by the load-balancing engines.
The request body should contain the xml of a `pipeline_configuration` (a `pipeline` and a `pipeline_execution_configuration` wrapped in a `pipeline_configuration` tag).
@@ -579,6 +580,7 @@ GET `/hop/registerWorkflow`
parameters::
* xml (optional): return the result as xml (default HTML) use `&xml=Y`
* json (optional): return the result as json (default HTML) use `&json=Y`
+* max_concurrent (optional): refuse the registration with result `SERVER_AT_CAPACITY` when the number of unfinished pipelines plus workflows on this server is already at this value. Used by the load-balancing engines.
The request body should contain the xml of a `workflow_configuration` (a `workflow` and a `workflow_execution_configuration` wrapped in a `workflow_configuration` tag), in the same form as the <<Register Pipeline,`registerPipeline`>> payload.
diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/pipeline-run-config.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/pipeline-run-config.adoc
index 0b1f4f3825..7a5d6da8ed 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/pipeline-run-config.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/pipeline-run-config.adoc
@@ -52,6 +52,7 @@ Detailed configuration depends on the xref:pipeline/pipeline-run-configurations/
* *xref:pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc[Beam Spark pipeline engine]*
* *xref:pipeline/pipeline-run-configurations/native-local-pipeline-engine.adoc[Hop local pipeline engine]*
* *xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Hop remote pipeline engine]*
+* *xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[Hop load-balancing pipeline engine]*
|===
== Samples
diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/workflow-run-config.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/workflow-run-config.adoc
index db9429f9f3..561a501ca9 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/workflow-run-config.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/workflow-run-config.adoc
@@ -42,7 +42,7 @@ None/All actions
|Name|the name to use for this workflow run configuration
|Description|the description to use for this workflow run configuration
|Execution information location|The xref:metadata-types/execution-information-location.adoc[] to use with this workflow run configuration.
-|Workflow engine type|Hop local workflow engine or Hop remote workflow engine
+|Workflow engine type|Hop local, remote, or load-balancing workflow engine
|Safe mode|(default: false) In safe mode, Hop performs a number of data type and other validation before starting the workflow execution.
|===
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc
new file mode 100644
index 0000000000..b59141ccb9
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc
@@ -0,0 +1,130 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+[[LoadBalancingPipelineEngine]]
+:imagesdir: ../assets/images
+:openvar: ${
+:closevar: }
+:description: The load-balancing pipeline engine assigns each pipeline execution to one Hop Server from a configured group.
+
+= Load-balancing Pipeline Engine
+
+The load-balancing run configuration assigns *one entire pipeline execution* to one Hop Server from a group.
+
+It does not split a pipeline's transforms across machines.
+To process batches of rows on the pool, run a local pipeline that uses xref:pipeline/transforms/pipeline-executor.adoc[Pipeline Executor] with this run configuration.
+
+Each Hop Server in the group still needs a xref:metadata-types/hop-server.adoc[Hop Server metadata] definition.
+
+image:run-configuration/load-balance-pipeline-run-configuration-editor.png[Load-balancing pipeline run configuration editor]
+
+=== Options
+
+The engine-specific options sit on three tabs under the selected engine type.
+
+==== Management
+
+[options="header",cols="1,3"]
+|===
+|Option|Description
+
+|Server poll delay / interval
+|Same meaning as on the xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[remote pipeline engine].
+
+|Export linked resources
+|Same meaning as on the remote pipeline engine.
+
+|===
+
+==== Load-balancing
+
+[options="header",cols="1,3"]
+|===
+|Option|Description
+
+|Run Configuration
+|The configuration to use *on* the selected Hop Server (usually a local engine on that server).
+Do not point this back at the load-balancing configuration or you get a cycle.
+
+|Distribution algorithm
+|*Even load* targets the same utilization on every eligible server.
+*Keep as few servers busy as possible* (pack) fills busy servers first so idle ones can be disabled and turned off.
+
+|Maximum retries
+|How many extra times to try another server after a submit or capacity failure.
+
+|Retry window (ms)
+|Stop retrying after this many milliseconds from the first attempt.
+`0` means retry count only.
+
+|Retry failed executions
+|Off by default.
+When enabled, also retry if the remote pipeline runs and fails (side effects possible).
+
+|Server probe timeout (ms)
+|How long to wait for each server status check.
+Default `3000`.
+
+|Configuration refresh interval (ms)
+|How often to re-read this run configuration from metadata so enable/disable and max concurrent changes take effect without restarting the dispatcher.
+Default `10000`.
+
+|Assignment state folder
+|Optional VFS folder for assignment ledger files, for example `{openvar}PROJECT_HOME{closevar}/loadbalance`.
+Leave empty to skip files.
+Results are also written to the run configuration's xref:metadata-types/execution-information-location.adoc[execution information location] when one is set.
+
+|===
+
+==== Hop Servers
+
+[options="header",cols="1,3"]
+|===
+|Option|Description
+
+|Hop servers
+|Table of workers: Hop Server name, whether it may receive new work, and the maximum number of unfinished pipelines *and* workflows on that server.
+
+|===
+
+=== How a server is chosen
+
+. The engine re-reads this run configuration from metadata.
+. It probes every enabled server (status, load, occupying executions, graceful shutdown).
+. Servers that are down, shutting down, disabled, or already at max concurrent are skipped.
+. The algorithm picks one eligible server.
+. The pipeline is registered with `max_concurrent` so the server itself refuses the work if it is already at capacity (`SERVER_AT_CAPACITY`).
+. On capacity or connectivity failure the engine retries another server until the retry budget is exhausted.
+
+=== Draining a server
+
+. Set *Enabled* to `N` in this table (picked up on the next configuration refresh), and/or
+. Shut the Hop Server down gracefully so it reports `shuttingDown` and refuses new work.
+
+Use the pack algorithm if you want idle machines to stay empty so they can be turned off.
+
+=== Kubernetes
+
+* Hop-aware pool: one Hop Server metadata object per pod (or StatefulSet DNS name).
+ Then even-load, pack, disable, and max concurrent work as described above.
+* A Kubernetes Service in front of several pods as a *single* remote Hop Server is still possible.
+ That is L4 round-robin from kube-proxy, not Hop-aware load balancing.
+
+=== See also
+
+* xref:workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc[Load-balancing workflow engine]
+* xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Remote pipeline engine]
+* xref:hop-server/index.adoc[Hop Server]
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/pipeline-run-configurations.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/pipeline-run-configurations.adoc
index 660797a34e..ae20816900 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/pipeline-run-configurations.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-run-configurations/pipeline-run-configurations.adoc
@@ -58,6 +58,7 @@ The available engine types are:
* *xref:pipeline/pipeline-run-configurations/beam-spark-pipeline-engine.adoc[Beam Spark pipeline engine]*: this configuration runs pipelines on Apache Spark over Apache Beam
* *xref:pipeline/pipeline-run-configurations/native-local-pipeline-engine.adoc[Hop local pipeline engine]*: this configuration runs pipelines locally in the native Hop engine
* *xref:pipeline/pipeline-run-configurations/native-remote-pipeline-engine.adoc[Hop remote pipeline engine]*: this configuration runs pipelines in the native Hop engine on a remote machine
+* *xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[Hop load-balancing pipeline engine]*: this configuration assigns each pipeline execution to one Hop Server from a group
* *xref:pipeline/pipeline-run-configurations/native-spark-pipeline-engine.adoc[Native Spark pipeline engine]*: this configuration runs batch pipelines on Apache Spark 4.x *without* Beam (see xref:pipeline/spark/getting-started-with-native-spark.adoc[getting started])
* *xref:pipeline/pipeline-run-configurations/single-threaded-pipeline-engine.adoc[Hop local single threaded pipeline engine]*: this configuration runs pipelines locally in a single-threaded fashion
|===
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/serverstatus.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/serverstatus.adoc
index 701dc9f7ce..6ed07ae573 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/serverstatus.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/serverstatus.adoc
@@ -58,6 +58,8 @@ Here is the list of possible output fields:
* OS version
* OS architecture
* # Active pipelines
+* # Finished pipelines
* # Active workflows
+* # Finished workflows
* Available
* Response (ns)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc b/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc
new file mode 100644
index 0000000000..3d5f9a937d
--- /dev/null
+++ b/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc
@@ -0,0 +1,44 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+[[LoadBalancingWorkflowEngine]]
+:imagesdir: ../../assets/images
+:openvar: ${
+:closevar: }
+:description: The load-balancing workflow engine assigns each workflow execution to one Hop Server from a configured group.
+
+= Load-balancing Workflow Engine
+
+The load-balancing run configuration assigns *one entire workflow execution* to one Hop Server from a group.
+
+It uses the same server table, algorithms, retry policy, admission control, and assignment ledger as the xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[load-balancing pipeline engine].
+
+Each Hop Server in the group still needs a xref:metadata-types/hop-server.adoc[Hop Server metadata] definition.
+
+image:run-configuration/load-balance-workflow-run-configuration-editor.png[Load-balancing workflow run configuration editor]
+
+The engine-specific options sit on the same three tabs as the pipeline engine: *Management* (poll and export), *Load-balancing* (run configuration, algorithm, retry, state folder), and *Hop Servers*.
+
+The run configuration to use *on* the selected server is usually a local workflow engine on that server.
+Do not point it back at this load-balancing configuration.
+
+Max concurrent on each server counts unfinished pipelines *and* workflows, so a mixed pool shares one cap.
+
+=== See also
+
+* xref:pipeline/pipeline-run-configurations/native-load-balancing-pipeline-engine.adoc[Load-balancing pipeline engine]
+* xref:workflow/workflow-run-configurations/native-remote-workflow-engine.adoc[Remote workflow engine]
+* xref:hop-server/index.adoc[Hop Server]
diff --git a/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/workflow-run-configurations.adoc b/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/workflow-run-configurations.adoc
index 7eb5510255..352dc9690f 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/workflow-run-configurations.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/workflow/workflow-run-configurations/workflow-run-configurations.adoc
@@ -52,4 +52,5 @@ The available engine types are:
* xref:workflow/workflow-run-configurations/native-local-workflow-engine.adoc[Local workflow engine]: this configuration runs pipelines on your machine
* xref:workflow/workflow-run-configurations/native-remote-workflow-engine.adoc[Remote workflow engine]: this configuration runs workflows on a Hop Server using a defined Run Configuration.
+* xref:workflow/workflow-run-configurations/native-load-balancing-workflow-engine.adoc[Load-balancing workflow engine]: this configuration assigns each workflow execution to one Hop Server from a group.
|===
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
new file mode 100644
index 0000000000..c31dbc38e3
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngine.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.engines.loadbalance;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.IExecutionInfoLocation;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
+import org.apache.hop.pipeline.engine.PipelineEnginePlugin;
+import org.apache.hop.pipeline.engines.remote.RemotePipelineEngine;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
+import org.apache.hop.server.loadbalance.LoadBalancingAlgorithm;
+import org.apache.hop.server.loadbalance.LoadBalancingAssignment;
+import org.apache.hop.server.loadbalance.LoadBalancingCoordinator;
+import org.apache.hop.server.loadbalance.LoadBalancingRetryPolicy;
+import org.apache.hop.server.loadbalance.ServerHealthSnapshot;
+import org.apache.hop.www.HopServerAdmission;
+
+@PipelineEnginePlugin(
+ id = "LoadBalancing",
+ name = "Hop load-balancing pipeline engine",
+ description = "Assigns the pipeline to one Hop server from a configured group")
+public class LoadBalancingPipelineEngine extends RemotePipelineEngine {
+
+ public static final String DETAIL_ASSIGNED_SERVER = "assignedServer";
+ public static final String DETAIL_ATTEMPT = "attempt";
+ public static final String DETAIL_ALGORITHM = "algorithm";
+ public static final String DETAIL_CONTAINER_ID = "containerId";
+ public static final String DETAIL_LOAD_AT_ASSIGNMENT = "loadAtAssignment";
+
+ private LoadBalancingCoordinator<LoadBalancingPipelineRunConfiguration> coordinator;
+ private LoadBalancingAssignment assignment;
+ private ExecutionInfoLocation executionInfoLocation;
+
+ public LoadBalancingPipelineEngine() {
+ super();
+ }
+
+ public LoadBalancingPipelineEngine(PipelineMeta subject) {
+ super(subject);
+ }
+
+ @Override
+ public IPipelineEngineRunConfiguration createDefaultPipelineEngineRunConfiguration() {
+ return new LoadBalancingPipelineRunConfiguration();
+ }
+
+ @Override
+ public void prepareExecution() throws HopException {
+ IPipelineEngineRunConfiguration engineRunConfiguration =
+ pipelineRunConfiguration.getEngineRunConfiguration();
+ if (!(engineRunConfiguration instanceof ILoadBalancingRunConfiguration)) {
+ throw new HopException(
+ "The load-balancing pipeline engine expects a load-balancing pipeline configuration");
+ }
+
+ this.logChannel = new LogChannel(this, getParent());
+ this.logChannel.setLogLevel(logLevel);
+
+ coordinator =
+ new LoadBalancingCoordinator<>(
+ this, metadataProvider, logChannel, pipelineRunConfiguration.getName(), false);
+ coordinator.reload();
+ validateRunConfigurationChain(pipelineRunConfiguration);
+
+ LoadBalancingRetryPolicy retry =
+ LoadBalancingRetryPolicy.from(coordinator.getCurrentConfig(), this);
+ Exception lastError = null;
+ while (retry.canAttempt()) {
+ int attempt = retry.beginAttempt();
+ containerId = null;
+ hopServer = null;
+ try {
+ ServerHealthSnapshot snapshot = coordinator.selectServer();
+ selectedHopServerName = snapshot.getHopServerName();
+ admissionMaxConcurrent = snapshot.getMaxConcurrent();
+ assignment = newAssignment(snapshot, attempt, retry);
+ coordinator.saveAssignment(assignment);
+
+ logChannel.logBasic(
+ "Executing this pipeline using the Load-balancing Pipeline Engine with run configuration '"
+ + pipelineRunConfiguration.getName()
+ + "', attempt "
+ + attempt
+ + " on server '"
+ + selectedHopServerName
+ + "'");
+
+ super.prepareExecution();
+
+ assignment.setExecutionId(getLogChannelId());
+ assignment.setStatus(LoadBalancingAssignment.STATUS_RUNNING);
+ assignment.setContainerId(containerId);
+ coordinator.saveAssignment(assignment);
+ registerLoadBalancingExecutionInformation(snapshot, attempt);
+ return;
+ } catch (Exception e) {
+ lastError = e;
+ if (assignment != null) {
+ assignment.setLastError(e.getMessage());
+ assignment.setStatus(LoadBalancingAssignment.STATUS_RETRYING);
+ coordinator.saveAssignment(assignment);
+ }
+ if (HopServerAdmission.isRetryableRegistrationFailure(e, containerId)
+ && retry.canAttempt()) {
+ logCapacityWait(attempt, e);
+ sleepBackoff(retry);
+ continue;
+ }
+ if (assignment != null) {
+ assignment.setStatus(LoadBalancingAssignment.STATUS_FAILED);
+ coordinator.saveAssignment(assignment);
+ }
+ throw new HopException("Error preparing load-balanced pipeline", e);
+ }
+ }
+ if (assignment != null) {
+ assignment.setStatus(LoadBalancingAssignment.STATUS_FAILED);
+ coordinator.saveAssignment(assignment);
+ }
+ throw new HopException(
+ "Load-balancing gave up after " + retry.getAttempt() + " attempt(s)", lastError);
+ }
+
+ @Override
+ public void startThreads() throws HopException {
+ try {
+ super.startThreads();
+ addExecutionFinishedListener(engine -> updateFinishedAssignmentAndState());
+ } catch (HopException e) {
+ if (assignment != null) {
+ assignment.setStatus(LoadBalancingAssignment.STATUS_FAILED);
+ assignment.setLastError(e.getMessage());
+ coordinator.saveAssignment(assignment);
+ }
+ throw e;
+ }
+ }
+
+ private LoadBalancingAssignment newAssignment(
+ ServerHealthSnapshot snapshot, int attempt, LoadBalancingRetryPolicy retry) {
+ LoadBalancingAssignment next = new LoadBalancingAssignment();
+ next.setExecutionId(getLogChannelId());
+ next.setRunConfigurationName(pipelineRunConfiguration.getName());
+ next.setExecutorName(subject == null ? null : subject.getName());
+ next.setExecutorType("pipeline");
+ next.setServerName(snapshot.getHopServerName());
+ next.setAttempt(attempt);
+ next.setFirstAttemptEpochMs(retry.getFirstAttemptEpochMs());
+ next.setStatus(LoadBalancingAssignment.STATUS_ASSIGNING);
+ next.setAlgorithm(
+ LoadBalancingAlgorithm.fromCodeOrDescription(
+ resolve(coordinator.getCurrentConfig().getAlgorithm()))
+ .getCode());
+ next.setOccupyingSlotsAtAssignment(snapshot.getOccupyingSlots());
+ next.setMaxConcurrent(snapshot.getMaxConcurrent());
+ return next;
+ }
+
+ private void logCapacityWait(int attempt, Exception error) {
+ String server =
+ StringUtils.isEmpty(selectedHopServerName)
+ ? "the load-balancing group"
+ : selectedHopServerName;
+ logChannel.logBasic(
+ "Load-balancing attempt "
+ + attempt
+ + " waiting for a free slot on "
+ + server
+ + ": "
+ + error.getMessage());
+ }
+
+ private void sleepBackoff(LoadBalancingRetryPolicy retry) {
+ long waitMs = Math.min(1000L, retry.remainingWindowMs());
+ if (waitMs <= 0) {
+ return;
+ }
+ try {
+ Thread.sleep(waitMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void registerLoadBalancingExecutionInformation(
+ ServerHealthSnapshot snapshot, int attempt) {
+ try {
+ lookupExecutionInformationLocation();
+ if (executionInfoLocation == null) {
+ return;
+ }
+ IExecutionInfoLocation location = executionInfoLocation.getExecutionInfoLocation();
+ location.registerExecution(ExecutionBuilder.fromExecutor(this).build());
+ ExecutionState state = ExecutionStateBuilder.fromExecutor(this, -1).build();
+ Map<String, String> details =
+ state.getDetails() == null ? new HashMap<>() : state.getDetails();
+ details.put(DETAIL_ASSIGNED_SERVER, selectedHopServerName);
+ details.put(DETAIL_ATTEMPT, Integer.toString(attempt));
+ details.put(
+ DETAIL_ALGORITHM,
+ LoadBalancingAlgorithm.fromCodeOrDescription(
+ resolve(coordinator.getCurrentConfig().getAlgorithm()))
+ .getCode());
+ details.put(DETAIL_CONTAINER_ID, containerId);
+ details.put(
+ DETAIL_LOAD_AT_ASSIGNMENT,
+ snapshot.getOccupyingSlots() + "/" + snapshot.getMaxConcurrent());
+ state.setDetails(details);
+ location.updateExecutionState(state);
+ } catch (Exception e) {
+ logChannel.logError("Error registering load-balancing execution information (non-fatal)", e);
+ }
+ }
+
+ private void lookupExecutionInformationLocation() throws HopException {
+ if (executionInfoLocation != null || pipelineRunConfiguration == null) {
+ return;
+ }
+ String locationName = resolve(pipelineRunConfiguration.getExecutionInfoLocationName());
+ if (StringUtils.isEmpty(locationName) || metadataProvider == null) {
+ return;
+ }
+ ExecutionInfoLocation location =
+ metadataProvider.getSerializer(ExecutionInfoLocation.class).load(locationName);
+ if (location == null) {
+ logChannel.logError(
+ "Execution information location '" + locationName + "' could not be found");
+ return;
+ }
+ location.getExecutionInfoLocation().initialize(this, metadataProvider);
+ executionInfoLocation = location;
+ }
+
+ private void updateFinishedAssignmentAndState() {
+ try {
+ if (assignment != null && coordinator != null) {
+ assignment.setStatus(
+ getErrors() > 0
+ ? LoadBalancingAssignment.STATUS_FAILED
+ : LoadBalancingAssignment.STATUS_FINISHED);
+ assignment.setContainerId(containerId);
+ coordinator.saveAssignment(assignment);
+ }
+ if (executionInfoLocation != null) {
+ IExecutionInfoLocation location = executionInfoLocation.getExecutionInfoLocation();
+ ExecutionState state = ExecutionStateBuilder.fromExecutor(this, -1).build();
+ Map<String, String> details =
+ state.getDetails() == null ? new HashMap<>() : state.getDetails();
+ if (assignment != null) {
+ details.put(DETAIL_ASSIGNED_SERVER, assignment.getServerName());
+ details.put(DETAIL_ATTEMPT, Integer.toString(assignment.getAttempt()));
+ details.put(DETAIL_ALGORITHM, assignment.getAlgorithm());
+ details.put(DETAIL_CONTAINER_ID, assignment.getContainerId());
+ }
+ state.setDetails(details);
+ location.updateExecutionState(state);
+ location.close();
+ }
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error updating load-balancing assignment state (non-fatal)", e);
+ }
+ }
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineRunConfiguration.java b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineRunConfiguration.java
new file mode 100644
index 0000000000..662136afc0
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineRunConfiguration.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.engines.loadbalance;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engines.EmptyPipelineRunConfiguration;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
+import org.apache.hop.server.loadbalance.LoadBalancingAlgorithm;
+import org.apache.hop.server.loadbalance.LoadBalancingServerEntry;
+
+@GuiPlugin(description = "Load-balancing pipeline run configuration widgets")
+@Getter
+@Setter
+public class LoadBalancingPipelineRunConfiguration extends EmptyPipelineRunConfiguration
+ implements IPipelineEngineRunConfiguration, ILoadBalancingRunConfiguration {
+
+ @GuiWidgetElement(
+ order = "20",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.METADATA,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.RunConfiguration.Label",
+ metadata = PipelineRunConfiguration.class)
+ @HopMetadataProperty(
+ key = "run_config",
+ hopMetadataPropertyType = HopMetadataPropertyType.PIPELINE_RUN_CONFIG)
+ protected String runConfigurationName;
+
+ @GuiWidgetElement(
+ order = "30",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.COMBO,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingAlgorithm.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingAlgorithm.ToolTip",
+ comboValuesMethod = "getAlgorithms")
+ @HopMetadataProperty(key = "algorithm")
+ protected String algorithm;
+
+ @GuiWidgetElement(
+ order = "40",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingMaxRetries.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingMaxRetries.ToolTip")
+ @HopMetadataProperty(key = "max_retries")
+ protected String maxRetries;
+
+ @GuiWidgetElement(
+ order = "50",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingRetryWindow.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingRetryWindow.ToolTip")
+ @HopMetadataProperty(key = "retry_window_ms")
+ protected String retryWindowMs;
+
+ @GuiWidgetElement(
+ order = "60",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.CHECKBOX,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingRetryOnFailure.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingRetryOnFailure.ToolTip")
+ @HopMetadataProperty(key = "retry_on_execution_failure")
+ protected boolean retryOnExecutionFailure;
+
+ @GuiWidgetElement(
+ order = "70",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingProbeTimeout.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingProbeTimeout.ToolTip")
+ @HopMetadataProperty(key = "probe_timeout_ms")
+ protected String probeTimeoutMs;
+
+ @GuiWidgetElement(
+ order = "80",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingConfigRefresh.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingConfigRefresh.ToolTip")
+ @HopMetadataProperty(key = "config_refresh_ms")
+ protected String configRefreshIntervalMs;
+
+ @GuiWidgetElement(
+ order = "90",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.FOLDER,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingStateFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.LoadBalancingStateFolder.ToolTip")
+ @HopMetadataProperty(key = "state_folder")
+ protected String stateFolder;
+
+ @GuiWidgetElement(
+ order = "100",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.ServerPollDelay.Label")
+ @HopMetadataProperty(key = "server_poll_delay")
+ protected String serverPollDelay;
+
+ @GuiWidgetElement(
+ order = "110",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.ServerPollInterval.Label")
+ @HopMetadataProperty(key = "server_poll_interval")
+ protected String serverPollInterval;
+
+ @GuiWidgetElement(
+ order = "120",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.CHECKBOX,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.ExportResources.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.ExportResources.ToolTip")
+ @HopMetadataProperty(key = "export_resources")
+ protected boolean exportingResources;
+
+ @GuiWidgetElement(
+ order = "130",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.NamedResourceSourceFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.NamedResourceSourceFolder.ToolTip")
+ @HopMetadataProperty(key = "resources_source_folder")
+ protected String namedResourcesSourceFolder;
+
+ @GuiWidgetElement(
+ order = "140",
+ parentId = PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.NamedResourceTargetFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.pipeline.config:PipelineRunConfigurationDialog.NamedResourceTargetFolder.ToolTip")
+ @HopMetadataProperty(key = "resources_target_folder")
+ protected String namedResourcesTargetFolder;
+
+ @HopMetadataProperty(key = "server", groupKey = "servers")
+ protected List<LoadBalancingServerEntry> servers;
+
+ /** Runtime-only: last selected server. Not stored in metadata. */
+ protected String hopServerName;
+
+ public LoadBalancingPipelineRunConfiguration() {
+ super();
+ this.servers = new ArrayList<>();
+ this.algorithm = LoadBalancingAlgorithm.EVEN_LOAD.getDescription();
+ this.maxRetries = "2";
+ this.retryWindowMs = "0";
+ this.probeTimeoutMs = "3000";
+ this.configRefreshIntervalMs = "10000";
+ }
+
+ public LoadBalancingPipelineRunConfiguration(LoadBalancingPipelineRunConfiguration config) {
+ super(config);
+ this.runConfigurationName = config.runConfigurationName;
+ this.algorithm = config.algorithm;
+ this.maxRetries = config.maxRetries;
+ this.retryWindowMs = config.retryWindowMs;
+ this.retryOnExecutionFailure = config.retryOnExecutionFailure;
+ this.probeTimeoutMs = config.probeTimeoutMs;
+ this.configRefreshIntervalMs = config.configRefreshIntervalMs;
+ this.stateFolder = config.stateFolder;
+ this.serverPollDelay = config.serverPollDelay;
+ this.serverPollInterval = config.serverPollInterval;
+ this.exportingResources = config.exportingResources;
+ this.namedResourcesSourceFolder = config.namedResourcesSourceFolder;
+ this.namedResourcesTargetFolder = config.namedResourcesTargetFolder;
+ this.hopServerName = config.hopServerName;
+ this.servers = new ArrayList<>();
+ if (config.servers != null) {
+ for (LoadBalancingServerEntry entry : config.servers) {
+ this.servers.add(new LoadBalancingServerEntry(entry));
+ }
+ }
+ }
+
+ @Override
+ public LoadBalancingPipelineRunConfiguration clone() {
+ return new LoadBalancingPipelineRunConfiguration(this);
+ }
+
+ public List<String> getAlgorithms(ILogChannel log, IHopMetadataProvider metadataProvider) {
+ List<String> list = new ArrayList<>();
+ for (LoadBalancingAlgorithm value : LoadBalancingAlgorithm.values()) {
+ list.add(value.getDescription());
+ }
+ return list;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java
index 5f44e4df78..3fd301fa46 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java
@@ -83,8 +83,10 @@ import org.apache.hop.pipeline.transform.TransformStatus;
import org.apache.hop.resource.ResourceUtil;
import org.apache.hop.resource.TopLevelResource;
import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.server.IRemoteCapableRunConfiguration;
import org.apache.hop.workflow.WorkflowMeta;
import org.apache.hop.workflow.engine.IWorkflowEngine;
+import org.apache.hop.www.HopServerAdmission;
import org.apache.hop.www.HopServerPipelineStatus;
import org.apache.hop.www.PrepareExecutionPipelineServlet;
import org.apache.hop.www.RegisterPackageServlet;
@@ -113,7 +115,7 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
protected PipelineMeta subject;
protected String pluginId;
protected PipelineRunConfiguration pipelineRunConfiguration;
- protected RemotePipelineRunConfiguration remotePipelineRunConfiguration;
+ protected IRemoteCapableRunConfiguration remotePipelineRunConfiguration;
protected boolean preparing;
protected boolean readyToStart;
protected boolean running;
@@ -172,6 +174,12 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
protected long serverPollDelay;
protected long serverPollInterval;
+ /** Set by a load-balancing engine after it picks a server; otherwise the config name is used. */
+ protected String selectedHopServerName;
+
+ /** Optional server-side admission cap sent as {@code max_concurrent} on register. */
+ protected int admissionMaxConcurrent;
+
public RemotePipelineEngine() {
super();
logChannel = LogChannel.GENERAL;
@@ -208,14 +216,14 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
* @param runConfiguration the remote run configuration to start from
* @throws HopException when the chain leads back to a run configuration it already passed
*/
- private void validateRunConfigurationChain(PipelineRunConfiguration runConfiguration)
+ protected void validateRunConfigurationChain(PipelineRunConfiguration runConfiguration)
throws HopException {
List<String> chain = new ArrayList<>();
chain.add(runConfiguration.getName());
PipelineRunConfiguration current = runConfiguration;
while (current != null
- && current.getEngineRunConfiguration() instanceof RemotePipelineRunConfiguration remote) {
+ && current.getEngineRunConfiguration() instanceof IRemoteCapableRunConfiguration remote) {
String linkedName = resolve(remote.getRunConfigurationName());
if (StringUtils.isEmpty(linkedName)) {
// Reported for the run configuration this engine was asked to run with.
@@ -240,14 +248,13 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
try {
IPipelineEngineRunConfiguration engineRunConfiguration =
pipelineRunConfiguration.getEngineRunConfiguration();
- if (!(engineRunConfiguration instanceof RemotePipelineRunConfiguration)) {
+ if (!(engineRunConfiguration instanceof IRemoteCapableRunConfiguration remoteCapable)) {
throw new HopException(
"The remote pipeline engine expects a remote pipeline configuration");
}
- remotePipelineRunConfiguration =
- (RemotePipelineRunConfiguration) pipelineRunConfiguration.getEngineRunConfiguration();
+ remotePipelineRunConfiguration = remoteCapable;
- String hopServerName = resolve(remotePipelineRunConfiguration.getHopServerName());
+ String hopServerName = resolveTargetHopServerName();
if (StringUtils.isEmpty(hopServerName)) {
throw new HopException("No remote Hop server was specified to run the pipeline on");
}
@@ -305,6 +312,16 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
}
}
+ protected String resolveTargetHopServerName() {
+ if (StringUtils.isNotEmpty(selectedHopServerName)) {
+ return resolve(selectedHopServerName);
+ }
+ if (remotePipelineRunConfiguration == null) {
+ return null;
+ }
+ return resolve(remotePipelineRunConfiguration.getHopServerName());
+ }
+
/**
* Send the pipeline for execution to a HopServer hop server.
*
@@ -383,7 +400,8 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
this,
topLevelResource.getArchiveName(),
RegisterPackageServlet.TYPE_PIPELINE,
- topLevelResource.getBaseResourceName());
+ topLevelResource.getBaseResourceName(),
+ admissionMaxConcurrent);
WebResult webResult = WebResult.fromXmlString(result);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
String message = cleanupMessage(webResult.getMessage());
@@ -406,7 +424,12 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine<P
pipelineMeta, executionConfiguration, serializableMetadataProvider)
.getXml(this);
String reply =
- hopServer.sendXml(this, xml, RegisterPipelineServlet.CONTEXT_PATH + "/?xml=Y");
+ hopServer.sendXml(
+ this,
+ xml,
+ RegisterPipelineServlet.CONTEXT_PATH
+ + "/?xml=Y"
+ + HopServerAdmission.querySuffix(admissionMaxConcurrent));
WebResult webResult = WebResult.fromXmlString(reply);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
String message = cleanupMessage(webResult.getMessage());
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineRunConfiguration.java b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineRunConfiguration.java
index f6d7644f27..a7b88df8ed 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineRunConfiguration.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineRunConfiguration.java
@@ -28,12 +28,13 @@ import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.pipeline.engines.EmptyPipelineRunConfiguration;
import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.server.IRemoteCapableRunConfiguration;
@GuiPlugin(description = "Remote pipeline run configuration widgets")
@Getter
@Setter
public class RemotePipelineRunConfiguration extends EmptyPipelineRunConfiguration
- implements IPipelineEngineRunConfiguration {
+ implements IPipelineEngineRunConfiguration, IRemoteCapableRunConfiguration {
@GuiWidgetElement(
order = "10",
diff --git a/engine/src/main/java/org/apache/hop/server/IRemoteCapableRunConfiguration.java b/engine/src/main/java/org/apache/hop/server/IRemoteCapableRunConfiguration.java
new file mode 100644
index 0000000000..1e6986e359
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/IRemoteCapableRunConfiguration.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server;
+
+/**
+ * Run-configuration options shared by the remote and load-balancing engines: a Hop Server (or one
+ * chosen at runtime), the run configuration to use on that server, poll timings, and resource
+ * export.
+ */
+public interface IRemoteCapableRunConfiguration {
+
+ String getHopServerName();
+
+ void setHopServerName(String hopServerName);
+
+ String getRunConfigurationName();
+
+ void setRunConfigurationName(String runConfigurationName);
+
+ String getServerPollDelay();
+
+ String getServerPollInterval();
+
+ boolean isExportingResources();
+
+ String getNamedResourcesSourceFolder();
+
+ String getNamedResourcesTargetFolder();
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/HopServerProbe.java b/engine/src/main/java/org/apache/hop/server/loadbalance/HopServerProbe.java
new file mode 100644
index 0000000000..339b2b45bf
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/HopServerProbe.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.www.HopServerPipelineStatus;
+import org.apache.hop.www.HopServerStatus;
+import org.apache.hop.www.HopServerWorkflowStatus;
+import org.apache.hop.www.RemoteHopServer;
+
+/** Probes a Hop Server for health, load, and occupying execution slots. */
+public final class HopServerProbe {
+
+ private HopServerProbe() {}
+
+ public static ServerHealthSnapshot probe(
+ HopServerMeta serverMeta, IVariables variables, boolean enabled, int maxConcurrent) {
+ ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
+ snapshot.setHopServerName(serverMeta == null ? null : serverMeta.getName());
+ snapshot.setEnabled(enabled);
+ snapshot.setMaxConcurrent(maxConcurrent);
+ if (serverMeta == null) {
+ snapshot.setAvailable(false);
+ snapshot.setErrorMessage("Hop server metadata is missing");
+ return snapshot;
+ }
+
+ long startNs = System.nanoTime();
+ try {
+ RemoteHopServer server = new RemoteHopServer(serverMeta);
+ HopServerStatus status = server.requestServerStatus(variables);
+ applyStatus(snapshot, status);
+ snapshot.setAvailable(true);
+ } catch (Exception e) {
+ snapshot.setAvailable(false);
+ snapshot.setErrorMessage("Error querying Hop server : " + e.getMessage());
+ } finally {
+ snapshot.setResponseNs(System.nanoTime() - startNs);
+ }
+ return snapshot;
+ }
+
+ public static ServerHealthSnapshot probeWithTimeout(
+ HopServerMeta serverMeta,
+ IVariables variables,
+ boolean enabled,
+ int maxConcurrent,
+ long timeoutMs) {
+ if (timeoutMs <= 0) {
+ return probe(serverMeta, variables, enabled, maxConcurrent);
+ }
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<ServerHealthSnapshot> future =
+ executor.submit(() -> probe(serverMeta, variables, enabled, maxConcurrent));
+ return future.get(timeoutMs, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ return ServerHealthSnapshot.unavailable(
+ serverMeta == null ? null : serverMeta.getName(),
+ enabled,
+ maxConcurrent,
+ "timeout after " + timeoutMs + "ms");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return ServerHealthSnapshot.unavailable(
+ serverMeta == null ? null : serverMeta.getName(),
+ enabled,
+ maxConcurrent,
+ "probe interrupted");
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause() == null ? e : e.getCause();
+ return ServerHealthSnapshot.unavailable(
+ serverMeta == null ? null : serverMeta.getName(),
+ enabled,
+ maxConcurrent,
+ "Error querying Hop server : " + cause.getMessage());
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ static void applyStatus(ServerHealthSnapshot snapshot, HopServerStatus status) {
+ snapshot.setStatusDescription(status.getStatusDescription());
+ snapshot.setShuttingDown(status.isShuttingDown());
+ snapshot.setLoadAvg(status.getLoadAvg());
+ snapshot.setMemoryFree(status.getMemoryFree());
+ snapshot.setMemoryTotal(status.getMemoryTotal());
+ snapshot.setCpuCores(status.getCpuCores());
+ snapshot.setCpuProcessTime(status.getCpuProcessTime());
+ snapshot.setOsName(status.getOsName());
+ snapshot.setOsVersion(status.getOsVersion());
+ snapshot.setOsArchitecture(status.getOsArchitecture());
+
+ int runningPipelines = 0;
+ int finishedPipelines = 0;
+ int occupyingPipelines = 0;
+ if (status.getPipelineStatusList() != null) {
+ for (HopServerPipelineStatus pipelineStatus : status.getPipelineStatusList()) {
+ if (pipelineStatus.isRunning()) {
+ runningPipelines++;
+ }
+ if (pipelineStatus.isFinished()) {
+ finishedPipelines++;
+ }
+ if (occupiesPipelineSlot(pipelineStatus)) {
+ occupyingPipelines++;
+ }
+ }
+ }
+ int runningWorkflows = 0;
+ int finishedWorkflows = 0;
+ int occupyingWorkflows = 0;
+ if (status.getWorkflowStatusList() != null) {
+ for (HopServerWorkflowStatus workflowStatus : status.getWorkflowStatusList()) {
+ if (workflowStatus.isRunning()) {
+ runningWorkflows++;
+ }
+ if (workflowStatus.isFinished()) {
+ finishedWorkflows++;
+ }
+ if (occupiesWorkflowSlot(workflowStatus)) {
+ occupyingWorkflows++;
+ }
+ }
+ }
+ snapshot.setRunningPipelines(runningPipelines);
+ snapshot.setFinishedPipelines(finishedPipelines);
+ snapshot.setRunningWorkflows(runningWorkflows);
+ snapshot.setFinishedWorkflows(finishedWorkflows);
+ snapshot.setOccupyingSlots(occupyingPipelines + occupyingWorkflows);
+ }
+
+ static boolean occupiesPipelineSlot(HopServerPipelineStatus status) {
+ return status != null && !status.isFinished() && !status.isStopped();
+ }
+
+ static boolean occupiesWorkflowSlot(HopServerWorkflowStatus status) {
+ return status != null && !status.isFinished() && !status.isStopped();
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/ILoadBalancingRunConfiguration.java b/engine/src/main/java/org/apache/hop/server/loadbalance/ILoadBalancingRunConfiguration.java
new file mode 100644
index 0000000000..cf8037bd61
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/ILoadBalancingRunConfiguration.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import java.util.List;
+import org.apache.hop.server.IRemoteCapableRunConfiguration;
+
+/** Engine-specific options for the load-balancing pipeline and workflow run configurations. */
+public interface ILoadBalancingRunConfiguration extends IRemoteCapableRunConfiguration {
+
+ List<LoadBalancingServerEntry> getServers();
+
+ void setServers(List<LoadBalancingServerEntry> servers);
+
+ String getAlgorithm();
+
+ void setAlgorithm(String algorithm);
+
+ String getMaxRetries();
+
+ void setMaxRetries(String maxRetries);
+
+ String getRetryWindowMs();
+
+ void setRetryWindowMs(String retryWindowMs);
+
+ boolean isRetryOnExecutionFailure();
+
+ void setRetryOnExecutionFailure(boolean retryOnExecutionFailure);
+
+ String getProbeTimeoutMs();
+
+ void setProbeTimeoutMs(String probeTimeoutMs);
+
+ String getConfigRefreshIntervalMs();
+
+ void setConfigRefreshIntervalMs(String configRefreshIntervalMs);
+
+ String getStateFolder();
+
+ void setStateFolder(String stateFolder);
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAlgorithm.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAlgorithm.java
new file mode 100644
index 0000000000..01493fafb8
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAlgorithm.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+
+/** How a load-balancing engine picks an eligible Hop Server. */
+public enum LoadBalancingAlgorithm implements IEnumHasCodeAndDescription {
+ EVEN_LOAD("even-load", "Even load"),
+ PACK("pack", "Keep as few servers busy as possible");
+
+ private final String code;
+ private final String description;
+
+ LoadBalancingAlgorithm(String code, String description) {
+ this.code = code;
+ this.description = description;
+ }
+
+ @Override
+ public String getCode() {
+ return code;
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+
+ public static LoadBalancingAlgorithm fromCodeOrDescription(String value) {
+ if (value == null || value.isBlank()) {
+ return EVEN_LOAD;
+ }
+ LoadBalancingAlgorithm byCode =
+ org.apache.hop.metadata.api.IEnumHasCode.lookupCode(
+ LoadBalancingAlgorithm.class, value, null);
+ if (byCode != null) {
+ return byCode;
+ }
+ return IEnumHasCodeAndDescription.lookupDescription(
+ LoadBalancingAlgorithm.class, value, EVEN_LOAD);
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignment.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignment.java
new file mode 100644
index 0000000000..713f4662b7
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignment.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/** Ledger record for one load-balanced execution attempt. */
+@Getter
+@Setter
+public class LoadBalancingAssignment {
+
+ public static final String STATUS_ASSIGNING = "ASSIGNING";
+ public static final String STATUS_RUNNING = "RUNNING";
+ public static final String STATUS_RETRYING = "RETRYING";
+ public static final String STATUS_FINISHED = "FINISHED";
+ public static final String STATUS_FAILED = "FAILED";
+
+ private String executionId;
+ private String runConfigurationName;
+ private String executorName;
+ private String executorType;
+ private String serverName;
+ private int attempt;
+ private long firstAttemptEpochMs;
+ private String status;
+ private String containerId;
+ private String lastError;
+ private String algorithm;
+ private int occupyingSlotsAtAssignment;
+ private int maxConcurrent;
+
+ public LoadBalancingAssignment() {}
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStore.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStore.java
new file mode 100644
index 0000000000..598c185eb9
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStore.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.OutputStream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+
+/** Writes assignment ledger files under an optional VFS state folder. */
+public class LoadBalancingAssignmentStore {
+
+ private final IVariables variables;
+ private final ILogChannel log;
+ private final ObjectMapper mapper = HopJson.newMapper();
+
+ public LoadBalancingAssignmentStore(IVariables variables, ILogChannel log) {
+ this.variables = variables;
+ this.log = log;
+ }
+
+ public void save(String stateFolder, LoadBalancingAssignment assignment) {
+ if (assignment == null || StringUtils.isEmpty(stateFolder)) {
+ return;
+ }
+ String resolvedFolder = variables.resolve(stateFolder);
+ if (StringUtils.isEmpty(resolvedFolder) || StringUtils.isEmpty(assignment.getExecutionId())) {
+ return;
+ }
+ String runConfig = sanitize(assignment.getRunConfigurationName());
+ String executionId = sanitize(assignment.getExecutionId());
+ String path = resolvedFolder + "/" + runConfig + "/assignments/" + executionId + ".json";
+ try {
+ FileObject file = HopVfs.getFileObject(path, variables);
+ FileObject parent = file.getParent();
+ if (parent != null && !parent.exists()) {
+ parent.createFolder();
+ }
+ try (OutputStream out = HopVfs.getOutputStream(file, false)) {
+ out.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(assignment));
+ out.write('\n');
+ }
+ } catch (Exception e) {
+ if (log != null) {
+ log.logError("Unable to write load-balancing assignment file '" + path + "'", e);
+ }
+ }
+ }
+
+ public LoadBalancingAssignment load(
+ String stateFolder, String runConfigurationName, String executionId) throws HopException {
+ if (StringUtils.isEmpty(stateFolder) || StringUtils.isEmpty(executionId)) {
+ return null;
+ }
+ String resolvedFolder = variables.resolve(stateFolder);
+ String path =
+ resolvedFolder
+ + "/"
+ + sanitize(runConfigurationName)
+ + "/assignments/"
+ + sanitize(executionId)
+ + ".json";
+ try {
+ FileObject file = HopVfs.getFileObject(path, variables);
+ if (!file.exists()) {
+ return null;
+ }
+ try (var in = HopVfs.getInputStream(file)) {
+ return mapper.readValue(in, LoadBalancingAssignment.class);
+ }
+ } catch (Exception e) {
+ throw new HopException("Unable to read load-balancing assignment file '" + path + "'", e);
+ }
+ }
+
+ static String sanitize(String name) {
+ if (StringUtils.isEmpty(name)) {
+ return "unnamed";
+ }
+ return name.replaceAll("[^A-Za-z0-9._-]", "_");
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
new file mode 100644
index 0000000000..0694c386a6
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinator.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.workflow.config.WorkflowRunConfiguration;
+import org.apache.hop.www.HopServerAtCapacityException;
+
+/**
+ * Reloads a load-balancing run configuration, probes the server group, and picks one server.
+ *
+ * @param <C> pipeline or workflow run configuration
+ */
+public class LoadBalancingCoordinator<C> {
+
+ public static final long DEFAULT_PROBE_TIMEOUT_MS = 3000L;
+ public static final long DEFAULT_CONFIG_REFRESH_MS = 10000L;
+
+ private final IVariables variables;
+ private final IHopMetadataProvider metadataProvider;
+ private final ILogChannel log;
+ private final String runConfigurationName;
+ private final boolean workflow;
+ private final LoadBalancingAssignmentStore assignmentStore;
+
+ private ILoadBalancingRunConfiguration currentConfig;
+ private long lastReloadEpochMs;
+
+ public LoadBalancingCoordinator(
+ IVariables variables,
+ IHopMetadataProvider metadataProvider,
+ ILogChannel log,
+ String runConfigurationName,
+ boolean workflow) {
+ this.variables = variables;
+ this.metadataProvider = metadataProvider;
+ this.log = log;
+ this.runConfigurationName = runConfigurationName;
+ this.workflow = workflow;
+ this.assignmentStore = new LoadBalancingAssignmentStore(variables, log);
+ }
+
+ public ILoadBalancingRunConfiguration reloadIfDue() throws HopException {
+ long refreshMs =
+ currentConfig == null
+ ? 0L
+ : Const.toLong(
+ variables.resolve(currentConfig.getConfigRefreshIntervalMs()),
+ DEFAULT_CONFIG_REFRESH_MS);
+ if (currentConfig == null
+ || refreshMs <= 0
+ || System.currentTimeMillis() - lastReloadEpochMs >= refreshMs) {
+ reload();
+ }
+ return currentConfig;
+ }
+
+ public ILoadBalancingRunConfiguration reload() throws HopException {
+ if (metadataProvider == null) {
+ throw new HopException("No metadata provider is available to reload the run configuration");
+ }
+ if (StringUtils.isEmpty(runConfigurationName)) {
+ throw new HopException("The load-balancing run configuration has no name");
+ }
+ ILoadBalancingRunConfiguration loaded;
+ if (workflow) {
+ WorkflowRunConfiguration runConfiguration =
+ metadataProvider.getSerializer(WorkflowRunConfiguration.class).load(runConfigurationName);
+ if (runConfiguration == null
+ || !(runConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration lb)) {
+ throw new HopException(
+ "Workflow run configuration '"
+ + runConfigurationName
+ + "' is not a load-balancing configuration");
+ }
+ loaded = lb;
+ } else {
+ PipelineRunConfiguration runConfiguration =
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).load(runConfigurationName);
+ if (runConfiguration == null
+ || !(runConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration lb)) {
+ throw new HopException(
+ "Pipeline run configuration '"
+ + runConfigurationName
+ + "' is not a load-balancing configuration");
+ }
+ loaded = lb;
+ }
+ currentConfig = loaded;
+ lastReloadEpochMs = System.currentTimeMillis();
+ return currentConfig;
+ }
+
+ public ServerHealthSnapshot selectServer() throws HopException {
+ ILoadBalancingRunConfiguration config = reloadIfDue();
+ List<LoadBalancingServerEntry> servers = config.getServers();
+ if (servers == null || servers.isEmpty()) {
+ throw new HopException(
+ "Load-balancing run configuration '"
+ + runConfigurationName
+ + "' has no Hop servers configured");
+ }
+
+ long probeTimeout =
+ Const.toLong(variables.resolve(config.getProbeTimeoutMs()), DEFAULT_PROBE_TIMEOUT_MS);
+ List<ServerHealthSnapshot> snapshots = probeServers(servers, probeTimeout);
+ LoadBalancingAlgorithm algorithm =
+ LoadBalancingAlgorithm.fromCodeOrDescription(variables.resolve(config.getAlgorithm()));
+ Optional<ServerHealthSnapshot> selected = LoadBalancingSelector.select(algorithm, snapshots);
+ if (selected.isEmpty()) {
+ throw noEligibleServerException(snapshots);
+ }
+ ServerHealthSnapshot snapshot = selected.get();
+ if (log != null) {
+ log.logBasic(
+ "Load-balancing selected Hop server '"
+ + snapshot.getHopServerName()
+ + "' using algorithm '"
+ + algorithm.getCode()
+ + "' ("
+ + snapshot.getOccupyingSlots()
+ + "/"
+ + snapshot.getMaxConcurrent()
+ + " slots, load "
+ + snapshot.getLoadAvg()
+ + ", response "
+ + snapshot.getResponseNs()
+ + "ns)");
+ }
+ return snapshot;
+ }
+
+ public void saveAssignment(LoadBalancingAssignment assignment) {
+ if (currentConfig == null || assignment == null) {
+ return;
+ }
+ assignmentStore.save(currentConfig.getStateFolder(), assignment);
+ }
+
+ public ILoadBalancingRunConfiguration getCurrentConfig() {
+ return currentConfig;
+ }
+
+ List<ServerHealthSnapshot> probeServers(List<LoadBalancingServerEntry> servers, long probeTimeout)
+ throws HopException {
+ ExecutorService executor =
+ Executors.newFixedThreadPool(Math.min(Math.max(1, servers.size()), 8));
+ try {
+ List<CompletableFuture<ServerHealthSnapshot>> futures = new ArrayList<>();
+ for (LoadBalancingServerEntry entry : servers) {
+ futures.add(CompletableFuture.supplyAsync(() -> probeOne(entry, probeTimeout), executor));
+ }
+ List<ServerHealthSnapshot> snapshots = new ArrayList<>();
+ for (CompletableFuture<ServerHealthSnapshot> future : futures) {
+ snapshots.add(future.join());
+ }
+ return snapshots;
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
+ private ServerHealthSnapshot probeOne(LoadBalancingServerEntry entry, long probeTimeout) {
+ String serverName = variables.resolve(entry.getHopServerName());
+ int maxConcurrent = (int) Const.toLong(variables.resolve(entry.getMaxConcurrent()), 1L);
+ if (StringUtils.isEmpty(serverName)) {
+ return ServerHealthSnapshot.unavailable(
+ serverName, entry.isEnabled(), maxConcurrent, "Hop server name is empty");
+ }
+ try {
+ HopServerMeta serverMeta =
+ metadataProvider.getSerializer(HopServerMeta.class).load(serverName);
+ if (serverMeta == null) {
+ return ServerHealthSnapshot.unavailable(
+ serverName,
+ entry.isEnabled(),
+ maxConcurrent,
+ "Hop server '" + serverName + "' not found");
+ }
+ return HopServerProbe.probeWithTimeout(
+ serverMeta, variables, entry.isEnabled(), maxConcurrent, probeTimeout);
+ } catch (Exception e) {
+ return ServerHealthSnapshot.unavailable(
+ serverName,
+ entry.isEnabled(),
+ maxConcurrent,
+ "Error loading Hop server '" + serverName + "': " + e.getMessage());
+ }
+ }
+
+ /**
+ * When every reachable server is full, throw {@link HopServerAtCapacityException} so the engine
+ * waits and retries. Unreachable or misconfigured groups stay a regular {@link HopException}.
+ */
+ static HopException noEligibleServerException(List<ServerHealthSnapshot> snapshots) {
+ if (isGroupAtCapacity(snapshots)) {
+ ServerHealthSnapshot atCapacity = firstAtCapacity(snapshots);
+ int occupying = atCapacity == null ? 0 : atCapacity.getOccupyingSlots();
+ int maxConcurrent = atCapacity == null ? 0 : atCapacity.getMaxConcurrent();
+ return new HopServerAtCapacityException(occupying, maxConcurrent);
+ }
+ return new HopException(describeNoEligibleServer(snapshots));
+ }
+
+ static boolean isGroupAtCapacity(List<ServerHealthSnapshot> snapshots) {
+ if (snapshots == null || snapshots.isEmpty()) {
+ return false;
+ }
+ boolean anyAtCapacity = false;
+ for (ServerHealthSnapshot snapshot : snapshots) {
+ if (snapshot == null || !snapshot.isEnabled()) {
+ continue;
+ }
+ if (snapshot.isEligible()) {
+ return false;
+ }
+ if (isAtCapacity(snapshot)) {
+ anyAtCapacity = true;
+ }
+ }
+ return anyAtCapacity;
+ }
+
+ static boolean isAtCapacity(ServerHealthSnapshot snapshot) {
+ return snapshot != null
+ && snapshot.isEnabled()
+ && snapshot.isAvailable()
+ && !snapshot.isShuttingDown()
+ && snapshot.getMaxConcurrent() > 0
+ && snapshot.getOccupyingSlots() >= snapshot.getMaxConcurrent();
+ }
+
+ static ServerHealthSnapshot firstAtCapacity(List<ServerHealthSnapshot> snapshots) {
+ if (snapshots == null) {
+ return null;
+ }
+ for (ServerHealthSnapshot snapshot : snapshots) {
+ if (isAtCapacity(snapshot)) {
+ return snapshot;
+ }
+ }
+ return null;
+ }
+
+ static String describeNoEligibleServer(List<ServerHealthSnapshot> snapshots) {
+ StringBuilder message =
+ new StringBuilder("No eligible Hop server in the load-balancing group:");
+ if (snapshots == null || snapshots.isEmpty()) {
+ message.append(" the server list is empty");
+ return message.toString();
+ }
+ for (ServerHealthSnapshot snapshot : snapshots) {
+ message
+ .append(Const.CR)
+ .append(" - ")
+ .append(snapshot.getHopServerName())
+ .append(": ")
+ .append(snapshot.skipReason());
+ }
+ return message.toString();
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicy.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicy.java
new file mode 100644
index 0000000000..beb44e26b7
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicy.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import org.apache.hop.core.Const;
+import org.apache.hop.core.variables.IVariables;
+
+/** Tracks retry attempts against a max count and an optional time window. */
+public class LoadBalancingRetryPolicy {
+
+ private final int maxRetries;
+ private final long retryWindowMs;
+ private final long firstAttemptEpochMs;
+ private int attempt;
+
+ public LoadBalancingRetryPolicy(int maxRetries, long retryWindowMs) {
+ this(maxRetries, retryWindowMs, System.currentTimeMillis());
+ }
+
+ public LoadBalancingRetryPolicy(int maxRetries, long retryWindowMs, long firstAttemptEpochMs) {
+ this.maxRetries = Math.max(0, maxRetries);
+ this.retryWindowMs = Math.max(0, retryWindowMs);
+ this.firstAttemptEpochMs = firstAttemptEpochMs;
+ this.attempt = 0;
+ }
+
+ public static LoadBalancingRetryPolicy from(
+ ILoadBalancingRunConfiguration config, IVariables variables) {
+ int maxRetries = (int) Const.toLong(variables.resolve(config.getMaxRetries()), 2L);
+ long windowMs = Const.toLong(variables.resolve(config.getRetryWindowMs()), 0L);
+ return new LoadBalancingRetryPolicy(maxRetries, windowMs);
+ }
+
+ public boolean canAttempt() {
+ if (attempt > maxRetries) {
+ return false;
+ }
+ if (retryWindowMs > 0 && attempt > 0) {
+ return System.currentTimeMillis() - firstAttemptEpochMs < retryWindowMs;
+ }
+ return true;
+ }
+
+ public int beginAttempt() {
+ attempt++;
+ return attempt;
+ }
+
+ public int getAttempt() {
+ return attempt;
+ }
+
+ public int getMaxRetries() {
+ return maxRetries;
+ }
+
+ public long getRetryWindowMs() {
+ return retryWindowMs;
+ }
+
+ public long getFirstAttemptEpochMs() {
+ return firstAttemptEpochMs;
+ }
+
+ public long remainingWindowMs() {
+ if (retryWindowMs <= 0) {
+ return Long.MAX_VALUE;
+ }
+ return Math.max(0L, retryWindowMs - (System.currentTimeMillis() - firstAttemptEpochMs));
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingSelector.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingSelector.java
new file mode 100644
index 0000000000..f86b09adbc
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingSelector.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+/** Picks one eligible Hop Server using the configured algorithm. */
+public final class LoadBalancingSelector {
+
+ private LoadBalancingSelector() {}
+
+ public static Optional<ServerHealthSnapshot> select(
+ LoadBalancingAlgorithm algorithm, List<ServerHealthSnapshot> snapshots) {
+ List<ServerHealthSnapshot> eligible = new ArrayList<>();
+ if (snapshots != null) {
+ for (ServerHealthSnapshot snapshot : snapshots) {
+ if (snapshot != null && snapshot.isEligible()) {
+ eligible.add(snapshot);
+ }
+ }
+ }
+ if (eligible.isEmpty()) {
+ return Optional.empty();
+ }
+ LoadBalancingAlgorithm resolved =
+ algorithm == null ? LoadBalancingAlgorithm.EVEN_LOAD : algorithm;
+ Comparator<ServerHealthSnapshot> comparator =
+ resolved == LoadBalancingAlgorithm.PACK ? packOrder() : evenLoadOrder();
+ eligible.sort(comparator);
+ return Optional.of(eligible.get(0));
+ }
+
+ static Comparator<ServerHealthSnapshot> evenLoadOrder() {
+ return Comparator.comparingDouble(ServerHealthSnapshot::utilization)
+ .thenComparingDouble(ServerHealthSnapshot::loadPerCore)
+ .thenComparingLong(ServerHealthSnapshot::getResponseNs)
+ .thenComparing(s -> s.getHopServerName() == null ? "" : s.getHopServerName());
+ }
+
+ static Comparator<ServerHealthSnapshot> packOrder() {
+ return Comparator.comparingDouble(ServerHealthSnapshot::utilization)
+ .reversed()
+ .thenComparingLong(ServerHealthSnapshot::getResponseNs)
+ .thenComparing(s -> s.getHopServerName() == null ? "" : s.getHopServerName());
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingServerEntry.java b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingServerEntry.java
new file mode 100644
index 0000000000..4b1f7b7bf9
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/LoadBalancingServerEntry.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+/** One Hop Server in a load-balancing run configuration. */
+@Getter
+@Setter
+public class LoadBalancingServerEntry {
+
+ @HopMetadataProperty(key = "hop_server")
+ private String hopServerName;
+
+ @HopMetadataProperty(key = "enabled")
+ private boolean enabled = true;
+
+ /** Variable-capable maximum of occupying pipelines plus workflows on this server. */
+ @HopMetadataProperty(key = "max_concurrent")
+ private String maxConcurrent;
+
+ public LoadBalancingServerEntry() {}
+
+ public LoadBalancingServerEntry(String hopServerName, boolean enabled, String maxConcurrent) {
+ this.hopServerName = hopServerName;
+ this.enabled = enabled;
+ this.maxConcurrent = maxConcurrent;
+ }
+
+ public LoadBalancingServerEntry(LoadBalancingServerEntry entry) {
+ this.hopServerName = entry.hopServerName;
+ this.enabled = entry.enabled;
+ this.maxConcurrent = entry.maxConcurrent;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/server/loadbalance/ServerHealthSnapshot.java b/engine/src/main/java/org/apache/hop/server/loadbalance/ServerHealthSnapshot.java
new file mode 100644
index 0000000000..8f8abdfd28
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/server/loadbalance/ServerHealthSnapshot.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/** Result of probing one Hop Server for load-balancing (and Get Server Status). */
+@Getter
+@Setter
+public class ServerHealthSnapshot {
+
+ private String hopServerName;
+ private boolean enabled;
+ private int maxConcurrent;
+ private boolean available;
+ private boolean shuttingDown;
+ private String statusDescription;
+ private String errorMessage;
+ private Double loadAvg;
+ private Long memoryFree;
+ private Long memoryTotal;
+ private Integer cpuCores;
+ private Long cpuProcessTime;
+ private String osName;
+ private String osVersion;
+ private String osArchitecture;
+ private int runningPipelines;
+ private int finishedPipelines;
+ private int runningWorkflows;
+ private int finishedWorkflows;
+ private int occupyingSlots;
+ private long responseNs;
+
+ public static ServerHealthSnapshot unavailable(
+ String hopServerName, boolean enabled, int maxConcurrent, String errorMessage) {
+ ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
+ snapshot.hopServerName = hopServerName;
+ snapshot.enabled = enabled;
+ snapshot.maxConcurrent = maxConcurrent;
+ snapshot.available = false;
+ snapshot.errorMessage = errorMessage;
+ return snapshot;
+ }
+
+ public boolean isEligible() {
+ return enabled
+ && available
+ && !shuttingDown
+ && maxConcurrent > 0
+ && occupyingSlots < maxConcurrent;
+ }
+
+ public double utilization() {
+ if (maxConcurrent <= 0) {
+ return Double.POSITIVE_INFINITY;
+ }
+ return (double) occupyingSlots / (double) maxConcurrent;
+ }
+
+ public double loadPerCore() {
+ int cores = cpuCores == null || cpuCores <= 0 ? 1 : cpuCores;
+ double load = loadAvg == null ? 0.0 : loadAvg;
+ return load / cores;
+ }
+
+ public String skipReason() {
+ if (!enabled) {
+ return "disabled";
+ }
+ if (!available) {
+ return errorMessage == null ? "unreachable" : errorMessage;
+ }
+ if (shuttingDown) {
+ return "shutting down";
+ }
+ if (maxConcurrent <= 0) {
+ return "max concurrent is " + maxConcurrent;
+ }
+ if (occupyingSlots >= maxConcurrent) {
+ return "at capacity (" + occupyingSlots + "/" + maxConcurrent + ")";
+ }
+ return null;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
new file mode 100644
index 0000000000..3c01cc5d31
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowEngine.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.workflow.engines.loadbalance;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.IExecutionInfoLocation;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
+import org.apache.hop.server.loadbalance.LoadBalancingAlgorithm;
+import org.apache.hop.server.loadbalance.LoadBalancingAssignment;
+import org.apache.hop.server.loadbalance.LoadBalancingCoordinator;
+import org.apache.hop.server.loadbalance.LoadBalancingRetryPolicy;
+import org.apache.hop.server.loadbalance.ServerHealthSnapshot;
+import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
+import org.apache.hop.workflow.engine.WorkflowEnginePlugin;
+import org.apache.hop.workflow.engines.remote.RemoteWorkflowEngine;
+import org.apache.hop.www.HopServerAdmission;
+
+@WorkflowEnginePlugin(
+ id = "LoadBalancing",
+ name = "Hop load-balancing workflow engine",
+ description = "Assigns the workflow to one Hop server from a configured group")
+public class LoadBalancingWorkflowEngine extends RemoteWorkflowEngine {
+
+ public static final String DETAIL_ASSIGNED_SERVER = "assignedServer";
+ public static final String DETAIL_ATTEMPT = "attempt";
+ public static final String DETAIL_ALGORITHM = "algorithm";
+ public static final String DETAIL_CONTAINER_ID = "containerId";
+ public static final String DETAIL_LOAD_AT_ASSIGNMENT = "loadAtAssignment";
+
+ private LoadBalancingCoordinator<LoadBalancingWorkflowRunConfiguration> coordinator;
+ private LoadBalancingAssignment assignment;
+ private ExecutionInfoLocation executionInfoLocation;
+
+ @Override
+ public IWorkflowEngineRunConfiguration createDefaultWorkflowEngineRunConfiguration() {
+ return new LoadBalancingWorkflowRunConfiguration();
+ }
+
+ @Override
+ protected void submitToRemoteServer() throws HopException {
+ IWorkflowEngineRunConfiguration engineRunConfiguration =
+ workflowRunConfiguration.getEngineRunConfiguration();
+ if (!(engineRunConfiguration instanceof ILoadBalancingRunConfiguration)) {
+ throw new HopException(
+ "The load-balancing workflow engine expects a load-balancing workflow configuration");
+ }
+
+ coordinator =
+ new LoadBalancingCoordinator<>(
+ this, metadataProvider, logChannel, workflowRunConfiguration.getName(), true);
+ coordinator.reload();
+ validateRunConfigurationChain(workflowRunConfiguration);
+
+ LoadBalancingRetryPolicy retry =
+ LoadBalancingRetryPolicy.from(coordinator.getCurrentConfig(), this);
+ Exception lastError = null;
+ while (retry.canAttempt()) {
+ int attempt = retry.beginAttempt();
+ containerId = null;
+ hopServer = null;
+ try {
+ ServerHealthSnapshot snapshot = coordinator.selectServer();
+ selectedHopServerName = snapshot.getHopServerName();
+ admissionMaxConcurrent = snapshot.getMaxConcurrent();
+ assignment = newAssignment(snapshot, attempt, retry);
+ coordinator.saveAssignment(assignment);
+
+ logChannel.logBasic(
+ "Executing this workflow using the Load-balancing Workflow Engine with run configuration '"
+ + workflowRunConfiguration.getName()
+ + "', attempt "
+ + attempt
+ + " on server '"
+ + selectedHopServerName
+ + "'");
+
+ super.submitToRemoteServer();
+ assignment.setStatus(LoadBalancingAssignment.STATUS_RUNNING);
+ assignment.setContainerId(containerId);
+ coordinator.saveAssignment(assignment);
+ registerLoadBalancingExecutionInformation(snapshot, attempt);
+ addExecutionFinishedListener(engine -> updateFinishedAssignmentAndState());
+ return;
+ } catch (Exception e) {
+ lastError = e;
+ if (assignment != null) {
+ assignment.setLastError(e.getMessage());
+ assignment.setStatus(LoadBalancingAssignment.STATUS_RETRYING);
+ coordinator.saveAssignment(assignment);
+ }
+ if (HopServerAdmission.isRetryableRegistrationFailure(e, containerId)
+ && retry.canAttempt()) {
+ logCapacityWait(attempt, e);
+ sleepBackoff(retry);
+ continue;
+ }
+ if (assignment != null) {
+ assignment.setStatus(LoadBalancingAssignment.STATUS_FAILED);
+ coordinator.saveAssignment(assignment);
+ }
+ if (e instanceof HopException hopException) {
+ throw hopException;
+ }
+ throw new HopException("Error submitting load-balanced workflow", e);
+ }
+ }
+ if (assignment != null) {
+ assignment.setStatus(LoadBalancingAssignment.STATUS_FAILED);
+ coordinator.saveAssignment(assignment);
+ }
+ throw new HopException(
+ "Load-balancing gave up after " + retry.getAttempt() + " attempt(s)", lastError);
+ }
+
+ private LoadBalancingAssignment newAssignment(
+ ServerHealthSnapshot snapshot, int attempt, LoadBalancingRetryPolicy retry) {
+ LoadBalancingAssignment next = new LoadBalancingAssignment();
+ next.setExecutionId(getLogChannelId());
+ next.setRunConfigurationName(workflowRunConfiguration.getName());
+ next.setExecutorName(getWorkflowName());
+ next.setExecutorType("workflow");
+ next.setServerName(snapshot.getHopServerName());
+ next.setAttempt(attempt);
+ next.setFirstAttemptEpochMs(retry.getFirstAttemptEpochMs());
+ next.setStatus(LoadBalancingAssignment.STATUS_ASSIGNING);
+ next.setAlgorithm(
+ LoadBalancingAlgorithm.fromCodeOrDescription(
+ resolve(coordinator.getCurrentConfig().getAlgorithm()))
+ .getCode());
+ next.setOccupyingSlotsAtAssignment(snapshot.getOccupyingSlots());
+ next.setMaxConcurrent(snapshot.getMaxConcurrent());
+ return next;
+ }
+
+ private void logCapacityWait(int attempt, Exception error) {
+ String server =
+ StringUtils.isEmpty(selectedHopServerName)
+ ? "the load-balancing group"
+ : selectedHopServerName;
+ logChannel.logBasic(
+ "Load-balancing attempt "
+ + attempt
+ + " waiting for a free slot on "
+ + server
+ + ": "
+ + error.getMessage());
+ }
+
+ private void sleepBackoff(LoadBalancingRetryPolicy retry) {
+ long waitMs = Math.min(1000L, retry.remainingWindowMs());
+ if (waitMs <= 0) {
+ return;
+ }
+ try {
+ Thread.sleep(waitMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void registerLoadBalancingExecutionInformation(
+ ServerHealthSnapshot snapshot, int attempt) {
+ try {
+ lookupExecutionInformationLocation();
+ if (executionInfoLocation == null) {
+ return;
+ }
+ IExecutionInfoLocation location = executionInfoLocation.getExecutionInfoLocation();
+ location.registerExecution(ExecutionBuilder.fromExecutor(this).build());
+ ExecutionState state = ExecutionStateBuilder.fromExecutor(this, -1).build();
+ Map<String, String> details =
+ state.getDetails() == null ? new HashMap<>() : state.getDetails();
+ details.put(DETAIL_ASSIGNED_SERVER, selectedHopServerName);
+ details.put(DETAIL_ATTEMPT, Integer.toString(attempt));
+ details.put(
+ DETAIL_ALGORITHM,
+ LoadBalancingAlgorithm.fromCodeOrDescription(
+ resolve(coordinator.getCurrentConfig().getAlgorithm()))
+ .getCode());
+ details.put(DETAIL_CONTAINER_ID, containerId);
+ details.put(
+ DETAIL_LOAD_AT_ASSIGNMENT,
+ snapshot.getOccupyingSlots() + "/" + snapshot.getMaxConcurrent());
+ state.setDetails(details);
+ location.updateExecutionState(state);
+ } catch (Exception e) {
+ logChannel.logError("Error registering load-balancing execution information (non-fatal)", e);
+ }
+ }
+
+ private void lookupExecutionInformationLocation() throws HopException {
+ if (executionInfoLocation != null || workflowRunConfiguration == null) {
+ return;
+ }
+ String locationName = resolve(workflowRunConfiguration.getExecutionInfoLocationName());
+ if (StringUtils.isEmpty(locationName) || metadataProvider == null) {
+ return;
+ }
+ ExecutionInfoLocation location =
+ metadataProvider.getSerializer(ExecutionInfoLocation.class).load(locationName);
+ if (location == null) {
+ logChannel.logError(
+ "Execution information location '" + locationName + "' could not be found");
+ return;
+ }
+ location.getExecutionInfoLocation().initialize(this, metadataProvider);
+ executionInfoLocation = location;
+ }
+
+ private void updateFinishedAssignmentAndState() {
+ try {
+ boolean failed = result != null && (result.getNrErrors() > 0 || !result.getResult());
+ if (assignment != null && coordinator != null) {
+ assignment.setStatus(
+ failed
+ ? LoadBalancingAssignment.STATUS_FAILED
+ : LoadBalancingAssignment.STATUS_FINISHED);
+ assignment.setContainerId(containerId);
+ coordinator.saveAssignment(assignment);
+ }
+ if (executionInfoLocation != null) {
+ IExecutionInfoLocation location = executionInfoLocation.getExecutionInfoLocation();
+ ExecutionState state = ExecutionStateBuilder.fromExecutor(this, -1).build();
+ Map<String, String> details =
+ state.getDetails() == null ? new HashMap<>() : state.getDetails();
+ if (assignment != null) {
+ details.put(DETAIL_ASSIGNED_SERVER, assignment.getServerName());
+ details.put(DETAIL_ATTEMPT, Integer.toString(assignment.getAttempt()));
+ details.put(DETAIL_ALGORITHM, assignment.getAlgorithm());
+ details.put(DETAIL_CONTAINER_ID, assignment.getContainerId());
+ }
+ state.setDetails(details);
+ location.updateExecutionState(state);
+ location.close();
+ }
+ } catch (Exception e) {
+ if (logChannel != null) {
+ logChannel.logError("Error updating load-balancing assignment state (non-fatal)", e);
+ }
+ }
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowRunConfiguration.java b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowRunConfiguration.java
new file mode 100644
index 0000000000..c825abd778
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/workflow/engines/loadbalance/LoadBalancingWorkflowRunConfiguration.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.workflow.engines.loadbalance;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
+import org.apache.hop.server.loadbalance.LoadBalancingAlgorithm;
+import org.apache.hop.server.loadbalance.LoadBalancingServerEntry;
+import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
+import org.apache.hop.workflow.config.WorkflowRunConfiguration;
+import org.apache.hop.workflow.engines.empty.EmptyWorkflowRunConfiguration;
+
+@GuiPlugin(description = "Load-balancing workflow run configuration widgets")
+@Getter
+@Setter
+public class LoadBalancingWorkflowRunConfiguration extends EmptyWorkflowRunConfiguration
+ implements IWorkflowEngineRunConfiguration, ILoadBalancingRunConfiguration {
+
+ @GuiWidgetElement(
+ order = "20",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.METADATA,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.RunConfiguration.Label",
+ metadata = WorkflowRunConfiguration.class)
+ @HopMetadataProperty(
+ key = "run_config",
+ hopMetadataPropertyType = HopMetadataPropertyType.WORKFLOW_RUN_CONFIG)
+ protected String runConfigurationName;
+
+ @GuiWidgetElement(
+ order = "30",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.COMBO,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingAlgorithm.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingAlgorithm.ToolTip",
+ comboValuesMethod = "getAlgorithms")
+ @HopMetadataProperty(key = "algorithm")
+ protected String algorithm;
+
+ @GuiWidgetElement(
+ order = "40",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingMaxRetries.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingMaxRetries.ToolTip")
+ @HopMetadataProperty(key = "max_retries")
+ protected String maxRetries;
+
+ @GuiWidgetElement(
+ order = "50",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingRetryWindow.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingRetryWindow.ToolTip")
+ @HopMetadataProperty(key = "retry_window_ms")
+ protected String retryWindowMs;
+
+ @GuiWidgetElement(
+ order = "60",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.CHECKBOX,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingRetryOnFailure.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingRetryOnFailure.ToolTip")
+ @HopMetadataProperty(key = "retry_on_execution_failure")
+ protected boolean retryOnExecutionFailure;
+
+ @GuiWidgetElement(
+ order = "70",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingProbeTimeout.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingProbeTimeout.ToolTip")
+ @HopMetadataProperty(key = "probe_timeout_ms")
+ protected String probeTimeoutMs;
+
+ @GuiWidgetElement(
+ order = "80",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingConfigRefresh.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingConfigRefresh.ToolTip")
+ @HopMetadataProperty(key = "config_refresh_ms")
+ protected String configRefreshIntervalMs;
+
+ @GuiWidgetElement(
+ order = "90",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.LoadBalancing",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.FOLDER,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingStateFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.LoadBalancingStateFolder.ToolTip")
+ @HopMetadataProperty(key = "state_folder")
+ protected String stateFolder;
+
+ @GuiWidgetElement(
+ order = "100",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.ServerPollDelay.Label")
+ @HopMetadataProperty(key = "server_poll_delay")
+ protected String serverPollDelay;
+
+ @GuiWidgetElement(
+ order = "110",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.ServerPollInterval.Label")
+ @HopMetadataProperty(key = "server_poll_interval")
+ protected String serverPollInterval;
+
+ @GuiWidgetElement(
+ order = "120",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.CHECKBOX,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.ExportResources.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.ExportResources.ToolTip")
+ @HopMetadataProperty(key = "export_resources")
+ protected boolean exportingResources;
+
+ @GuiWidgetElement(
+ order = "130",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.FOLDER,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.NamedResourceSourceFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.NamedResourceSourceFolder.ToolTip")
+ @HopMetadataProperty(key = "resources_source_folder")
+ protected String namedResourcesSourceFolder;
+
+ @GuiWidgetElement(
+ order = "140",
+ parentId = WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID,
+ group =
+ "i18n:org.apache.hop.ui.server.loadbalance:LoadBalancingRunConfiguration.Tab.Management",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS,
+ type = GuiElementType.TEXT,
+ label =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.NamedResourceTargetFolder.Label",
+ toolTip =
+ "i18n:org.apache.hop.ui.workflow.config:WorkflowRunConfigurationDialog.NamedResourceTargetFolder.ToolTip")
+ @HopMetadataProperty(key = "resources_target_folder")
+ protected String namedResourcesTargetFolder;
+
+ @HopMetadataProperty(key = "server", groupKey = "servers")
+ protected List<LoadBalancingServerEntry> servers;
+
+ protected String hopServerName;
+
+ public LoadBalancingWorkflowRunConfiguration() {
+ super();
+ this.servers = new ArrayList<>();
+ this.algorithm = LoadBalancingAlgorithm.EVEN_LOAD.getDescription();
+ this.maxRetries = "2";
+ this.retryWindowMs = "0";
+ this.probeTimeoutMs = "3000";
+ this.configRefreshIntervalMs = "10000";
+ }
+
+ public LoadBalancingWorkflowRunConfiguration(LoadBalancingWorkflowRunConfiguration config) {
+ super(config);
+ this.runConfigurationName = config.runConfigurationName;
+ this.algorithm = config.algorithm;
+ this.maxRetries = config.maxRetries;
+ this.retryWindowMs = config.retryWindowMs;
+ this.retryOnExecutionFailure = config.retryOnExecutionFailure;
+ this.probeTimeoutMs = config.probeTimeoutMs;
+ this.configRefreshIntervalMs = config.configRefreshIntervalMs;
+ this.stateFolder = config.stateFolder;
+ this.serverPollDelay = config.serverPollDelay;
+ this.serverPollInterval = config.serverPollInterval;
+ this.exportingResources = config.exportingResources;
+ this.namedResourcesSourceFolder = config.namedResourcesSourceFolder;
+ this.namedResourcesTargetFolder = config.namedResourcesTargetFolder;
+ this.hopServerName = config.hopServerName;
+ this.servers = new ArrayList<>();
+ if (config.servers != null) {
+ for (LoadBalancingServerEntry entry : config.servers) {
+ this.servers.add(new LoadBalancingServerEntry(entry));
+ }
+ }
+ }
+
+ @Override
+ public LoadBalancingWorkflowRunConfiguration clone() {
+ return new LoadBalancingWorkflowRunConfiguration(this);
+ }
+
+ public List<String> getAlgorithms(ILogChannel log, IHopMetadataProvider metadataProvider) {
+ List<String> list = new ArrayList<>();
+ for (LoadBalancingAlgorithm value : LoadBalancingAlgorithm.values()) {
+ list.add(value.getDescription());
+ }
+ return list;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java
index b56bf33ef0..74db2c0783 100644
--- a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java
+++ b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java
@@ -60,6 +60,7 @@ import org.apache.hop.pipeline.engines.remote.RemotePipelineEngine;
import org.apache.hop.resource.ResourceUtil;
import org.apache.hop.resource.TopLevelResource;
import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.server.IRemoteCapableRunConfiguration;
import org.apache.hop.workflow.ActionResult;
import org.apache.hop.workflow.IActionListener;
import org.apache.hop.workflow.IDelegationListener;
@@ -74,6 +75,7 @@ import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.apache.hop.workflow.engine.IWorkflowEngine;
import org.apache.hop.workflow.engine.WorkflowEnginePlugin;
+import org.apache.hop.www.HopServerAdmission;
import org.apache.hop.www.HopServerWorkflowStatus;
import org.apache.hop.www.RegisterPackageServlet;
import org.apache.hop.www.RegisterWorkflowServlet;
@@ -98,7 +100,14 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
protected WorkflowMeta workflowMeta;
protected String pluginId;
protected WorkflowRunConfiguration workflowRunConfiguration;
- protected RemoteWorkflowRunConfiguration remoteWorkflowRunConfiguration;
+ protected IRemoteCapableRunConfiguration remoteWorkflowRunConfiguration;
+
+ /** Set by a load-balancing engine after it picks a server; otherwise the config name is used. */
+ protected String selectedHopServerName;
+
+ /** Optional server-side admission cap sent as {@code max_concurrent} on register. */
+ protected int admissionMaxConcurrent;
+
protected Result previousResult;
protected Result result;
protected IHopMetadataProvider metadataProvider;
@@ -215,14 +224,14 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
* @param runConfiguration the remote run configuration to start from
* @throws HopException when the chain leads back to a run configuration it already passed
*/
- void validateRunConfigurationChain(WorkflowRunConfiguration runConfiguration)
+ protected void validateRunConfigurationChain(WorkflowRunConfiguration runConfiguration)
throws HopException {
List<String> chain = new ArrayList<>();
chain.add(runConfiguration.getName());
WorkflowRunConfiguration current = runConfiguration;
while (current != null
- && current.getEngineRunConfiguration() instanceof RemoteWorkflowRunConfiguration remote) {
+ && current.getEngineRunConfiguration() instanceof IRemoteCapableRunConfiguration remote) {
String linkedName = resolve(remote.getRunConfigurationName());
if (StringUtils.isEmpty(linkedName)) {
// Reported for the run configuration this engine was asked to run with.
@@ -272,57 +281,7 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
result = previousResult;
}
- IWorkflowEngineRunConfiguration engineRunConfiguration =
- workflowRunConfiguration.getEngineRunConfiguration();
- if (!(engineRunConfiguration instanceof RemoteWorkflowRunConfiguration)) {
- throw new HopException(
- "The remote workflow engine expects a remote workflow configuration");
- }
- remoteWorkflowRunConfiguration =
- (RemoteWorkflowRunConfiguration) workflowRunConfiguration.getEngineRunConfiguration();
-
- String hopServerName = resolve(remoteWorkflowRunConfiguration.getHopServerName());
- if (StringUtils.isEmpty(hopServerName)) {
- throw new HopException("No remote Hop server was specified to run the workflow on");
- }
- String remoteRunConfigurationName = remoteWorkflowRunConfiguration.getRunConfigurationName();
- if (StringUtils.isEmpty(remoteRunConfigurationName)) {
- throw new HopException("No run configuration was specified to the remote workflow with");
- }
- if (metadataProvider == null) {
- throw new HopException(
- "The remote workflow engine didn't receive a metadata to load hop server '"
- + hopServerName
- + "'");
- }
- validateRunConfigurationChain(workflowRunConfiguration);
-
- logChannel.logBasic(
- "Executing this workflow using the Remote Workflow Engine with run configuration '"
- + workflowRunConfiguration.getName()
- + "'");
-
- HopServerMeta hopServerMeta =
- metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName);
- if (hopServerMeta == null) {
- throw new HopException("Hop server '" + hopServerName + "' could not be found");
- }
- hopServer = new RemoteHopServer(hopServerMeta);
-
- WorkflowExecutionConfiguration workflowExecutionConfiguration =
- new WorkflowExecutionConfiguration();
- workflowExecutionConfiguration.setRunConfiguration(remoteRunConfigurationName);
- if (logLevel != null) {
- workflowExecutionConfiguration.setLogLevel(logLevel);
- }
- if (previousResult != null) {
- // This contains result rows, files, ...
- //
- workflowExecutionConfiguration.setPreviousResult(previousResult);
- }
- workflowExecutionConfiguration.setGatheringMetrics(gatheringMetrics);
-
- sendToHopServer(this, workflowMeta, workflowExecutionConfiguration, metadataProvider);
+ submitToRemoteServer();
fireExecutionStartedListeners();
initialized = true;
@@ -449,6 +408,72 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
}
}
+ /**
+ * Resolve the target server, send the workflow and start it. Extracted so the load-balancing
+ * engine can retry this part without monitoring.
+ */
+ protected void submitToRemoteServer() throws HopException {
+ IWorkflowEngineRunConfiguration engineRunConfiguration =
+ workflowRunConfiguration.getEngineRunConfiguration();
+ if (!(engineRunConfiguration instanceof IRemoteCapableRunConfiguration remoteCapable)) {
+ throw new HopException("The remote workflow engine expects a remote workflow configuration");
+ }
+ remoteWorkflowRunConfiguration = remoteCapable;
+
+ String hopServerName = resolveTargetHopServerName();
+ if (StringUtils.isEmpty(hopServerName)) {
+ throw new HopException("No remote Hop server was specified to run the workflow on");
+ }
+ String remoteRunConfigurationName = remoteWorkflowRunConfiguration.getRunConfigurationName();
+ if (StringUtils.isEmpty(remoteRunConfigurationName)) {
+ throw new HopException("No run configuration was specified to the remote workflow with");
+ }
+ if (metadataProvider == null) {
+ throw new HopException(
+ "The remote workflow engine didn't receive a metadata to load hop server '"
+ + hopServerName
+ + "'");
+ }
+ validateRunConfigurationChain(workflowRunConfiguration);
+
+ logChannel.logBasic(
+ "Executing this workflow using the Remote Workflow Engine with run configuration '"
+ + workflowRunConfiguration.getName()
+ + "'");
+
+ HopServerMeta hopServerMeta =
+ metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName);
+ if (hopServerMeta == null) {
+ throw new HopException("Hop server '" + hopServerName + "' could not be found");
+ }
+ hopServer = new RemoteHopServer(hopServerMeta);
+
+ WorkflowExecutionConfiguration workflowExecutionConfiguration =
+ new WorkflowExecutionConfiguration();
+ workflowExecutionConfiguration.setRunConfiguration(remoteRunConfigurationName);
+ if (logLevel != null) {
+ workflowExecutionConfiguration.setLogLevel(logLevel);
+ }
+ if (previousResult != null) {
+ // This contains result rows, files, ...
+ //
+ workflowExecutionConfiguration.setPreviousResult(previousResult);
+ }
+ workflowExecutionConfiguration.setGatheringMetrics(gatheringMetrics);
+
+ sendToHopServer(this, workflowMeta, workflowExecutionConfiguration, metadataProvider);
+ }
+
+ protected String resolveTargetHopServerName() {
+ if (StringUtils.isNotEmpty(selectedHopServerName)) {
+ return resolve(selectedHopServerName);
+ }
+ if (remoteWorkflowRunConfiguration == null) {
+ return null;
+ }
+ return resolve(remoteWorkflowRunConfiguration.getHopServerName());
+ }
+
/**
* Send to hop server.
*
@@ -514,7 +539,8 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
this,
topLevelResource.getArchiveName(),
RegisterPackageServlet.TYPE_WORKFLOW,
- topLevelResource.getBaseResourceName());
+ topLevelResource.getBaseResourceName(),
+ admissionMaxConcurrent);
WebResult webResult = WebResult.fromXmlString(result);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new HopException(
@@ -530,7 +556,12 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine<W
.getXml(variables);
String reply =
- hopServer.sendXml(this, xml, RegisterWorkflowServlet.CONTEXT_PATH + "/?xml=Y");
+ hopServer.sendXml(
+ this,
+ xml,
+ RegisterWorkflowServlet.CONTEXT_PATH
+ + "/?xml=Y"
+ + HopServerAdmission.querySuffix(admissionMaxConcurrent));
WebResult webResult = WebResult.fromXmlString(reply);
if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) {
throw new HopException(
diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowRunConfiguration.java b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowRunConfiguration.java
index 09dcc87ce2..c58aab249b 100644
--- a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowRunConfiguration.java
+++ b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowRunConfiguration.java
@@ -23,13 +23,14 @@ import org.apache.hop.core.gui.plugin.GuiWidgetElement;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.server.IRemoteCapableRunConfiguration;
import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.apache.hop.workflow.engines.empty.EmptyWorkflowRunConfiguration;
@GuiPlugin(description = "Remote workflow run configuration widgets")
public class RemoteWorkflowRunConfiguration extends EmptyWorkflowRunConfiguration
- implements IWorkflowEngineRunConfiguration {
+ implements IWorkflowEngineRunConfiguration, IRemoteCapableRunConfiguration {
@GuiWidgetElement(
order = "10",
diff --git a/engine/src/main/java/org/apache/hop/www/BaseWorkflowServlet.java b/engine/src/main/java/org/apache/hop/www/BaseWorkflowServlet.java
index 79470adbe0..c51a2c6d71 100644
--- a/engine/src/main/java/org/apache/hop/www/BaseWorkflowServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/BaseWorkflowServlet.java
@@ -48,6 +48,11 @@ public abstract class BaseWorkflowServlet extends BodyHttpServlet {
protected IWorkflowEngine<WorkflowMeta> createWorkflow(
WorkflowConfiguration workflowConfiguration) throws HopException {
+ return createWorkflow(workflowConfiguration, 0);
+ }
+
+ protected IWorkflowEngine<WorkflowMeta> createWorkflow(
+ WorkflowConfiguration workflowConfiguration, int maxConcurrent) throws HopException {
WorkflowExecutionConfiguration workflowExecutionConfiguration =
workflowConfiguration.getWorkflowExecutionConfiguration();
@@ -89,8 +94,14 @@ public abstract class BaseWorkflowServlet extends BodyHttpServlet {
workflow.setStartActionMeta(startActionMeta);
}
- getWorkflowMap()
- .addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, workflowConfiguration);
+ HopServerAdmission.admitAndAdd(
+ getPipelineMap(),
+ getWorkflowMap(),
+ maxConcurrent,
+ () ->
+ getWorkflowMap()
+ .addWorkflow(
+ workflow.getWorkflowName(), serverObjectId, workflow, workflowConfiguration));
// Remember the generated container ID
//
@@ -101,6 +112,11 @@ public abstract class BaseWorkflowServlet extends BodyHttpServlet {
protected IPipelineEngine<PipelineMeta> createPipeline(
PipelineConfiguration pipelineConfiguration) throws HopException {
+ return createPipeline(pipelineConfiguration, 0);
+ }
+
+ protected IPipelineEngine<PipelineMeta> createPipeline(
+ PipelineConfiguration pipelineConfiguration, int maxConcurrent) throws HopException {
PipelineMeta pipelineMeta = pipelineConfiguration.getPipelineMeta();
PipelineExecutionConfiguration pipelineExecutionConfiguration =
pipelineConfiguration.getPipelineExecutionConfiguration();
@@ -157,8 +173,14 @@ public abstract class BaseWorkflowServlet extends BodyHttpServlet {
}
pipeline.setContainerId(serverObjectId);
- getPipelineMap()
- .addPipeline(pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration);
+ HopServerAdmission.admitAndAdd(
+ getPipelineMap(),
+ getWorkflowMap(),
+ maxConcurrent,
+ () ->
+ getPipelineMap()
+ .addPipeline(
+ pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration));
return pipeline;
}
diff --git a/engine/src/main/java/org/apache/hop/www/HopServerAdmission.java b/engine/src/main/java/org/apache/hop/www/HopServerAdmission.java
new file mode 100644
index 0000000000..2fd498b834
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/www/HopServerAdmission.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.www;
+
+import jakarta.servlet.http.HttpServletRequest;
+import java.util.Locale;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.engine.IWorkflowEngine;
+
+/**
+ * Server-side capacity check used by register/export servlets. When a client sends {@link
+ * #PARAMETER_MAX_CONCURRENT}, registration is refused if the number of unfinished pipelines plus
+ * workflows is already at that cap.
+ */
+public final class HopServerAdmission {
+
+ public static final String PARAMETER_MAX_CONCURRENT = "max_concurrent";
+ public static final String RESULT_AT_CAPACITY = "SERVER_AT_CAPACITY";
+
+ private static final Object LOCK = new Object();
+
+ private HopServerAdmission() {}
+
+ public static int parseMaxConcurrent(HttpServletRequest request) {
+ if (request == null) {
+ return 0;
+ }
+ return parseMaxConcurrent(request.getParameter(PARAMETER_MAX_CONCURRENT));
+ }
+
+ public static int parseMaxConcurrent(String raw) {
+ if (StringUtils.isEmpty(raw)) {
+ return 0;
+ }
+ return (int) Const.toLong(raw, 0L);
+ }
+
+ public static boolean isAtCapacityResult(WebResult webResult) {
+ return webResult != null && RESULT_AT_CAPACITY.equalsIgnoreCase(webResult.getResult());
+ }
+
+ public static boolean isRetryableRegistrationFailure(Throwable error, String containerId) {
+ if (StringUtils.isNotEmpty(containerId)) {
+ return false;
+ }
+ for (Throwable current = error; current != null; current = current.getCause()) {
+ if (current instanceof HopServerAtCapacityException) {
+ return true;
+ }
+ String message = current.getMessage();
+ if (message != null
+ && (message.contains(RESULT_AT_CAPACITY)
+ || message.toLowerCase(Locale.ROOT).contains("at capacity"))) {
+ return true;
+ }
+ if (current.getCause() == current) {
+ break;
+ }
+ }
+ return false;
+ }
+
+ public static int countOccupyingSlots(PipelineMap pipelineMap, WorkflowMap workflowMap) {
+ int count = 0;
+ if (pipelineMap != null) {
+ for (HopServerObjectEntry entry : pipelineMap.getPipelineObjects()) {
+ IPipelineEngine<PipelineMeta> pipeline = pipelineMap.getPipeline(entry);
+ if (occupiesSlot(pipeline)) {
+ count++;
+ }
+ }
+ }
+ if (workflowMap != null) {
+ for (HopServerObjectEntry entry : workflowMap.getWorkflowObjects()) {
+ IWorkflowEngine<WorkflowMeta> workflow = workflowMap.getWorkflow(entry);
+ if (occupiesSlot(workflow)) {
+ count++;
+ }
+ }
+ }
+ return count;
+ }
+
+ public static boolean occupiesSlot(IPipelineEngine<PipelineMeta> pipeline) {
+ return pipeline != null && !pipeline.isFinished();
+ }
+
+ public static boolean occupiesSlot(IWorkflowEngine<WorkflowMeta> workflow) {
+ return workflow != null && !workflow.isFinished();
+ }
+
+ /**
+ * If {@code maxConcurrent} is greater than zero and the server is already at that many occupying
+ * slots, throw {@link HopServerAtCapacityException}. Otherwise run {@code add} while holding the
+ * admission lock so the count-and-add is atomic.
+ */
+ public static void admitAndAdd(
+ PipelineMap pipelineMap, WorkflowMap workflowMap, int maxConcurrent, Runnable add)
+ throws HopServerAtCapacityException {
+ synchronized (LOCK) {
+ if (maxConcurrent > 0) {
+ int occupying = countOccupyingSlots(pipelineMap, workflowMap);
+ if (occupying >= maxConcurrent) {
+ throw new HopServerAtCapacityException(occupying, maxConcurrent);
+ }
+ }
+ add.run();
+ }
+ }
+
+ public static String querySuffix(int maxConcurrent) {
+ if (maxConcurrent <= 0) {
+ return "";
+ }
+ return "&" + PARAMETER_MAX_CONCURRENT + "=" + maxConcurrent;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/www/HopServerAtCapacityException.java b/engine/src/main/java/org/apache/hop/www/HopServerAtCapacityException.java
new file mode 100644
index 0000000000..83b99f6659
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/www/HopServerAtCapacityException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.www;
+
+import org.apache.hop.core.exception.HopException;
+
+/** The server refused a registration because it is already at {@code max_concurrent}. */
+public class HopServerAtCapacityException extends HopException {
+ private final int occupyingSlots;
+ private final int maxConcurrent;
+
+ public HopServerAtCapacityException(int occupyingSlots, int maxConcurrent) {
+ super(
+ HopServerAdmission.RESULT_AT_CAPACITY
+ + ": "
+ + occupyingSlots
+ + " occupying slots, max "
+ + maxConcurrent);
+ this.occupyingSlots = occupyingSlots;
+ this.maxConcurrent = maxConcurrent;
+ }
+
+ public int getOccupyingSlots() {
+ return occupyingSlots;
+ }
+
+ public int getMaxConcurrent() {
+ return maxConcurrent;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/www/RegisterPackageServlet.java b/engine/src/main/java/org/apache/hop/www/RegisterPackageServlet.java
index f38f6535ce..b65e99b9e1 100644
--- a/engine/src/main/java/org/apache/hop/www/RegisterPackageServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/RegisterPackageServlet.java
@@ -93,44 +93,51 @@ public class RegisterPackageServlet extends BaseWorkflowServlet {
SerializableMetadataProvider metadataProvider =
new SerializableMetadataProvider(metaStoreJson);
- if (isWorkflow) {
- Node node =
- getConfigNodeFromZIP(
- archiveUrl,
- Workflow.CONFIGURATION_IN_EXPORT_FILENAME,
- WorkflowExecutionConfiguration.XML_TAG);
- WorkflowExecutionConfiguration workflowExecutionConfiguration =
- new WorkflowExecutionConfiguration(node);
-
- WorkflowMeta workflowMeta =
- new WorkflowMeta(Variables.getADefaultVariableSpace(), fileUrl, metadataProvider);
- WorkflowConfiguration workflowConfiguration =
- new WorkflowConfiguration(
- workflowMeta, workflowExecutionConfiguration, metadataProvider);
-
- IWorkflowEngine<WorkflowMeta> workflow = createWorkflow(workflowConfiguration);
- resultId = workflow.getContainerId();
- } else {
- Node node =
- getConfigNodeFromZIP(
- archiveUrl,
- Pipeline.CONFIGURATION_IN_EXPORT_FILENAME,
- PipelineExecutionConfiguration.XML_TAG);
- PipelineExecutionConfiguration pipelineExecutionConfiguration =
- new PipelineExecutionConfiguration(node);
-
- PipelineMeta pipelineMeta =
- new PipelineMeta(fileUrl, metadataProvider, Variables.getADefaultVariableSpace());
-
- PipelineConfiguration pipelineConfiguration =
- new PipelineConfiguration(
- pipelineMeta, pipelineExecutionConfiguration, metadataProvider);
-
- IPipelineEngine<PipelineMeta> pipeline = createPipeline(pipelineConfiguration);
- resultId = pipeline.getContainerId();
+ int maxConcurrent = HopServerAdmission.parseMaxConcurrent(request);
+ try {
+ if (isWorkflow) {
+ Node node =
+ getConfigNodeFromZIP(
+ archiveUrl,
+ Workflow.CONFIGURATION_IN_EXPORT_FILENAME,
+ WorkflowExecutionConfiguration.XML_TAG);
+ WorkflowExecutionConfiguration workflowExecutionConfiguration =
+ new WorkflowExecutionConfiguration(node);
+
+ WorkflowMeta workflowMeta =
+ new WorkflowMeta(Variables.getADefaultVariableSpace(), fileUrl, metadataProvider);
+ WorkflowConfiguration workflowConfiguration =
+ new WorkflowConfiguration(
+ workflowMeta, workflowExecutionConfiguration, metadataProvider);
+
+ IWorkflowEngine<WorkflowMeta> workflow =
+ createWorkflow(workflowConfiguration, maxConcurrent);
+ resultId = workflow.getContainerId();
+ } else {
+ Node node =
+ getConfigNodeFromZIP(
+ archiveUrl,
+ Pipeline.CONFIGURATION_IN_EXPORT_FILENAME,
+ PipelineExecutionConfiguration.XML_TAG);
+ PipelineExecutionConfiguration pipelineExecutionConfiguration =
+ new PipelineExecutionConfiguration(node);
+
+ PipelineMeta pipelineMeta =
+ new PipelineMeta(fileUrl, metadataProvider, Variables.getADefaultVariableSpace());
+
+ PipelineConfiguration pipelineConfiguration =
+ new PipelineConfiguration(
+ pipelineMeta, pipelineExecutionConfiguration, metadataProvider);
+
+ IPipelineEngine<PipelineMeta> pipeline =
+ createPipeline(pipelineConfiguration, maxConcurrent);
+ resultId = pipeline.getContainerId();
+ }
+
+ return new WebResult(WebResult.STRING_OK, fileUrl, resultId);
+ } catch (HopServerAtCapacityException e) {
+ return new WebResult(HopServerAdmission.RESULT_AT_CAPACITY, e.getMessage());
}
-
- return new WebResult(WebResult.STRING_OK, fileUrl, resultId);
}
return null;
diff --git a/engine/src/main/java/org/apache/hop/www/RegisterPipelineServlet.java b/engine/src/main/java/org/apache/hop/www/RegisterPipelineServlet.java
index 3bc8f81d8b..a8d2ad17bb 100644
--- a/engine/src/main/java/org/apache/hop/www/RegisterPipelineServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/RegisterPipelineServlet.java
@@ -53,13 +53,18 @@ public class RegisterPipelineServlet extends BaseWorkflowServlet {
// Parse the XML, create a pipeline configuration
PipelineConfiguration pipelineConfiguration = PipelineConfiguration.fromXml(xml);
- IPipelineEngine<PipelineMeta> pipeline = createPipeline(pipelineConfiguration);
+ try {
+ IPipelineEngine<PipelineMeta> pipeline =
+ createPipeline(pipelineConfiguration, HopServerAdmission.parseMaxConcurrent(request));
- String message =
- "Pipeline '"
- + pipeline.getPipelineMeta().getName()
- + "' was added to HopServer with id "
- + pipeline.getContainerId();
- return new WebResult(WebResult.STRING_OK, message, pipeline.getContainerId());
+ String message =
+ "Pipeline '"
+ + pipeline.getPipelineMeta().getName()
+ + "' was added to HopServer with id "
+ + pipeline.getContainerId();
+ return new WebResult(WebResult.STRING_OK, message, pipeline.getContainerId());
+ } catch (HopServerAtCapacityException e) {
+ return new WebResult(HopServerAdmission.RESULT_AT_CAPACITY, e.getMessage());
+ }
}
}
diff --git a/engine/src/main/java/org/apache/hop/www/RegisterWorkflowServlet.java b/engine/src/main/java/org/apache/hop/www/RegisterWorkflowServlet.java
index 561b893fc0..308d136bd0 100644
--- a/engine/src/main/java/org/apache/hop/www/RegisterWorkflowServlet.java
+++ b/engine/src/main/java/org/apache/hop/www/RegisterWorkflowServlet.java
@@ -53,13 +53,18 @@ public class RegisterWorkflowServlet extends BaseWorkflowServlet {
// Parse the XML, create a workflow configuration
WorkflowConfiguration workflowConfiguration = WorkflowConfiguration.fromXml(xml, variables);
- IWorkflowEngine<WorkflowMeta> workflow = createWorkflow(workflowConfiguration);
+ try {
+ IWorkflowEngine<WorkflowMeta> workflow =
+ createWorkflow(workflowConfiguration, HopServerAdmission.parseMaxConcurrent(request));
- String message =
- "Workflow '"
- + workflow.getWorkflowName()
- + "' was added to the list with id "
- + workflow.getContainerId();
- return new WebResult(WebResult.STRING_OK, message, workflow.getContainerId());
+ String message =
+ "Workflow '"
+ + workflow.getWorkflowName()
+ + "' was added to the list with id "
+ + workflow.getContainerId();
+ return new WebResult(WebResult.STRING_OK, message, workflow.getContainerId());
+ } catch (HopServerAtCapacityException e) {
+ return new WebResult(HopServerAdmission.RESULT_AT_CAPACITY, e.getMessage());
+ }
}
}
diff --git a/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java b/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java
index 3d782378f1..e879c50690 100644
--- a/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java
+++ b/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java
@@ -271,6 +271,11 @@ public class RemoteHopServer {
// Method is defined as package-protected in order to be accessible by unit tests
HttpPost buildSendExportMethod(IVariables variables, String type, String load, InputStream is) {
+ return buildSendExportMethod(variables, type, load, is, 0);
+ }
+
+ HttpPost buildSendExportMethod(
+ IVariables variables, String type, String load, InputStream is, int maxConcurrent) {
String serviceUrl = RegisterPackageServlet.CONTEXT_PATH;
if (type != null && load != null) {
serviceUrl +=
@@ -281,7 +286,8 @@ public class RemoteHopServer {
+ "&"
+ RegisterPackageServlet.PARAMETER_LOAD
+ "="
- + URLEncoder.encode(load, UTF_8);
+ + URLEncoder.encode(load, UTF_8)
+ + HopServerAdmission.querySuffix(maxConcurrent);
}
String urlString = constructUrl(variables, serviceUrl);
@@ -307,10 +313,16 @@ public class RemoteHopServer {
*/
public String sendExport(IVariables variables, String filename, String type, String load)
throws Exception {
+ return sendExport(variables, filename, type, load, 0);
+ }
+
+ public String sendExport(
+ IVariables variables, String filename, String type, String load, int maxConcurrent)
+ throws Exception {
// Request content will be retrieved directly from the input stream
try (InputStream is = HopVfs.getInputStream(HopVfs.getFileObject(filename))) {
// Execute request
- HttpPost method = buildSendExportMethod(variables, type, load, is);
+ HttpPost method = buildSendExportMethod(variables, type, load, is, maxConcurrent);
try {
return executeAuth(variables, method);
} finally {
diff --git a/engine/src/test/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngineTest.java b/engine/src/test/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngineTest.java
new file mode 100644
index 0000000000..091f94c337
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/pipeline/engines/loadbalance/LoadBalancingPipelineEngineTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.engines.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogLevel;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
+import org.apache.hop.pipeline.engines.local.LocalPipelineRunConfiguration;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+class LoadBalancingPipelineEngineTest {
+
+ @Test
+ void runConfigurationThatRefersToItselfIsRejected() throws Exception {
+ MemoryMetadataProvider metadataProvider = new MemoryMetadataProvider();
+ save(metadataProvider, loadBalancing("lb", "lb"));
+
+ HopException e = assertThrows(HopException.class, () -> prepare(metadataProvider, "lb"));
+
+ assertTrue(e.getMessage().contains("lb -> lb"), "The chain should be reported: " + e);
+ }
+
+ @Test
+ void runConfigurationThatLeadsToALocalOneIsAcceptedUntilServersAreMissing() throws Exception {
+ MemoryMetadataProvider metadataProvider = new MemoryMetadataProvider();
+ save(metadataProvider, loadBalancing("lb", "local"));
+ save(metadataProvider, local("local"));
+
+ HopException e = assertThrows(HopException.class, () -> prepare(metadataProvider, "lb"));
+
+ assertTrue(
+ e.getMessage().contains("no Hop servers") || e.getMessage().contains("No eligible"),
+ "It should fail on the empty server group, not the chain: " + e);
+ }
+
+ private static PipelineRunConfiguration loadBalancing(String name, String runConfigurationName) {
+ LoadBalancingPipelineRunConfiguration engineConfiguration =
+ new LoadBalancingPipelineRunConfiguration();
+ engineConfiguration.setEnginePluginId("LoadBalancing");
+ engineConfiguration.setRunConfigurationName(runConfigurationName);
+ return new PipelineRunConfiguration(
+ name, "", null, new ArrayList<>(), engineConfiguration, null, false);
+ }
+
+ private static PipelineRunConfiguration local(String name) {
+ LocalPipelineRunConfiguration engineConfiguration = new LocalPipelineRunConfiguration();
+ engineConfiguration.setEnginePluginId("Local");
+ return new PipelineRunConfiguration(
+ name, "", null, new ArrayList<>(), engineConfiguration, null, false);
+ }
+
+ private static void save(IHopMetadataProvider metadataProvider, PipelineRunConfiguration c)
+ throws HopException {
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).save(c);
+ }
+
+ private static void prepare(IHopMetadataProvider metadataProvider, String name) throws Exception {
+ LoadBalancingPipelineEngine engine = new LoadBalancingPipelineEngine();
+ engine.setLogLevel(LogLevel.BASIC);
+ engine.setMetadataProvider(metadataProvider);
+ engine.setPipelineRunConfiguration(
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).load(name));
+ engine.setPipelineMeta(new PipelineMeta());
+ engine.prepareExecution();
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/server/loadbalance/HopServerProbeTest.java b/engine/src/test/java/org/apache/hop/server/loadbalance/HopServerProbeTest.java
new file mode 100644
index 0000000000..70ccf4bbfb
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/server/loadbalance/HopServerProbeTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.pipeline.Pipeline;
+import org.apache.hop.www.HopServerPipelineStatus;
+import org.apache.hop.www.HopServerStatus;
+import org.apache.hop.www.HopServerWorkflowStatus;
+import org.junit.jupiter.api.Test;
+
+class HopServerProbeTest {
+
+ @Test
+ void occupyingSlotsIgnoreFinishedAndStopped() {
+ HopServerStatus status = new HopServerStatus();
+ status.setPipelineStatusList(
+ List.of(
+ pipeline(Pipeline.STRING_RUNNING),
+ pipeline(Pipeline.STRING_WAITING),
+ pipeline(Pipeline.STRING_FINISHED),
+ pipeline(Pipeline.STRING_STOPPED)));
+ status.setWorkflowStatusList(
+ List.of(workflow(true, false), workflow(false, true), workflow(false, false)));
+
+ ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
+ HopServerProbe.applyStatus(snapshot, status);
+
+ assertEquals(1, snapshot.getRunningPipelines());
+ assertEquals(1, snapshot.getFinishedPipelines());
+ assertEquals(1, snapshot.getRunningWorkflows());
+ assertEquals(1, snapshot.getFinishedWorkflows());
+ assertEquals(4, snapshot.getOccupyingSlots());
+ assertTrue(snapshot.getLoadAvg() == null || snapshot.getLoadAvg() == 0.0);
+ }
+
+ private static HopServerPipelineStatus pipeline(String statusDescription) {
+ return new HopServerPipelineStatus("p", "id", statusDescription);
+ }
+
+ private static HopServerWorkflowStatus workflow(boolean running, boolean finished) {
+ HopServerWorkflowStatus status = new HopServerWorkflowStatus();
+ if (running) {
+ status.setStatusDescription("Running");
+ } else if (finished) {
+ status.setStatusDescription("Finished");
+ } else {
+ status.setStatusDescription("Waiting");
+ }
+ return status;
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStoreTest.java b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStoreTest.java
new file mode 100644
index 0000000000..620ad10542
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingAssignmentStoreTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(RestoreHopEngineEnvironmentExtension.class)
+class LoadBalancingAssignmentStoreTest {
+
+ @Test
+ void writesAndReadsAssignmentJson() throws Exception {
+ Path folder = Files.createTempDirectory("hop-lb-");
+ LoadBalancingAssignmentStore store =
+ new LoadBalancingAssignmentStore(new Variables(), LogChannel.GENERAL);
+
+ LoadBalancingAssignment assignment = new LoadBalancingAssignment();
+ assignment.setExecutionId("exec-1");
+ assignment.setRunConfigurationName("pool");
+ assignment.setExecutorName("demo");
+ assignment.setServerName("server-a");
+ assignment.setAttempt(2);
+ assignment.setStatus(LoadBalancingAssignment.STATUS_RUNNING);
+ assignment.setAlgorithm(LoadBalancingAlgorithm.EVEN_LOAD.getCode());
+
+ store.save(folder.toString(), assignment);
+ LoadBalancingAssignment loaded = store.load(folder.toString(), "pool", "exec-1");
+
+ assertNotNull(loaded);
+ assertEquals("exec-1", loaded.getExecutionId());
+ assertEquals("server-a", loaded.getServerName());
+ assertEquals(2, loaded.getAttempt());
+ assertEquals(LoadBalancingAssignment.STATUS_RUNNING, loaded.getStatus());
+ }
+
+ @Test
+ void sanitizeReplacesUnsafeCharacters() {
+ assertEquals("a_b.hpl", LoadBalancingAssignmentStore.sanitize("a/b.hpl"));
+ assertEquals("unnamed", LoadBalancingAssignmentStore.sanitize(""));
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
new file mode 100644
index 0000000000..b26fa2f7ad
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingCoordinatorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.www.HopServerAtCapacityException;
+import org.junit.jupiter.api.Test;
+
+class LoadBalancingCoordinatorTest {
+
+ @Test
+ void groupAtCapacityWhenEveryReachableServerIsFull() {
+ ServerHealthSnapshot fullA = eligible("a", 2, 2);
+ ServerHealthSnapshot fullB = eligible("b", 2, 2);
+ assertTrue(LoadBalancingCoordinator.isGroupAtCapacity(List.of(fullA, fullB)));
+ HopException error = LoadBalancingCoordinator.noEligibleServerException(List.of(fullA, fullB));
+ assertInstanceOf(HopServerAtCapacityException.class, error);
+ assertEquals(2, ((HopServerAtCapacityException) error).getOccupyingSlots());
+ assertEquals(2, ((HopServerAtCapacityException) error).getMaxConcurrent());
+ }
+
+ @Test
+ void groupAtCapacityWhenSomeAreUnreachableAndOthersAreFull() {
+ ServerHealthSnapshot down = ServerHealthSnapshot.unavailable("down", true, 2, "timeout");
+ ServerHealthSnapshot full = eligible("full", 2, 2);
+ assertTrue(LoadBalancingCoordinator.isGroupAtCapacity(List.of(down, full)));
+ assertInstanceOf(
+ HopServerAtCapacityException.class,
+ LoadBalancingCoordinator.noEligibleServerException(List.of(down, full)));
+ }
+
+ @Test
+ void notAtCapacityWhenAServerStillHasASlot() {
+ ServerHealthSnapshot full = eligible("full", 2, 2);
+ ServerHealthSnapshot open = eligible("open", 1, 2);
+ assertFalse(LoadBalancingCoordinator.isGroupAtCapacity(List.of(full, open)));
+ }
+
+ @Test
+ void unreachableGroupIsARegularException() {
+ ServerHealthSnapshot down = ServerHealthSnapshot.unavailable("down", true, 2, "timeout");
+ assertFalse(LoadBalancingCoordinator.isGroupAtCapacity(List.of(down)));
+ HopException error = LoadBalancingCoordinator.noEligibleServerException(List.of(down));
+ assertFalse(error instanceof HopServerAtCapacityException);
+ assertTrue(error.getMessage().contains("No eligible Hop server"));
+ }
+
+ private static ServerHealthSnapshot eligible(String name, int occupying, int max) {
+ ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
+ snapshot.setHopServerName(name);
+ snapshot.setEnabled(true);
+ snapshot.setAvailable(true);
+ snapshot.setOccupyingSlots(occupying);
+ snapshot.setMaxConcurrent(max);
+ return snapshot;
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicyTest.java b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicyTest.java
new file mode 100644
index 0000000000..cef0621f38
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingRetryPolicyTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+class LoadBalancingRetryPolicyTest {
+
+ @Test
+ void allowsFirstAttemptAndConfiguredRetries() {
+ LoadBalancingRetryPolicy retry = new LoadBalancingRetryPolicy(2, 0);
+ assertTrue(retry.canAttempt());
+ assertEquals(1, retry.beginAttempt());
+ assertTrue(retry.canAttempt());
+ assertEquals(2, retry.beginAttempt());
+ assertTrue(retry.canAttempt());
+ assertEquals(3, retry.beginAttempt());
+ assertFalse(retry.canAttempt());
+ }
+
+ @Test
+ void windowOfZeroMeansCountOnly() {
+ LoadBalancingRetryPolicy retry = new LoadBalancingRetryPolicy(0, 0);
+ assertTrue(retry.canAttempt());
+ retry.beginAttempt();
+ assertFalse(retry.canAttempt());
+ }
+
+ @Test
+ void expiredWindowStopsFurtherAttempts() {
+ LoadBalancingRetryPolicy retry =
+ new LoadBalancingRetryPolicy(10, 1, System.currentTimeMillis() - 50);
+ assertTrue(retry.canAttempt());
+ retry.beginAttempt();
+ assertFalse(retry.canAttempt());
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingSelectorTest.java b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingSelectorTest.java
new file mode 100644
index 0000000000..0332b142d4
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/server/loadbalance/LoadBalancingSelectorTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.server.loadbalance;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+class LoadBalancingSelectorTest {
+
+ @Test
+ void evenLoadPicksTheLeastUtilizedServer() {
+ ServerHealthSnapshot busy = eligible("busy", 3, 4, 0.1, 10);
+ ServerHealthSnapshot idle = eligible("idle", 0, 4, 2.0, 50);
+ Optional<ServerHealthSnapshot> selected =
+ LoadBalancingSelector.select(LoadBalancingAlgorithm.EVEN_LOAD, List.of(busy, idle));
+ assertTrue(selected.isPresent());
+ assertEquals("idle", selected.get().getHopServerName());
+ }
+
+ @Test
+ void packPicksTheFullestEligibleServer() {
+ ServerHealthSnapshot busy = eligible("busy", 3, 4, 2.0, 50);
+ ServerHealthSnapshot idle = eligible("idle", 0, 4, 0.1, 10);
+ Optional<ServerHealthSnapshot> selected =
+ LoadBalancingSelector.select(LoadBalancingAlgorithm.PACK, List.of(busy, idle));
+ assertTrue(selected.isPresent());
+ assertEquals("busy", selected.get().getHopServerName());
+ }
+
+ @Test
+ void skipsDisabledDownShuttingDownAndFullServers() {
+ ServerHealthSnapshot disabled = eligible("disabled", 0, 4, 0, 1);
+ disabled.setEnabled(false);
+ ServerHealthSnapshot down = ServerHealthSnapshot.unavailable("down", true, 4, "timeout");
+ ServerHealthSnapshot draining = eligible("draining", 0, 4, 0, 1);
+ draining.setShuttingDown(true);
+ ServerHealthSnapshot full = eligible("full", 4, 4, 0, 1);
+ ServerHealthSnapshot ok = eligible("ok", 1, 4, 0, 1);
+
+ Optional<ServerHealthSnapshot> selected =
+ LoadBalancingSelector.select(
+ LoadBalancingAlgorithm.EVEN_LOAD, List.of(disabled, down, draining, full, ok));
+ assertTrue(selected.isPresent());
+ assertEquals("ok", selected.get().getHopServerName());
+ }
+
+ @Test
+ void emptyWhenNothingIsEligible() {
+ Optional<ServerHealthSnapshot> selected =
+ LoadBalancingSelector.select(
+ LoadBalancingAlgorithm.EVEN_LOAD,
+ List.of(ServerHealthSnapshot.unavailable("a", true, 4, "down")));
+ assertTrue(selected.isEmpty());
+ }
+
+ private static ServerHealthSnapshot eligible(
+ String name, int occupying, int max, double loadAvg, long responseNs) {
+ ServerHealthSnapshot snapshot = new ServerHealthSnapshot();
+ snapshot.setHopServerName(name);
+ snapshot.setEnabled(true);
+ snapshot.setAvailable(true);
+ snapshot.setOccupyingSlots(occupying);
+ snapshot.setMaxConcurrent(max);
+ snapshot.setLoadAvg(loadAvg);
+ snapshot.setCpuCores(1);
+ snapshot.setResponseNs(responseNs);
+ return snapshot;
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/www/HopServerAdmissionTest.java b/engine/src/test/java/org/apache/hop/www/HopServerAdmissionTest.java
new file mode 100644
index 0000000000..69e5a51cec
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/www/HopServerAdmissionTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.www;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.pipeline.PipelineConfiguration;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.junit.jupiter.api.Test;
+
+class HopServerAdmissionTest {
+
+ @Test
+ void parseMaxConcurrentTreatsBlankAsUnlimited() {
+ assertEquals(0, HopServerAdmission.parseMaxConcurrent((String) null));
+ assertEquals(0, HopServerAdmission.parseMaxConcurrent(""));
+ assertEquals(4, HopServerAdmission.parseMaxConcurrent("4"));
+ }
+
+ @Test
+ void querySuffixOnlyWhenCapped() {
+ assertEquals("", HopServerAdmission.querySuffix(0));
+ assertEquals("&max_concurrent=3", HopServerAdmission.querySuffix(3));
+ }
+
+ @Test
+ void finishedPipelinesDoNotOccupySlots() {
+ PipelineMap pipelines = new PipelineMap();
+ IPipelineEngine<PipelineMeta> running = mockEngine(false);
+ IPipelineEngine<PipelineMeta> finished = mockEngine(true);
+ pipelines.addPipeline("run", "1", running, mock(PipelineConfiguration.class));
+ pipelines.addPipeline("done", "2", finished, mock(PipelineConfiguration.class));
+
+ assertEquals(1, HopServerAdmission.countOccupyingSlots(pipelines, new WorkflowMap()));
+ }
+
+ @Test
+ void admitAndAddRefusesWhenAtCapacity() {
+ PipelineMap pipelines = new PipelineMap();
+ pipelines.addPipeline("run", "1", mockEngine(false), mock(PipelineConfiguration.class));
+ pipelines.addPipeline("run2", "2", mockEngine(false), mock(PipelineConfiguration.class));
+
+ assertThrows(
+ HopServerAtCapacityException.class,
+ () -> HopServerAdmission.admitAndAdd(pipelines, new WorkflowMap(), 2, () -> {}));
+ }
+
+ @Test
+ void admitAndAddAllowsWhenUnderCapacity() throws HopServerAtCapacityException {
+ PipelineMap pipelines = new PipelineMap();
+ pipelines.addPipeline("run", "1", mockEngine(false), mock(PipelineConfiguration.class));
+ boolean[] added = {false};
+ HopServerAdmission.admitAndAdd(pipelines, new WorkflowMap(), 2, () -> added[0] = true);
+ assertTrue(added[0]);
+ }
+
+ @Test
+ void retryableOnlyBeforeAContainerIsAssigned() {
+ HopException atCapacity = new HopServerAtCapacityException(2, 2);
+ assertTrue(HopServerAdmission.isRetryableRegistrationFailure(atCapacity, null));
+ assertFalse(HopServerAdmission.isRetryableRegistrationFailure(atCapacity, "container-1"));
+ assertTrue(
+ HopServerAdmission.isRetryableRegistrationFailure(
+ new HopException("SERVER_AT_CAPACITY: 2 occupying slots, max 2"), null));
+ assertTrue(
+ HopServerAdmission.isRetryableRegistrationFailure(
+ new HopException("Error preparing load-balanced pipeline", atCapacity), null));
+ assertTrue(
+ HopServerAdmission.isRetryableRegistrationFailure(
+ new HopException(
+ "No eligible Hop server in the load-balancing group:\n - hop-server-a: at capacity (2/2)"),
+ null));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static IPipelineEngine<PipelineMeta> mockEngine(boolean finished) {
+ IPipelineEngine<PipelineMeta> engine = mock(IPipelineEngine.class);
+ when(engine.isFinished()).thenReturn(finished);
+ return engine;
+ }
+}
diff --git a/integration-tests/load-balance/.gitignore b/integration-tests/load-balance/.gitignore
new file mode 100644
index 0000000000..cabe184273
--- /dev/null
+++ b/integration-tests/load-balance/.gitignore
@@ -0,0 +1,2 @@
+output/
+state/
diff --git a/integration-tests/load-balance/0001-child-delay.hpl b/integration-tests/load-balance/0001-child-delay.hpl
new file mode 100644
index 0000000000..14879c8689
--- /dev/null
+++ b/integration-tests/load-balance/0001-child-delay.hpl
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0001-child-delay</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Small worker pipeline that occupies a server slot for 2 seconds</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/08/18 12:53:30.940</created_date>
+ <modified_date>2026/08/18 12:53:30.940</modified_date>
+ </info>
+ <transform>
+ <type>RowGenerator</type>
+ <name>1 row</name>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <limit>1</limit>
+ <fields/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Delay</type>
+ <name>Delay 1s</name>
+ <timeout>1</timeout>
+ <scaletime>seconds</scaletime>
+ <scaletime_from_field>N</scaletime_from_field>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>Dummy</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>448</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>1 row</from>
+ <to>Delay 1s</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Delay 1s</from>
+ <to>Dummy</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/load-balance/0001-child-delay.hwf b/integration-tests/load-balance/0001-child-delay.hwf
new file mode 100644
index 0000000000..2af7a5bb36
--- /dev/null
+++ b/integration-tests/load-balance/0001-child-delay.hwf
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0001-child-delay</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Small worker workflow that occupies a server slot for 2 seconds</description>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>64</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0001-child-delay.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0001-child-delay.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>240</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Success</name>
+ <description/>
+ <type>SUCCESS</type>
+ <attributes/>
+ <parallel>N</parallel>
+ <xloc>448</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0001-child-delay.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0001-child-delay.hpl</from>
+ <to>Success</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/load-balance/0001-drive-100-pipelines.hpl b/integration-tests/load-balance/0001-drive-100-pipelines.hpl
new file mode 100644
index 0000000000..3038dab338
--- /dev/null
+++ b/integration-tests/load-balance/0001-drive-100-pipelines.hpl
@@ -0,0 +1,182 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0001-drive-100-pipelines</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Submit 100 child pipelines through the load-balancing run configuration</description>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>100 rows</from>
+ <to>Pipeline executor</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Pipeline executor</from>
+ <to>Count successes</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Count successes</from>
+ <to>Set EXEC_OK</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>100 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>100</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Pipeline executor</name>
+ <type>PipelineExecutor</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>4</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <run_configuration>load-balance</run_configuration>
+ <filename>${PROJECT_HOME}/0001-child-delay.hpl</filename>
+ <filenameInField>N</filenameInField>
+ <group_size>1</group_size>
+ <group_field/>
+ <group_time/>
+ <parameters>
+ <inherit_all_vars>Y</inherit_all_vars>
+ </parameters>
+ <execution_result_target_transform>Count successes</execution_result_target_transform>
+ <execution_time_field>ExecutionTime</execution_time_field>
+ <execution_result_field>ExecutionResult</execution_result_field>
+ <execution_errors_field>ExecutionNrErrors</execution_errors_field>
+ <execution_lines_read_field/>
+ <execution_lines_written_field/>
+ <execution_lines_input_field/>
+ <execution_lines_output_field/>
+ <execution_lines_rejected_field/>
+ <execution_lines_updated_field/>
+ <execution_lines_deleted_field/>
+ <execution_files_retrieved_field/>
+ <execution_exit_status_field/>
+ <execution_log_text_field/>
+ <execution_log_channelid_field/>
+ <result_rows_target_transform/>
+ <result_files_target_transform/>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Count successes</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <add_linenr>N</add_linenr>
+ <all_rows>N</all_rows>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <aggregate>okCount</aggregate>
+ <type>COUNT_ANY</type>
+ </field>
+ <field>
+ <aggregate>errorSum</aggregate>
+ <subject>ExecutionNrErrors</subject>
+ <type>SUM</type>
+ </field>
+ </fields>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <ignore_aggregate>N</ignore_aggregate>
+ <prefix>grp</prefix>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Set EXEC_OK</name>
+ <type>SetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <field_name>okCount</field_name>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ <field>
+ <field_name>errorSum</field_name>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ </fields>
+ <use_formatting>Y</use_formatting>
+ <attributes/>
+ <GUI>
+ <xloc>704</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/load-balance/0001-server-status.hpl b/integration-tests/load-balance/0001-server-status.hpl
new file mode 100644
index 0000000000..e744115ee7
--- /dev/null
+++ b/integration-tests/load-balance/0001-server-status.hpl
@@ -0,0 +1,244 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0001-server-status</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Read finished pipeline and workflow counts from both Hop Servers</description>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Servers</from>
+ <to>Get Server Status</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Get Server Status</from>
+ <to>Server A?</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server A?</from>
+ <to>Set server A</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server A?</from>
+ <to>Server B?</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server B?</from>
+ <to>Set server B</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Servers</name>
+ <type>DataGrid</type>
+ <fields>
+ <field>
+ <name>serverName</name>
+ <type>String</type>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <data>
+ <line>
+ <item>hop-server-a</item>
+ </line>
+ <line>
+ <item>hop-server-b</item>
+ </line>
+ </data>
+ <distribute>N</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <name>Get Server Status</name>
+ <type>GetServerStatus</type>
+ <description/>
+ <distribute>N</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <server_field>serverName</server_field>
+ <error_message_field>errorMessage</error_message_field>
+ <status_description_field/>
+ <server_load_field/>
+ <active_pipelines_field>activePipelines</active_pipelines_field>
+ <finished_pipelines_field>finishedPipelines</finished_pipelines_field>
+ <active_workflows_field>activeWorkflows</active_workflows_field>
+ <finished_workflows_field>finishedWorkflows</finished_workflows_field>
+ <available_field>available</available_field>
+ <response_ns_field/>
+ <attributes/>
+ <GUI>
+ <xloc>272</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Server A?</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to>Set server A</send_true_to>
+ <send_false_to>Server B?</send_false_to>
+ <compare>
+ <condition>
+ <function>=</function>
+ <leftvalue>serverName</leftvalue>
+ <negated>N</negated>
+ <operator>-</operator>
+ <value>
+ <isnull>N</isnull>
+ <name>constant</name>
+ <text>hop-server-a</text>
+ <type>String</type>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Set server A</name>
+ <type>SetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <field_name>finishedPipelines</field_name>
+ <variable_name>SERVER_A_PIPELINES</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ <field>
+ <field_name>finishedWorkflows</field_name>
+ <variable_name>SERVER_A_WORKFLOWS</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ </fields>
+ <use_formatting>Y</use_formatting>
+ <attributes/>
+ <GUI>
+ <xloc>688</xloc>
+ <yloc>32</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Server B?</name>
+ <type>FilterRows</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <send_true_to>Set server B</send_true_to>
+ <compare>
+ <condition>
+ <function>=</function>
+ <leftvalue>serverName</leftvalue>
+ <negated>N</negated>
+ <operator>-</operator>
+ <value>
+ <isnull>N</isnull>
+ <name>constant</name>
+ <text>hop-server-b</text>
+ <type>String</type>
+ </value>
+ </condition>
+ </compare>
+ <attributes/>
+ <GUI>
+ <xloc>480</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Set server B</name>
+ <type>SetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <field_name>finishedPipelines</field_name>
+ <variable_name>SERVER_B_PIPELINES</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ <field>
+ <field_name>finishedWorkflows</field_name>
+ <variable_name>SERVER_B_WORKFLOWS</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ </fields>
+ <use_formatting>Y</use_formatting>
+ <attributes/>
+ <GUI>
+ <xloc>688</xloc>
+ <yloc>176</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/load-balance/0002-drive-100-workflows.hpl b/integration-tests/load-balance/0002-drive-100-workflows.hpl
new file mode 100644
index 0000000000..969b0bb486
--- /dev/null
+++ b/integration-tests/load-balance/0002-drive-100-workflows.hpl
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0002-drive-100-workflows</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Submit 100 child workflows through the load-balancing run configuration</description>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>100 rows</from>
+ <to>Workflow executor</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Workflow executor</from>
+ <to>Count successes</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Count successes</from>
+ <to>Set EXEC_OK</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>100 rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ </fields>
+ <limit>100</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field/>
+ <last_time_field/>
+ <attributes/>
+ <GUI>
+ <xloc>80</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Workflow executor</name>
+ <type>WorkflowExecutor</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>4</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <run_configuration>load-balance</run_configuration>
+ <filename>${PROJECT_HOME}/0001-child-delay.hwf</filename>
+ <group_size>1</group_size>
+ <group_field/>
+ <group_time/>
+ <parameters>
+ <inherit_all_vars>Y</inherit_all_vars>
+ </parameters>
+ <execution_result_target_transform>Count successes</execution_result_target_transform>
+ <execution_time_field>ExecutionTime</execution_time_field>
+ <execution_result_field>ExecutionResult</execution_result_field>
+ <execution_errors_field>ExecutionNrErrors</execution_errors_field>
+ <execution_lines_read_field/>
+ <execution_lines_written_field/>
+ <execution_lines_input_field/>
+ <execution_lines_output_field/>
+ <execution_lines_rejected_field/>
+ <execution_lines_updated_field/>
+ <execution_lines_deleted_field/>
+ <execution_files_retrieved_field/>
+ <execution_exit_status_field/>
+ <execution_log_text_field/>
+ <execution_log_channelid_field/>
+ <result_rows_target_transform/>
+ <result_files_target_transform/>
+ <attributes/>
+ <GUI>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Count successes</name>
+ <type>GroupBy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <add_linenr>N</add_linenr>
+ <all_rows>N</all_rows>
+ <directory>${java.io.tmpdir}</directory>
+ <fields>
+ <field>
+ <aggregate>okCount</aggregate>
+ <type>COUNT_ANY</type>
+ </field>
+ <field>
+ <aggregate>errorSum</aggregate>
+ <subject>ExecutionNrErrors</subject>
+ <type>SUM</type>
+ </field>
+ </fields>
+ <give_back_row>N</give_back_row>
+ <group>
+ </group>
+ <ignore_aggregate>N</ignore_aggregate>
+ <prefix>grp</prefix>
+ <attributes/>
+ <GUI>
+ <xloc>512</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Set EXEC_OK</name>
+ <type>SetVariable</type>
+ <description/>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <field_name>okCount</field_name>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ <field>
+ <field_name>errorSum</field_name>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_type>PARENT_WORKFLOW</variable_type>
+ <default_value>0</default_value>
+ </field>
+ </fields>
+ <use_formatting>Y</use_formatting>
+ <attributes/>
+ <GUI>
+ <xloc>704</xloc>
+ <yloc>80</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/load-balance/dev-env-config.json b/integration-tests/load-balance/dev-env-config.json
new file mode 100644
index 0000000000..0463fd4e3a
--- /dev/null
+++ b/integration-tests/load-balance/dev-env-config.json
@@ -0,0 +1,3 @@
+{
+ "variables" : [ ]
+}
diff --git a/integration-tests/load-balance/hop-config.json b/integration-tests/load-balance/hop-config.json
new file mode 100644
index 0000000000..382bdc5e12
--- /dev/null
+++ b/integration-tests/load-balance/hop-config.json
@@ -0,0 +1,33 @@
+{
+ "variables": [],
+ "LocaleDefault": "en_BE",
+ "guiProperties": {
+ },
+ "projectsConfig": {
+ "enabled": true,
+ "projectMandatory": true,
+ "environmentMandatory": true,
+ "defaultProject": "default",
+ "defaultEnvironment": null,
+ "standardParentProject": "default",
+ "standardProjectsFolder": null,
+ "projectConfigurations": [
+ {
+ "projectName": "default",
+ "projectHome": "${HOP_CONFIG_FOLDER}",
+ "configFilename": "project-config.json"
+ }
+ ],
+ "lifecycleEnvironments": [
+ {
+ "name": "dev",
+ "purpose": "Testing",
+ "projectName": "default",
+ "configurationFiles": [
+ "${PROJECT_HOME}/dev-env-config.json"
+ ]
+ }
+ ],
+ "projectLifecycles": []
+ }
+}
diff --git a/integration-tests/load-balance/main-0001-load-balance-pipeline-executor.hwf b/integration-tests/load-balance/main-0001-load-balance-pipeline-executor.hwf
new file mode 100644
index 0000000000..46e5ef3d47
--- /dev/null
+++ b/integration-tests/load-balance/main-0001-load-balance-pipeline-executor.hwf
@@ -0,0 +1,261 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0001-load-balance-pipeline-executor</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Run 100 child pipelines through the load-balancing pool, then read finished counts from both Hop Servers</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/08/18 14:42:45.312</created_date>
+ <modified_date>2026/08/18 14:42:45.312</modified_date>
+ <parameters/>
+ <actions>
+ <action>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <weekDay>1</weekDay>
+ <minutes>0</minutes>
+ <hour>12</hour>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <xloc>48</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <replacevars>Y</replacevars>
+ <fields>
+ <field>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>SERVER_A_PIPELINES</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>SERVER_B_PIPELINES</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ </fields>
+ <file_variable_type>CURRENT_WORKFLOW</file_variable_type>
+ <name>Init variables</name>
+ <description/>
+ <type>SET_VARIABLES</type>
+ <attributes/>
+ <xloc>192</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0001-drive-100-pipelines.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Drive 100 pipelines</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0001-server-status.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Check server status</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>576</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>SERVER_A_PIPELINES</variablename>
+ <comparevalue>1</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>greaterequal</successnumbercondition>
+ <name>Server A used</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>192</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>SERVER_B_PIPELINES</variablename>
+ <comparevalue>1</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>greaterequal</successnumbercondition>
+ <name>Server B used</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_OK</variablename>
+ <comparevalue>100</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>100 executions</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>576</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_ERRORS</variablename>
+ <comparevalue>0</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>No execution errors</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>768</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Init variables</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Init variables</from>
+ <to>Drive 100 pipelines</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Drive 100 pipelines</from>
+ <to>Check server status</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check server status</from>
+ <to>Server A used</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server A used</from>
+ <to>Server B used</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server B used</from>
+ <to>100 executions</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>100 executions</from>
+ <to>No execution errors</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ </hops>
+ <notepads/>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/load-balance/main-0002-load-balance-workflow-executor.hwf b/integration-tests/load-balance/main-0002-load-balance-workflow-executor.hwf
new file mode 100644
index 0000000000..e89979ec02
--- /dev/null
+++ b/integration-tests/load-balance/main-0002-load-balance-workflow-executor.hwf
@@ -0,0 +1,261 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0002-load-balance-workflow-executor</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Run 100 child workflows through the load-balancing pool, then read finished counts from both Hop Servers</description>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/08/18 14:42:45.408</created_date>
+ <modified_date>2026/08/18 14:42:45.408</modified_date>
+ <parameters/>
+ <actions>
+ <action>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <DayOfMonth>1</DayOfMonth>
+ <weekDay>1</weekDay>
+ <minutes>0</minutes>
+ <hour>12</hour>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <xloc>48</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <replacevars>Y</replacevars>
+ <fields>
+ <field>
+ <variable_name>EXEC_OK</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>EXEC_ERRORS</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>SERVER_A_WORKFLOWS</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ <field>
+ <variable_name>SERVER_B_WORKFLOWS</variable_name>
+ <variable_value>0</variable_value>
+ <variable_type>CURRENT_WORKFLOW</variable_type>
+ </field>
+ </fields>
+ <file_variable_type>CURRENT_WORKFLOW</file_variable_type>
+ <name>Init variables</name>
+ <description/>
+ <type>SET_VARIABLES</type>
+ <attributes/>
+ <xloc>192</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0002-drive-100-workflows.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Drive 100 workflows</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <filename>${PROJECT_HOME}/0001-server-status.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <create_parent_folder>N</create_parent_folder>
+ <set_logfile>N</set_logfile>
+ <set_append_logfile>N</set_append_logfile>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <run_configuration>local</run_configuration>
+ <name>Check server status</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <xloc>576</xloc>
+ <yloc>160</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>SERVER_A_WORKFLOWS</variablename>
+ <comparevalue>1</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>greaterequal</successnumbercondition>
+ <name>Server A used</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>192</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>SERVER_B_WORKFLOWS</variablename>
+ <comparevalue>1</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>greaterequal</successnumbercondition>
+ <name>Server B used</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>384</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_OK</variablename>
+ <comparevalue>100</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>100 executions</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>576</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ <action>
+ <valuetype>variable</valuetype>
+ <fieldtype>number</fieldtype>
+ <variablename>EXEC_ERRORS</variablename>
+ <comparevalue>0</comparevalue>
+ <successcondition>equal</successcondition>
+ <successwhenvarset>N</successwhenvarset>
+ <successbooleancondition>true</successbooleancondition>
+ <successnumbercondition>equal</successnumbercondition>
+ <name>No execution errors</name>
+ <description/>
+ <type>SIMPLE_EVAL</type>
+ <attributes/>
+ <xloc>768</xloc>
+ <yloc>272</yloc>
+ <parallel>N</parallel>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Init variables</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Init variables</from>
+ <to>Drive 100 workflows</to>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Drive 100 workflows</from>
+ <to>Check server status</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Check server status</from>
+ <to>Server A used</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server A used</from>
+ <to>Server B used</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Server B used</from>
+ <to>100 executions</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>100 executions</from>
+ <to>No execution errors</to>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ <enabled>Y</enabled>
+ </hop>
+ </hops>
+ <notepads/>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json b/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
new file mode 100644
index 0000000000..4db3be0aaf
--- /dev/null
+++ b/integration-tests/load-balance/metadata/pipeline-run-configuration/load-balance.json
@@ -0,0 +1,37 @@
+{
+ "engineRunConfiguration": {
+ "LoadBalancing": {
+ "run_config": "local",
+ "algorithm": "Even load",
+ "max_retries": "10",
+ "retry_window_ms": "30000",
+ "retry_on_execution_failure": false,
+ "probe_timeout_ms": "3000",
+ "config_refresh_ms": "2000",
+ "state_folder": "${PROJECT_HOME}/state",
+ "server_poll_delay": "200",
+ "server_poll_interval": "500",
+ "export_resources": true,
+ "resources_source_folder": "${PROJECT_HOME}",
+ "resources_target_folder": "",
+ "server": [
+ {
+ "hop_server": "hop-server-a",
+ "enabled": true,
+ "max_concurrent": "2"
+ },
+ {
+ "hop_server": "hop-server-b",
+ "enabled": true,
+ "max_concurrent": "2"
+ }
+ ]
+ }
+ },
+ "defaultSelection": false,
+ "configurationVariables": [],
+ "name": "load-balance",
+ "description": "Assigns each pipeline execution to hop-server-a or hop-server-b",
+ "dataProfile": "",
+ "executionInfoLocationName": ""
+}
diff --git a/integration-tests/load-balance/metadata/pipeline-run-configuration/local.json b/integration-tests/load-balance/metadata/pipeline-run-configuration/local.json
new file mode 100644
index 0000000000..a147ae48a2
--- /dev/null
+++ b/integration-tests/load-balance/metadata/pipeline-run-configuration/local.json
@@ -0,0 +1,25 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "feedback_size": "50000",
+ "sample_size": "100",
+ "sample_type_in_gui": "Last",
+ "wait_time": "20",
+ "rowset_size": "10000",
+ "safe_mode": false,
+ "show_feedback": false,
+ "topo_sort": false,
+ "gather_metrics": false,
+ "transactional": false,
+ "detect_buffer_deadlocks": true,
+ "mitigate_buffer_deadlocks": true,
+ "buffer_deadlock_spill_directory": ""
+ }
+ },
+ "defaultSelection": true,
+ "configurationVariables": [],
+ "name": "local",
+ "description": "Runs pipelines locally with the standard local Hop pipeline engine",
+ "dataProfile": "",
+ "executionInfoLocationName": ""
+}
diff --git a/integration-tests/load-balance/metadata/server/hop-server-a.json b/integration-tests/load-balance/metadata/server/hop-server-a.json
new file mode 100644
index 0000000000..13ece3c753
--- /dev/null
+++ b/integration-tests/load-balance/metadata/server/hop-server-a.json
@@ -0,0 +1,13 @@
+{
+ "webAppName": "",
+ "sslMode": false,
+ "proxyPort": "",
+ "hostname": "hop-server-a",
+ "nonProxyHosts": "",
+ "overrideExistingProperties": false,
+ "password": "Encrypted 2be98afc86aa7f2e4cb1aa265cd86aac8",
+ "port": "8181",
+ "name": "hop-server-a",
+ "proxyHostname": "",
+ "username": "cluster"
+}
diff --git a/integration-tests/load-balance/metadata/server/hop-server-b.json b/integration-tests/load-balance/metadata/server/hop-server-b.json
new file mode 100644
index 0000000000..dc6fc4efdd
--- /dev/null
+++ b/integration-tests/load-balance/metadata/server/hop-server-b.json
@@ -0,0 +1,13 @@
+{
+ "webAppName": "",
+ "sslMode": false,
+ "proxyPort": "",
+ "hostname": "hop-server-b",
+ "nonProxyHosts": "",
+ "overrideExistingProperties": false,
+ "password": "Encrypted 2be98afc86aa7f2e4cb1aa265cd86aac8",
+ "port": "8181",
+ "name": "hop-server-b",
+ "proxyHostname": "",
+ "username": "cluster"
+}
diff --git a/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json b/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
new file mode 100644
index 0000000000..0cfe08588c
--- /dev/null
+++ b/integration-tests/load-balance/metadata/workflow-run-configuration/load-balance.json
@@ -0,0 +1,35 @@
+{
+ "engineRunConfiguration": {
+ "LoadBalancing": {
+ "run_config": "local",
+ "algorithm": "Even load",
+ "max_retries": "10",
+ "retry_window_ms": "30000",
+ "retry_on_execution_failure": false,
+ "probe_timeout_ms": "3000",
+ "config_refresh_ms": "2000",
+ "state_folder": "${PROJECT_HOME}/state",
+ "server_poll_delay": "200",
+ "server_poll_interval": "500",
+ "export_resources": true,
+ "resources_source_folder": "${PROJECT_HOME}",
+ "resources_target_folder": "",
+ "server": [
+ {
+ "hop_server": "hop-server-a",
+ "enabled": true,
+ "max_concurrent": "2"
+ },
+ {
+ "hop_server": "hop-server-b",
+ "enabled": true,
+ "max_concurrent": "2"
+ }
+ ]
+ }
+ },
+ "defaultSelection": false,
+ "name": "load-balance",
+ "description": "Assigns each workflow execution to hop-server-a or hop-server-b",
+ "executionInfoLocationName": ""
+}
diff --git a/integration-tests/load-balance/metadata/workflow-run-configuration/local.json b/integration-tests/load-balance/metadata/workflow-run-configuration/local.json
new file mode 100644
index 0000000000..55dedee5ad
--- /dev/null
+++ b/integration-tests/load-balance/metadata/workflow-run-configuration/local.json
@@ -0,0 +1,11 @@
+{
+ "engineRunConfiguration": {
+ "Local": {
+ "safe_mode": false
+ }
+ },
+ "defaultSelection": true,
+ "name": "local",
+ "description": "Runs workflows locally with the standard local Hop workflow engine",
+ "executionInfoLocationName": ""
+}
diff --git a/integration-tests/load-balance/project-config.json b/integration-tests/load-balance/project-config.json
new file mode 100644
index 0000000000..a276308fc2
--- /dev/null
+++ b/integration-tests/load-balance/project-config.json
@@ -0,0 +1,14 @@
+{
+ "metadataBaseFolder" : "${PROJECT_HOME}/metadata",
+ "unitTestsBasePath" : "${PROJECT_HOME}",
+ "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets",
+ "enforcingExecutionInHome" : true,
+ "parentProjectName" : "",
+ "config" : {
+ "variables" : [ {
+ "name" : "HOP_LICENSE_HEADER_FILE",
+ "value" : "${PROJECT_HOME}/../asf-header.txt",
+ "description" : "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects"
+ } ]
+ }
+}
diff --git a/integration-tests/scripts/run-tests-docker.sh b/integration-tests/scripts/run-tests-docker.sh
index ba24cf199b..ad8213f4df 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -473,16 +473,16 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
echo "Project compose exists."
EXECUTED_COMPOSE_FILES=("${EXECUTED_COMPOSE_FILES[@]}" "${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml")
# Rebuild project images so SPARK_VERSION (and similar) build args take effect.
- # hop_server also must rebuild: its hop-server service image (apache/hop:Development
- # from docker/Dockerfile) otherwise stays cached and can miss client-side assembly
- # plugins needed by remote-export ITs (main-0008/0009/0010).
+ # hop_server and load-balance must rebuild: their hop-server service image
+ # (apache/hop:Development from docker/Dockerfile) otherwise stays cached and can
+ # miss client-side assembly plugins (remote-export ITs, LoadBalancing engine).
if [ "${PROJECT_NAME}" = "spark" ]; then
echo "Spark IT cluster version: ${SPARK_VERSION} (hadoop ${HADOOP_VERSION})"
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER} SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} SPARK_VERSION=${SPARK_VERSION} HADOOP_VERSION=${HADOOP_VERSION} SPARK_BASE_URL=${SPARK_BASE_URL} \
docker compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build --abort-on-container-exit \
|| COMPOSE_EXIT=$?
- elif [ "${PROJECT_NAME}" = "hop_server" ]; then
- echo "Rebuilding hop_server images so remote Hop Server matches current assemblies"
+ elif [ "${PROJECT_NAME}" = "hop_server" ] || [ "${PROJECT_NAME}" = "load-balance" ]; then
+ echo "Rebuilding ${PROJECT_NAME} images so remote Hop Server matches current assemblies"
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER} SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
docker compose -f ${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build --abort-on-container-exit \
|| COMPOSE_EXIT=$?
diff --git a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java
index 7edf90a73a..20a5b6684a 100644
--- a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java
+++ b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java
@@ -44,6 +44,7 @@ import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.ColumnsResizer;
import org.apache.hop.ui.core.widget.ComboVar;
+import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
@@ -88,8 +89,7 @@ public class PipelineExecutorDialog extends BaseTransformDialog {
private Label wlPath;
private TextVar wPath;
- protected Label wlRunConfiguration;
- protected ComboVar wRunConfiguration;
+ protected MetaSelectionLine<PipelineRunConfiguration> wRunConfiguration;
private Button wbPipelineNameInField;
@@ -240,24 +240,20 @@ public class PipelineExecutorDialog extends BaseTransformDialog {
});
/* End */
- wlRunConfiguration = new Label(shell, SWT.RIGHT);
- wlRunConfiguration.setText(
- BaseMessages.getString(PKG, "PipelineExecutorDialog.RunConfiguration.Label"));
- PropsUi.setLook(wlRunConfiguration);
- FormData fdlRunConfiguration = new FormData();
- fdlRunConfiguration.left = new FormAttachment(0, 0);
- fdlRunConfiguration.top = new FormAttachment(wPipelineNameField, margin);
- fdlRunConfiguration.right = new FormAttachment(middle, -margin);
- wlRunConfiguration.setLayoutData(fdlRunConfiguration);
-
- wRunConfiguration = new ComboVar(variables, shell, SWT.LEFT | SWT.BORDER);
- PropsUi.setLook(wlRunConfiguration);
+ wRunConfiguration =
+ new MetaSelectionLine<>(
+ variables,
+ metadataProvider,
+ PipelineRunConfiguration.class,
+ shell,
+ SWT.SINGLE | SWT.LEFT | SWT.BORDER,
+ BaseMessages.getString(PKG, "PipelineExecutorDialog.RunConfiguration.Label"),
+ BaseMessages.getString(PKG, "PipelineExecutorDialog.RunConfiguration.Tooltip"));
FormData fdRunConfiguration = new FormData();
- fdRunConfiguration.left = new FormAttachment(middle, 0);
- fdRunConfiguration.top = new FormAttachment(wlRunConfiguration, 0, SWT.CENTER);
+ fdRunConfiguration.left = new FormAttachment(0, 0);
+ fdRunConfiguration.top = new FormAttachment(wPipelineNameField, margin);
fdRunConfiguration.right = new FormAttachment(100, 0);
wRunConfiguration.setLayoutData(fdRunConfiguration);
- PropsUi.setLook(wRunConfiguration);
//
// Add a tab folder for the parameters and various input and output
diff --git a/plugins/transforms/pipelineexecutor/src/main/resources/org/apache/hop/pipeline/transforms/pipelineexecutor/messages/messages_en_US.properties b/plugins/transforms/pipelineexecutor/src/main/resources/org/apache/hop/pipeline/transforms/pipelineexecutor/messages/messages_en_US.properties
index 8fb00a2948..781cc76a8c 100644
--- a/plugins/transforms/pipelineexecutor/src/main/resources/org/apache/hop/pipeline/transforms/pipelineexecutor/messages/messages_en_US.properties
+++ b/plugins/transforms/pipelineexecutor/src/main/resources/org/apache/hop/pipeline/transforms/pipelineexecutor/messages/messages_en_US.properties
@@ -71,6 +71,7 @@ PipelineExecutorDialog.ResultRows.Tooltip=You can specify the target and specifi
PipelineExecutorDialog.RowGroup.Title=Row grouping
PipelineExecutorDialog.RowGroup.Tooltip=Determine how many rows of data are passed to the pipeline before execution
PipelineExecutorDialog.RunConfiguration.Label=Run configuration
+PipelineExecutorDialog.RunConfiguration.Tooltip=The pipeline run configuration used to execute the specified pipeline. Use the buttons to create, edit or open the selected configuration.
PipelineExecutorDialog.Shell.Title=Pipeline executor
PipelineExecutorDialog.TransformName.Label=Transform name
PipelineExecutorMeta.CheckResult.NoInputReceived=No input is received from previous transforms
diff --git a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java
index b8c5696c26..e4e58819fa 100644
--- a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java
+++ b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java
@@ -25,10 +25,8 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransform;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.server.HopServerMeta;
-import org.apache.hop.www.HopServerPipelineStatus;
-import org.apache.hop.www.HopServerStatus;
-import org.apache.hop.www.HopServerWorkflowStatus;
-import org.apache.hop.www.RemoteHopServer;
+import org.apache.hop.server.loadbalance.HopServerProbe;
+import org.apache.hop.server.loadbalance.ServerHealthSnapshot;
public class GetServerStatus extends BaseTransform<GetServerStatusMeta, GetServerStatusData> {
public GetServerStatus(
@@ -68,57 +66,23 @@ public class GetServerStatus extends BaseTransform<GetServerStatusMeta, GetServe
if (serverMeta == null) {
throw new HopException("Hop server '" + serverName + "' couldn't be found");
}
- RemoteHopServer server = new RemoteHopServer(serverMeta);
-
- String errorMessage;
- String statusDescription = null;
- Double serverLoad = null;
- Long memoryFree = null;
- Long memoryTotal = null;
- Long cpuCores = null;
- Long cpuProcessTime = null;
- String osName = null;
- String osVersion = null;
- String osArchitecture = null;
- Long activePipelines = null;
- Long activeWorkflows = null;
- Boolean available = null;
- Long responseNs = null;
- long startTime = System.nanoTime();
- long endTime;
-
- try {
- errorMessage = null;
- HopServerStatus status = server.requestServerStatus(this);
- statusDescription = status.getStatusDescription();
- serverLoad = status.getLoadAvg();
- memoryFree = status.getMemoryFree();
- memoryTotal = status.getMemoryTotal();
- cpuCores = (long) status.getCpuCores();
- cpuProcessTime = status.getCpuProcessTime();
- osName = status.getOsName();
- osVersion = status.getOsVersion();
- osArchitecture = status.getOsArchitecture();
- activePipelines = 0L;
- for (HopServerPipelineStatus pipelineStatus : status.getPipelineStatusList()) {
- if (pipelineStatus.isRunning()) {
- activePipelines++;
- }
- }
- activeWorkflows = 0L;
- for (HopServerWorkflowStatus workflowStatus : status.getWorkflowStatusList()) {
- if (workflowStatus.isRunning()) {
- activeWorkflows++;
- }
- }
-
- available = true;
- } catch (Exception e) {
- errorMessage = "Error querying Hop server : " + e.getMessage();
- } finally {
- endTime = System.nanoTime();
- }
- responseNs = endTime - startTime;
+ ServerHealthSnapshot snapshot = HopServerProbe.probe(serverMeta, this, true, 0);
+ String errorMessage = snapshot.getErrorMessage();
+ String statusDescription = snapshot.getStatusDescription();
+ Double serverLoad = snapshot.getLoadAvg();
+ Long memoryFree = snapshot.getMemoryFree();
+ Long memoryTotal = snapshot.getMemoryTotal();
+ Long cpuCores = snapshot.getCpuCores() == null ? null : snapshot.getCpuCores().longValue();
+ Long cpuProcessTime = snapshot.getCpuProcessTime();
+ String osName = snapshot.getOsName();
+ String osVersion = snapshot.getOsVersion();
+ String osArchitecture = snapshot.getOsArchitecture();
+ Long activePipelines = snapshot.isAvailable() ? (long) snapshot.getRunningPipelines() : null;
+ Long finishedPipelines = snapshot.isAvailable() ? (long) snapshot.getFinishedPipelines() : null;
+ Long activeWorkflows = snapshot.isAvailable() ? (long) snapshot.getRunningWorkflows() : null;
+ Long finishedWorkflows = snapshot.isAvailable() ? (long) snapshot.getFinishedWorkflows() : null;
+ Boolean available = snapshot.isAvailable() ? Boolean.TRUE : null;
+ Long responseNs = snapshot.getResponseNs();
// Add the fields to the output row
//
@@ -157,9 +121,15 @@ public class GetServerStatus extends BaseTransform<GetServerStatusMeta, GetServe
if (StringUtils.isNotEmpty(meta.getActivePipelinesField())) {
outputRow[outIndex++] = activePipelines;
}
+ if (StringUtils.isNotEmpty(meta.getFinishedPipelinesField())) {
+ outputRow[outIndex++] = finishedPipelines;
+ }
if (StringUtils.isNotEmpty(meta.getActiveWorkflowsField())) {
outputRow[outIndex++] = activeWorkflows;
}
+ if (StringUtils.isNotEmpty(meta.getFinishedWorkflowsField())) {
+ outputRow[outIndex++] = finishedWorkflows;
+ }
if (StringUtils.isNotEmpty(meta.getAvailableField())) {
outputRow[outIndex++] = available;
}
diff --git a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusDialog.java b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusDialog.java
index eed6a9c49b..245245c0dc 100644
--- a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusDialog.java
+++ b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusDialog.java
@@ -57,7 +57,9 @@ public class GetServerStatusDialog extends BaseTransformDialog {
private Text wOsVersion;
private Text wOsArchitecture;
private Text wActivePipelines;
+ private Text wFinishedPipelines;
private Text wActiveWorkflows;
+ private Text wFinishedWorkflows;
private Text wAvailable;
private Text wResponseNs;
@@ -298,6 +300,24 @@ public class GetServerStatusDialog extends BaseTransformDialog {
wActivePipelines.setLayoutData(fdActivePipelines);
lastControl = wActivePipelines;
+ Label wlFinishedPipelines = new Label(wContent, SWT.RIGHT);
+ wlFinishedPipelines.setText(
+ BaseMessages.getString(PKG, "GetServerStatusDialog.FinishedPipelines"));
+ PropsUi.setLook(wlFinishedPipelines);
+ FormData fdlFinishedPipelines = new FormData();
+ fdlFinishedPipelines.left = new FormAttachment(0, 0);
+ fdlFinishedPipelines.top = new FormAttachment(lastControl, margin);
+ fdlFinishedPipelines.right = new FormAttachment(middle, -margin);
+ wlFinishedPipelines.setLayoutData(fdlFinishedPipelines);
+ wFinishedPipelines = new Text(wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wFinishedPipelines);
+ FormData fdFinishedPipelines = new FormData();
+ fdFinishedPipelines.left = new FormAttachment(middle, 0);
+ fdFinishedPipelines.top = new FormAttachment(wlFinishedPipelines, 0, SWT.CENTER);
+ fdFinishedPipelines.right = new FormAttachment(100, 0);
+ wFinishedPipelines.setLayoutData(fdFinishedPipelines);
+ lastControl = wFinishedPipelines;
+
Label wlActiveWorkflows = new Label(wContent, SWT.RIGHT);
wlActiveWorkflows.setText(BaseMessages.getString(PKG, "GetServerStatusDialog.ActiveWorkflows"));
PropsUi.setLook(wlActiveWorkflows);
@@ -315,6 +335,24 @@ public class GetServerStatusDialog extends BaseTransformDialog {
wActiveWorkflows.setLayoutData(fdActiveWorkflows);
lastControl = wActiveWorkflows;
+ Label wlFinishedWorkflows = new Label(wContent, SWT.RIGHT);
+ wlFinishedWorkflows.setText(
+ BaseMessages.getString(PKG, "GetServerStatusDialog.FinishedWorkflows"));
+ PropsUi.setLook(wlFinishedWorkflows);
+ FormData fdlFinishedWorkflows = new FormData();
+ fdlFinishedWorkflows.left = new FormAttachment(0, 0);
+ fdlFinishedWorkflows.top = new FormAttachment(lastControl, margin);
+ fdlFinishedWorkflows.right = new FormAttachment(middle, -margin);
+ wlFinishedWorkflows.setLayoutData(fdlFinishedWorkflows);
+ wFinishedWorkflows = new Text(wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER);
+ PropsUi.setLook(wFinishedWorkflows);
+ FormData fdFinishedWorkflows = new FormData();
+ fdFinishedWorkflows.left = new FormAttachment(middle, 0);
+ fdFinishedWorkflows.top = new FormAttachment(wlFinishedWorkflows, 0, SWT.CENTER);
+ fdFinishedWorkflows.right = new FormAttachment(100, 0);
+ wFinishedWorkflows.setLayoutData(fdFinishedWorkflows);
+ lastControl = wFinishedWorkflows;
+
Label wlAvailable = new Label(wContent, SWT.RIGHT);
wlAvailable.setText(BaseMessages.getString(PKG, "GetServerStatusDialog.Available"));
PropsUi.setLook(wlAvailable);
@@ -387,7 +425,9 @@ public class GetServerStatusDialog extends BaseTransformDialog {
wOsVersion.setText(Const.NVL(input.getOsVersionField(), ""));
wOsArchitecture.setText(Const.NVL(input.getOsArchitectureField(), ""));
wActivePipelines.setText(Const.NVL(input.getActivePipelinesField(), ""));
+ wFinishedPipelines.setText(Const.NVL(input.getFinishedPipelinesField(), ""));
wActiveWorkflows.setText(Const.NVL(input.getActiveWorkflowsField(), ""));
+ wFinishedWorkflows.setText(Const.NVL(input.getFinishedWorkflowsField(), ""));
wAvailable.setText(Const.NVL(input.getAvailableField(), ""));
wResponseNs.setText(Const.NVL(input.getResponseNsField(), ""));
}
@@ -421,7 +461,9 @@ public class GetServerStatusDialog extends BaseTransformDialog {
in.setOsVersionField(wOsVersion.getText());
in.setOsArchitectureField(wOsArchitecture.getText());
in.setActivePipelinesField(wActivePipelines.getText());
+ in.setFinishedPipelinesField(wFinishedPipelines.getText());
in.setActiveWorkflowsField(wActiveWorkflows.getText());
+ in.setFinishedWorkflowsField(wFinishedWorkflows.getText());
in.setAvailableField(wAvailable.getText());
in.setResponseNsField(wResponseNs.getText());
diff --git a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusMeta.java b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusMeta.java
index 3d075c5aa9..3a4820d6a2 100644
--- a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusMeta.java
+++ b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatusMeta.java
@@ -77,9 +77,15 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
@HopMetadataProperty(key = "active_pipelines_field")
private String activePipelinesField;
+ @HopMetadataProperty(key = "finished_pipelines_field")
+ private String finishedPipelinesField;
+
@HopMetadataProperty(key = "active_workflows_field")
private String activeWorkflowsField;
+ @HopMetadataProperty(key = "finished_workflows_field")
+ private String finishedWorkflowsField;
+
@HopMetadataProperty(key = "available_field")
private String availableField;
@@ -99,7 +105,9 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
osVersionField = "osVersion";
osArchitectureField = "osArchitecture";
activePipelinesField = "activePipelines";
+ finishedPipelinesField = "finishedPipelines";
activeWorkflowsField = "activeWorkflows";
+ finishedWorkflowsField = "finishedWorkflows";
availableField = "available";
responseNsField = "responseNs";
}
@@ -117,7 +125,9 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
this.osVersionField = meta.osVersionField;
this.osArchitectureField = meta.osArchitectureField;
this.activePipelinesField = meta.activePipelinesField;
+ this.finishedPipelinesField = meta.finishedPipelinesField;
this.activeWorkflowsField = meta.activeWorkflowsField;
+ this.finishedWorkflowsField = meta.finishedWorkflowsField;
this.availableField = meta.availableField;
this.responseNsField = meta.responseNsField;
}
@@ -169,9 +179,15 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
if (StringUtils.isNotEmpty(activePipelinesField)) {
inputRowMeta.addValueMeta(new ValueMetaInteger(activePipelinesField));
}
+ if (StringUtils.isNotEmpty(finishedPipelinesField)) {
+ inputRowMeta.addValueMeta(new ValueMetaInteger(finishedPipelinesField));
+ }
if (StringUtils.isNotEmpty(activeWorkflowsField)) {
inputRowMeta.addValueMeta(new ValueMetaInteger(activeWorkflowsField));
}
+ if (StringUtils.isNotEmpty(finishedWorkflowsField)) {
+ inputRowMeta.addValueMeta(new ValueMetaInteger(finishedWorkflowsField));
+ }
if (StringUtils.isNotEmpty(availableField)) {
inputRowMeta.addValueMeta(new ValueMetaBoolean(availableField));
}
@@ -372,6 +388,22 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
this.activePipelinesField = activePipelinesField;
}
+ /**
+ * Gets finishedPipelinesField
+ *
+ * @return value of finishedPipelinesField
+ */
+ public String getFinishedPipelinesField() {
+ return finishedPipelinesField;
+ }
+
+ /**
+ * @param finishedPipelinesField The finishedPipelinesField to set
+ */
+ public void setFinishedPipelinesField(String finishedPipelinesField) {
+ this.finishedPipelinesField = finishedPipelinesField;
+ }
+
/**
* Gets activeWorkflowsField
*
@@ -388,6 +420,22 @@ public class GetServerStatusMeta extends BaseTransformMeta<GetServerStatus, GetS
this.activeWorkflowsField = activeWorkflowsField;
}
+ /**
+ * Gets finishedWorkflowsField
+ *
+ * @return value of finishedWorkflowsField
+ */
+ public String getFinishedWorkflowsField() {
+ return finishedWorkflowsField;
+ }
+
+ /**
+ * @param finishedWorkflowsField The finishedWorkflowsField to set
+ */
+ public void setFinishedWorkflowsField(String finishedWorkflowsField) {
+ this.finishedWorkflowsField = finishedWorkflowsField;
+ }
+
/**
* Gets availableField
*
diff --git a/plugins/transforms/serverstatus/src/main/resources/org/apache/hop/pipeline/transforms/serverstatus/messages/messages_en_US.properties b/plugins/transforms/serverstatus/src/main/resources/org/apache/hop/pipeline/transforms/serverstatus/messages/messages_en_US.properties
index f6fae87a8b..6d55dab6c5 100644
--- a/plugins/transforms/serverstatus/src/main/resources/org/apache/hop/pipeline/transforms/serverstatus/messages/messages_en_US.properties
+++ b/plugins/transforms/serverstatus/src/main/resources/org/apache/hop/pipeline/transforms/serverstatus/messages/messages_en_US.properties
@@ -19,7 +19,9 @@ GetServerStatus.Transform.Description=Get the status of a Hop server
GetServerStatus.Transform.Keywords=server,status,health,monitor,metrics
GetServerStatus.Transform.Name=Get server status
GetServerStatusDialog.ActivePipelines=# Active pipelines field name
+GetServerStatusDialog.FinishedPipelines=# Finished pipelines field name
GetServerStatusDialog.ActiveWorkflows=# Active workflows field name
+GetServerStatusDialog.FinishedWorkflows=# Finished workflows field name
GetServerStatusDialog.Available=Available field name
GetServerStatusDialog.CpuCores=CPU cores field name
GetServerStatusDialog.CpuProcessTime=CPU process time field name
diff --git a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java
index 82f27ceceb..e54e727f72 100644
--- a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java
+++ b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java
@@ -42,6 +42,7 @@ import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.ColumnsResizer;
import org.apache.hop.ui.core.widget.ComboVar;
+import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
@@ -94,8 +95,7 @@ public class WorkflowExecutorDialog extends BaseTransformDialog {
private boolean gotPreviousFields = false;
- protected Label wlRunConfiguration;
- protected ComboVar wRunConfiguration;
+ protected MetaSelectionLine<WorkflowRunConfiguration> wRunConfiguration;
private CTabFolder wTabFolder;
@@ -240,24 +240,20 @@ public class WorkflowExecutorDialog extends BaseTransformDialog {
}
});
- wlRunConfiguration = new Label(shell, SWT.RIGHT);
- wlRunConfiguration.setText(
- BaseMessages.getString(PKG, "WorkflowExecutorDialog.RunConfiguration.Label"));
- PropsUi.setLook(wlRunConfiguration);
- FormData fdlRunConfiguration = new FormData();
- fdlRunConfiguration.left = new FormAttachment(0, 0);
- fdlRunConfiguration.top = new FormAttachment(wWorkflowNameField, margin);
- fdlRunConfiguration.right = new FormAttachment(middle, -margin);
- wlRunConfiguration.setLayoutData(fdlRunConfiguration);
-
- wRunConfiguration = new ComboVar(variables, shell, SWT.LEFT | SWT.BORDER);
- PropsUi.setLook(wlRunConfiguration);
+ wRunConfiguration =
+ new MetaSelectionLine<>(
+ variables,
+ metadataProvider,
+ WorkflowRunConfiguration.class,
+ shell,
+ SWT.SINGLE | SWT.LEFT | SWT.BORDER,
+ BaseMessages.getString(PKG, "WorkflowExecutorDialog.RunConfiguration.Label"),
+ BaseMessages.getString(PKG, "WorkflowExecutorDialog.RunConfiguration.Tooltip"));
FormData fdRunConfiguration = new FormData();
- fdRunConfiguration.left = new FormAttachment(middle, 0);
- fdRunConfiguration.top = new FormAttachment(wlRunConfiguration, 0, SWT.CENTER);
- fdRunConfiguration.right = new FormAttachment(wbBrowse, -margin);
+ fdRunConfiguration.left = new FormAttachment(0, 0);
+ fdRunConfiguration.top = new FormAttachment(wWorkflowNameField, margin);
+ fdRunConfiguration.right = new FormAttachment(100, 0);
wRunConfiguration.setLayoutData(fdRunConfiguration);
- PropsUi.setLook(wRunConfiguration);
//
// Add a tab folder for the parameters and various input and output
diff --git a/plugins/transforms/workflowexecutor/src/main/resources/org/apache/hop/pipeline/transforms/workflowexecutor/messages/messages_en_US.properties b/plugins/transforms/workflowexecutor/src/main/resources/org/apache/hop/pipeline/transforms/workflowexecutor/messages/messages_en_US.properties
index d5d8aac442..dafea20319 100644
--- a/plugins/transforms/workflowexecutor/src/main/resources/org/apache/hop/pipeline/transforms/workflowexecutor/messages/messages_en_US.properties
+++ b/plugins/transforms/workflowexecutor/src/main/resources/org/apache/hop/pipeline/transforms/workflowexecutor/messages/messages_en_US.properties
@@ -70,6 +70,7 @@ WorkflowExecutorDialog.ResultRowsTarget.Label=Target transform for result rows
WorkflowExecutorDialog.RowGroup.Title=Row grouping
WorkflowExecutorDialog.RowGroup.Tooltip=Determine how many rows of data are passed to the workflow before execution
WorkflowExecutorDialog.RunConfiguration.Label=Run configuration
+WorkflowExecutorDialog.RunConfiguration.Tooltip=The workflow run configuration used to execute the specified workflow. Use the buttons to create, edit or open the selected configuration.
WorkflowExecutorDialog.Shell.Title=Workflow executor
WorkflowExecutorDialog.TransformName.Label=Transform name
WorkflowExecutorDialog.Workflow.Label=Workflow
diff --git a/rcp/src/test/java/org/apache/hop/ui/core/gui/GuiCompositeWidgetsGroupTest.java b/rcp/src/test/java/org/apache/hop/ui/core/gui/GuiCompositeWidgetsGroupTest.java
new file mode 100644
index 0000000000..9864165fdb
--- /dev/null
+++ b/rcp/src/test/java/org/apache/hop/ui/core/gui/GuiCompositeWidgetsGroupTest.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.core.gui;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.lang.reflect.Field;
+import java.util.Set;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiRegistry;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.ui.testing.SwtBotTestBase;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Shell;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag("uitest")
+class GuiCompositeWidgetsGroupTest extends SwtBotTestBase {
+
+ private static final String FLAT_PARENT = "GuiCompositeWidgetsGroupTest-flat";
+ private static final String GROUPED_PARENT = "GuiCompositeWidgetsGroupTest-grouped";
+
+ @BeforeAll
+ static void registerSampleWidgets() {
+ register(FlatSample.class);
+ register(GroupedSample.class);
+ }
+
+ @Test
+ void ungroupedWidgetsStayOnAFlatForm() {
+ Shell shell = new Shell(display);
+ shell.setLayout(new FormLayout());
+ try {
+ FlatSample source = new FlatSample();
+ source.setName("alpha");
+ GuiCompositeWidgets widgets = new GuiCompositeWidgets(new Variables());
+ widgets.createCompositeWidgets(source, null, shell, FLAT_PARENT, null);
+ widgets.setWidgetsContents(source, shell, FLAT_PARENT);
+
+ assertEquals(0, countTabFolders(shell));
+ assertNotNull(widgets.getWidgetsMap().get("name"));
+
+ source.setName("beta");
+ widgets.setWidgetsContents(source, shell, FLAT_PARENT);
+ widgets.getWidgetsContents(source, FLAT_PARENT);
+ assertEquals("beta", source.getName());
+ } finally {
+ shell.dispose();
+ }
+ }
+
+ @Test
+ void groupedWidgetsOpenOnTabsAndRoundTripValues() {
+ Shell shell = new Shell(display);
+ shell.setLayout(new FormLayout());
+ try {
+ GroupedSample source = new GroupedSample();
+ source.setFirst("one");
+ source.setSecond("two");
+ GuiCompositeWidgets widgets = new GuiCompositeWidgets(new Variables());
+ widgets.createCompositeWidgets(source, null, shell, GROUPED_PARENT, null);
+ widgets.setWidgetsContents(source, shell, GROUPED_PARENT);
+
+ CTabFolder folder = findTabFolder(shell);
+ assertNotNull(folder);
+ assertEquals(2, folder.getItemCount());
+ assertEquals("First tab", folder.getItem(0).getText());
+ assertEquals("Second tab", folder.getItem(1).getText());
+ assertFalse(folder.getItem(0).getControl() instanceof Label);
+
+ source.setFirst("uno");
+ source.setSecond("dos");
+ widgets.setWidgetsContents(source, shell, GROUPED_PARENT);
+ widgets.getWidgetsContents(source, GROUPED_PARENT);
+ assertEquals("uno", source.getFirst());
+ assertEquals("dos", source.getSecond());
+ } finally {
+ shell.dispose();
+ }
+ }
+
+ @Test
+ void extraGroupBecomesAnotherTab() {
+ Shell shell = new Shell(display);
+ shell.setLayout(new FormLayout());
+ try {
+ GroupedSample source = new GroupedSample();
+ GuiCompositeWidgets widgets = new GuiCompositeWidgets(new Variables());
+ widgets.registerExtraGroup("Extra", "30", null, parent -> new Label(parent, 0).setText("x"));
+ widgets.createCompositeWidgets(source, null, shell, GROUPED_PARENT, null);
+
+ CTabFolder folder = findTabFolder(shell);
+ assertNotNull(folder);
+ assertEquals(3, folder.getItemCount());
+ assertEquals("Extra", folder.getItem(2).getText());
+ } finally {
+ shell.dispose();
+ }
+ }
+
+ @Test
+ void extraGroupOnUngroupedFieldsKeepsThemOnAGeneralTab() {
+ Shell shell = new Shell(display);
+ shell.setLayout(new FormLayout());
+ try {
+ FlatSample source = new FlatSample();
+ source.setName("kept");
+ GuiCompositeWidgets widgets = new GuiCompositeWidgets(new Variables());
+ widgets.registerExtraGroup("Extra", "30", null, parent -> new Label(parent, 0).setText("x"));
+ widgets.createCompositeWidgets(source, null, shell, FLAT_PARENT, null);
+ widgets.setWidgetsContents(source, shell, FLAT_PARENT);
+
+ CTabFolder folder = findTabFolder(shell);
+ assertNotNull(folder);
+ assertEquals(2, folder.getItemCount());
+ assertEquals("General", folder.getItem(0).getText());
+ assertEquals("Extra", folder.getItem(1).getText());
+ assertNotNull(widgets.getWidgetsMap().get("name"));
+
+ source.setName("still-there");
+ widgets.setWidgetsContents(source, shell, FLAT_PARENT);
+ widgets.getWidgetsContents(source, FLAT_PARENT);
+ assertEquals("still-there", source.getName());
+ } finally {
+ shell.dispose();
+ }
+ }
+
+ @Test
+ void hidingAFieldOnOneTabDoesNotCollapseAnotherTab() {
+ Shell shell = new Shell(display);
+ shell.setLayout(new FormLayout());
+ try {
+ GroupedSample source = new GroupedSample();
+ source.setFirst("one");
+ source.setSecond("two");
+ GuiCompositeWidgets widgets = new GuiCompositeWidgets(new Variables());
+ widgets.createCompositeWidgets(source, null, shell, GROUPED_PARENT, null);
+ widgets.setWidgetsContents(source, shell, GROUPED_PARENT);
+
+ Control first = widgets.getWidgetsMap().get("first");
+ Control second = widgets.getWidgetsMap().get("second");
+ assertNotNull(first);
+ assertNotNull(second);
+ assertNotEquals(first.getParent(), second.getParent());
+
+ widgets.setWidgetsHidden(source, Set.of("first"));
+
+ assertFalse(first.getVisible());
+ assertTrue(second.getVisible());
+ assertTrue(second.getLayoutData() instanceof FormData);
+ FormData secondData = (FormData) second.getLayoutData();
+ assertTrue(secondData.height == -1 || secondData.height > 0);
+
+ source.setSecond("dos");
+ widgets.setWidgetsContents(source, shell, GROUPED_PARENT);
+ widgets.getWidgetsContents(source, GROUPED_PARENT);
+ assertEquals("dos", source.getSecond());
+ } finally {
+ shell.dispose();
+ }
+ }
+
+ private static void register(Class<?> type) {
+ GuiRegistry registry = GuiRegistry.getInstance();
+ String parentId = null;
+ for (Field field : type.getDeclaredFields()) {
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ if (element != null) {
+ parentId = element.parentId();
+ break;
+ }
+ }
+ if (parentId == null || registry.findGuiElements(type.getName(), parentId) != null) {
+ return;
+ }
+ for (Field field : type.getDeclaredFields()) {
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ if (element != null) {
+ registry.addGuiWidgetElement(type.getName(), element, field);
+ }
+ }
+ }
+
+ private static CTabFolder findTabFolder(Composite parent) {
+ for (Control child : parent.getChildren()) {
+ if (child instanceof CTabFolder folder) {
+ return folder;
+ }
+ }
+ return null;
+ }
+
+ private static int countTabFolders(Composite parent) {
+ int count = 0;
+ for (Control child : parent.getChildren()) {
+ if (child instanceof CTabFolder) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ @GuiPlugin
+ @Getter
+ @Setter
+ public static class FlatSample {
+ @GuiWidgetElement(
+ id = "name",
+ parentId = FLAT_PARENT,
+ type = GuiElementType.TEXT,
+ label = "Name")
+ private String name;
+ }
+
+ @GuiPlugin
+ @Getter
+ @Setter
+ public static class GroupedSample {
+ @GuiWidgetElement(
+ id = "first",
+ parentId = GROUPED_PARENT,
+ type = GuiElementType.TEXT,
+ label = "First",
+ group = "First tab",
+ groupOrder = "10",
+ groupType = GuiWidgetGroupType.TABS)
+ private String first;
+
+ @GuiWidgetElement(
+ id = "second",
+ parentId = GROUPED_PARENT,
+ type = GuiElementType.TEXT,
+ label = "Second",
+ group = "Second tab",
+ groupOrder = "20",
+ groupType = GuiWidgetGroupType.TABS)
+ private String second;
+ }
+}
diff --git a/ui/src/main/java/org/apache/hop/ui/core/gui/GuiCompositeWidgets.java b/ui/src/main/java/org/apache/hop/ui/core/gui/GuiCompositeWidgets.java
index bccb0d1baf..422c6ce04f 100644
--- a/ui/src/main/java/org/apache/hop/ui/core/gui/GuiCompositeWidgets.java
+++ b/ui/src/main/java/org/apache/hop/ui/core/gui/GuiCompositeWidgets.java
@@ -21,11 +21,14 @@ import java.beans.PropertyDescriptor;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Consumer;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@@ -36,6 +39,8 @@ import org.apache.hop.core.exception.HopRuntimeException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiElements;
import org.apache.hop.core.gui.plugin.GuiRegistry;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroups;
import org.apache.hop.core.gui.plugin.ITypeFilename;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.LogChannel;
@@ -43,6 +48,7 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.ui.core.ConstUi;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
import org.apache.hop.ui.core.widget.ComboVar;
@@ -50,9 +56,14 @@ import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.core.widget.PasswordTextVar;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.util.SwtSvgImageUtil;
import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
import org.eclipse.swt.custom.ScrolledComposite;
+import org.eclipse.swt.graphics.Image;
import org.eclipse.swt.graphics.Rectangle;
+import org.eclipse.swt.layout.FillLayout;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
@@ -69,6 +80,7 @@ import org.eclipse.swt.widgets.Text;
public class GuiCompositeWidgets {
public static final String CONST_PARENT_ID = ", parent ID: ";
public static final String NONE_DATABASE_META = "NoneDatabaseMeta";
+ private static final Class<?> PKG = GuiCompositeWidgets.class;
@Setter @Getter private IVariables variables;
@@ -88,6 +100,8 @@ public class GuiCompositeWidgets {
@Getter @Setter private IGuiPluginCompositeButtonsListener compositeButtonsListener;
+ private final List<ExtraGroup> extraGroups = new ArrayList<>();
+
/** Parent composite of the last {@link #createCompositeWidgets} call (for button refresh). */
private Composite widgetsParentComposite;
@@ -166,7 +180,7 @@ public class GuiCompositeWidgets {
//
boolean useNewLayout = isConfigPlugin(sourceData.getClass());
this.widgetsUseNewLayout = useNewLayout;
- addCompositeWidgets(sourceData, parent, guiElements, lastControl, useNewLayout);
+ layoutElements(sourceData, parent, guiElements, lastControl, useNewLayout);
if (compositeWidgetsListener != null) {
compositeWidgetsListener.widgetsCreated(this);
@@ -177,6 +191,183 @@ public class GuiCompositeWidgets {
parent.layout(true, true);
}
+ /**
+ * Add a group that is not driven by {@link org.apache.hop.core.gui.plugin.GuiWidgetElement}
+ * fields, for example a {@code TableView}. It is shown with the annotated groups when {@link
+ * GuiWidgetGroupType#TABS} (or LIST / BOXES) is used.
+ */
+ public void registerExtraGroup(
+ String group, String groupOrder, Image image, Consumer<Composite> contents) {
+ extraGroups.add(new ExtraGroup(group, groupOrder, image, contents));
+ }
+
+ private void layoutElements(
+ Object sourceData,
+ Composite parent,
+ GuiElements guiElements,
+ Control lastControl,
+ boolean useNewLayout) {
+ List<WidgetGroup> groups = collectGroups(guiElements);
+ if (groups.isEmpty()) {
+ addCompositeWidgets(sourceData, parent, guiElements, lastControl, useNewLayout);
+ return;
+ }
+
+ if (GuiWidgetGroups.hasMixedTypes(guiElements.getChildren())) {
+ LogChannel.UI.logError(
+ "Mixed widget group types on parent "
+ + widgetsParentGuiElementId
+ + "; falling back to tabs");
+ }
+ GuiWidgetGroupType type = GuiWidgetGroups.typeOf(guiElements.getChildren());
+ if (type != GuiWidgetGroupType.TABS && type != GuiWidgetGroupType.NONE) {
+ LogChannel.UI.logBasic(
+ "Widget group type "
+ + type
+ + " is not implemented yet; showing tabs for parent "
+ + widgetsParentGuiElementId);
+ }
+ layoutTabs(sourceData, parent, groups, useNewLayout);
+ }
+
+ private List<WidgetGroup> collectGroups(GuiElements guiElements) {
+ if (!GuiWidgetGroups.hasGroups(guiElements.getChildren()) && extraGroups.isEmpty()) {
+ return List.of();
+ }
+
+ // Extra groups force a container; ungrouped fields land on the General tab.
+ String defaultLabel = BaseMessages.getString(PKG, "GuiCompositeWidgets.Group.General");
+ Map<String, WidgetGroup> byKey = new LinkedHashMap<>();
+ for (GuiWidgetGroups.Bucket bucket :
+ GuiWidgetGroups.from(guiElements.getChildren(), defaultLabel)) {
+ WidgetGroup group = new WidgetGroup(bucket.getLabel(), bucket.getOrder(), bucket.getImage());
+ group.elements.addAll(bucket.getElements());
+ byKey.put(bucket.getKey(), group);
+ }
+
+ for (ExtraGroup extra : extraGroups) {
+ WidgetGroup group =
+ byKey.computeIfAbsent(
+ extra.group, k -> new WidgetGroup(extra.group, extra.groupOrder, null));
+ group.extraImage = extra.image;
+ group.extras.add(extra.contents);
+ if (StringUtils.isEmpty(group.order) && StringUtils.isNotEmpty(extra.groupOrder)) {
+ group.order = extra.groupOrder;
+ }
+ }
+
+ List<WidgetGroup> groups = new ArrayList<>(byKey.values());
+ groups.sort(
+ Comparator.comparing((WidgetGroup g) -> Const.NVL(g.order, ""))
+ .thenComparing(g -> Const.NVL(g.label, "")));
+ return groups;
+ }
+
+ private void layoutTabs(
+ Object sourceData, Composite parent, List<WidgetGroup> groups, boolean useNewLayout) {
+ CTabFolder folder = new CTabFolder(parent, SWT.BORDER);
+ PropsUi.setLook(folder);
+ FormData fdFolder = new FormData();
+ fdFolder.left = new FormAttachment(0, 0);
+ fdFolder.top =
+ widgetsFirstLastControl == null
+ ? new FormAttachment(0, 0)
+ : new FormAttachment(widgetsFirstLastControl, PropsUi.getMargin());
+ fdFolder.right = new FormAttachment(100, 0);
+ fdFolder.bottom = new FormAttachment(100, 0);
+ folder.setLayoutData(fdFolder);
+
+ for (WidgetGroup group : groups) {
+ CTabItem item = new CTabItem(folder, SWT.NONE);
+ item.setFont(GuiResource.getInstance().getFontDefault());
+ item.setText(Const.NVL(group.label, ""));
+ Image image = group.extraImage;
+ if (image == null) {
+ image = loadGroupImage(parent, group.image);
+ }
+ if (image != null) {
+ item.setImage(image);
+ }
+
+ ScrolledComposite scrolled = new ScrolledComposite(folder, SWT.V_SCROLL | SWT.H_SCROLL);
+ scrolled.setLayout(new FillLayout());
+ Composite composite = new Composite(scrolled, SWT.NONE);
+ PropsUi.setLook(composite);
+ FormLayout layout = new FormLayout();
+ layout.marginWidth = PropsUi.getFormMargin();
+ layout.marginHeight = PropsUi.getFormMargin();
+ composite.setLayout(layout);
+ scrolled.setContent(composite);
+ scrolled.setExpandHorizontal(true);
+ scrolled.setExpandVertical(true);
+ item.setControl(scrolled);
+
+ Control last = null;
+ for (GuiElements child : group.elements) {
+ last = addCompositeWidgets(sourceData, composite, child, last, useNewLayout);
+ }
+ for (Consumer<Composite> extra : group.extras) {
+ extra.accept(composite);
+ }
+ composite.pack();
+ Rectangle bounds = composite.getBounds();
+ scrolled.setMinWidth(bounds.width);
+ scrolled.setMinHeight(bounds.height);
+ }
+
+ if (folder.getItemCount() > 0) {
+ folder.setSelection(0);
+ }
+ }
+
+ private Image loadGroupImage(Composite parent, String filename) {
+ if (StringUtils.isEmpty(filename)) {
+ return null;
+ }
+ try {
+ int size = (int) (ConstUi.SMALL_ICON_SIZE * PropsUi.getInstance().getZoomFactor());
+ Image image =
+ SwtSvgImageUtil.getImage(
+ parent.getDisplay(), getClass().getClassLoader(), filename, size, size);
+ if (image != null) {
+ parent.addListener(SWT.Dispose, e -> image.dispose());
+ }
+ return image;
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error loading widget group image " + filename, e);
+ return null;
+ }
+ }
+
+ private static final class ExtraGroup {
+ private final String group;
+ private final String groupOrder;
+ private final Image image;
+ private final Consumer<Composite> contents;
+
+ private ExtraGroup(String group, String groupOrder, Image image, Consumer<Composite> contents) {
+ this.group = group;
+ this.groupOrder = groupOrder;
+ this.image = image;
+ this.contents = contents;
+ }
+ }
+
+ private static final class WidgetGroup {
+ private final String label;
+ private String order;
+ private String image;
+ private Image extraImage;
+ private final List<GuiElements> elements = new ArrayList<>();
+ private final List<Consumer<Composite>> extras = new ArrayList<>();
+
+ private WidgetGroup(String label, String order, String image) {
+ this.label = label;
+ this.order = order;
+ this.image = image;
+ }
+ }
+
/**
* Check if a class has the @ConfigPlugin annotation
*
@@ -202,6 +393,9 @@ public class GuiCompositeWidgets {
* <p>Call this from {@link IGuiPluginCompositeWidgetsListener} when a plugin has options that
* only apply to some of its settings, to keep the ones that cannot do anything out of the way.
*
+ * <p>When widgets sit on different group containers (tabs), each container is re-hung on its own.
+ * Hiding a field on one tab must not collapse or re-attach rows on another tab.
+ *
* @param sourceData the object the widgets were created for
* @param hiddenIds ids of the GUI elements to hide; every other element is made visible again
*/
@@ -221,42 +415,53 @@ public class GuiCompositeWidgets {
List<GuiElements> children = new ArrayList<>(root.getChildren());
Collections.sort(children);
- Control lastVisible = widgetsFirstLastControl;
+ // The registry appends without checking for duplicates, so an element can be in here more
+ // than once if the GUI plugins were scanned twice. There is only ever one widget per id, and
+ // hanging it below itself on a second pass would make the layout circular.
+ Map<Composite, List<GuiElements>> byParent = new LinkedHashMap<>();
Set<String> handled = new HashSet<>();
for (GuiElements element : children) {
- if (element.isIgnored() || element.getId() == null) {
+ if (element.isIgnored() || element.getId() == null || !handled.add(element.getId())) {
continue;
}
- // The registry appends without checking for duplicates, so an element can be in here more
- // than once if the GUI plugins were scanned twice. There is only ever one widget per id, and
- // hanging it below itself on a second pass would make the layout circular.
- //
- if (!handled.add(element.getId())) {
+ Control widget = widgetsMap.get(element.getId());
+ if (widget == null || widget.isDisposed()) {
continue;
}
- Control label = labelsMap.get(element.getId());
- Control widget = widgetsMap.get(element.getId());
- Control action = actionWidgetsMap.get(element.getId());
- boolean hidden = hiddenIds.contains(element.getId());
+ byParent.computeIfAbsent(widget.getParent(), parent -> new ArrayList<>()).add(element);
+ }
- setControlVisible(label, !hidden);
- setControlVisible(widget, !hidden);
- setControlVisible(action, !hidden);
+ for (Map.Entry<Composite, List<GuiElements>> entry : byParent.entrySet()) {
+ Composite parent = entry.getKey();
+ Control lastVisible = parent == widgetsParentComposite ? widgetsFirstLastControl : null;
+ for (GuiElements element : entry.getValue()) {
+ Control label = labelsMap.get(element.getId());
+ Control widget = widgetsMap.get(element.getId());
+ Control action = actionWidgetsMap.get(element.getId());
+ boolean hidden = hiddenIds.contains(element.getId());
- if (hidden) {
- collapseRow(lastVisible, label, widget, action);
- continue;
- }
+ setControlVisible(label, !hidden);
+ setControlVisible(widget, !hidden);
+ setControlVisible(action, !hidden);
- restoreRowHeight(label);
- restoreRowHeight(widget);
- restoreRowHeight(action);
+ if (hidden) {
+ collapseRow(lastVisible, label, widget, action);
+ continue;
+ }
- if (widget == null || widget.isDisposed()) {
- continue;
+ restoreRowHeight(label);
+ restoreRowHeight(widget);
+ restoreRowHeight(action);
+
+ if (widget == null || widget.isDisposed()) {
+ continue;
+ }
+ reattachRow(element, label, widget, action, lastVisible);
+ lastVisible = widget;
+ }
+ if (parent != null && !parent.isDisposed()) {
+ parent.layout(true, true);
}
- reattachRow(element, label, widget, action, lastVisible);
- lastVisible = widget;
}
widgetsParentComposite.layout(true, true);
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/PaletteEngineFilter.java b/ui/src/main/java/org/apache/hop/ui/hopgui/PaletteEngineFilter.java
index 6362e78f6f..c5a77b066a 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/PaletteEngineFilter.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/PaletteEngineFilter.java
@@ -46,11 +46,12 @@ import org.apache.hop.workflow.engine.WorkflowEnginePluginType;
* <p>An empty stored value means "no filter" — the palette behaves exactly as it did before this
* feature was introduced, preserving the new-user experience.
*
- * <p>The Hop engines — local "Local", remote "Remote" and the local single-threaded "LocalSingle" —
- * are grouped under a single {@link #HOP_ENGINE_GROUP_ID Hop} entry: from a user's perspective they
- * are the same engine surface (same plugin compatibility, same transform palette) and splitting
- * them in the combo adds noise. A plugin is shown when <em>any</em> engine in the group accepts it,
- * hidden only when <em>every</em> member of the group says UNSUPPORTED.
+ * <p>The Hop engines — local "Local", remote "Remote", load-balancing "LoadBalancing" and the local
+ * single-threaded "LocalSingle" — are grouped under a single {@link #HOP_ENGINE_GROUP_ID Hop}
+ * entry: from a user's perspective they are the same engine surface (same plugin compatibility,
+ * same transform palette) and splitting them in the combo adds noise. A plugin is shown when
+ * <em>any</em> engine in the group accepts it, hidden only when <em>every</em> member of the group
+ * says UNSUPPORTED.
*/
public final class PaletteEngineFilter {
@@ -74,14 +75,15 @@ public final class PaletteEngineFilter {
/**
* Pipeline engine plugin ids collapsed into the {@link #HOP_ENGINE_GROUP_ID Hop} group. The local
- * single-threaded engine ("LocalSingle") supports the same transforms as the local and remote
- * engines, so it shares their combo entry instead of getting one of its own.
+ * single-threaded and load-balancing engines support the same transforms as the local and remote
+ * engines, so they share their combo entry instead of getting one of their own.
*/
private static final List<String> HOP_PIPELINE_ENGINE_IDS =
- List.of("Local", "Remote", "LocalSingle");
+ List.of("Local", "Remote", "LoadBalancing", "LocalSingle");
/** Workflow counterpart — there is no single-threaded workflow engine. */
- private static final List<String> HOP_WORKFLOW_ENGINE_IDS = List.of("Local", "Remote");
+ private static final List<String> HOP_WORKFLOW_ENGINE_IDS =
+ List.of("Local", "Remote", "LoadBalancing");
private final String engineId;
diff --git a/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java b/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
index 815c91c69e..018a56ba4f 100644
--- a/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
@@ -36,6 +36,7 @@ import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.pipeline.engine.PipelineEnginePluginType;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
@@ -48,6 +49,7 @@ import org.apache.hop.ui.core.widget.ComboVar;
import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.server.loadbalance.LoadBalancingRunConfigurationWidgets;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CTabFolder;
import org.eclipse.swt.custom.CTabItem;
@@ -87,6 +89,7 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
private Composite wPluginSpecificComp;
private ScrolledComposite wsPluginSpecificComp;
private GuiCompositeWidgets guiCompositeWidgets;
+ private LoadBalancingRunConfigurationWidgets loadBalancingWidgets;
private Map<String, IPipelineEngineRunConfiguration> metaMap;
private TableView wVariables;
@@ -437,9 +440,18 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
for (Control child : wPluginSpecificComp.getChildren()) {
child.dispose();
}
+ guiCompositeWidgets = null;
+ loadBalancingWidgets = null;
if (workingConfiguration.getEngineRunConfiguration() != null) {
guiCompositeWidgets = new GuiCompositeWidgets(manager.getVariables());
+ if (workingConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration) {
+ loadBalancingWidgets =
+ new LoadBalancingRunConfigurationWidgets(
+ manager.getVariables(), manager.getMetadataProvider());
+ loadBalancingWidgets.registerServersGroup(guiCompositeWidgets, e -> setChanged());
+ }
guiCompositeWidgets.createCompositeWidgets(
workingConfiguration.getEngineRunConfiguration(),
null,
@@ -545,6 +557,9 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
try {
wExecutionInfoLocation.fillItems();
wProfile.fillItems();
+ if (loadBalancingWidgets != null) {
+ loadBalancingWidgets.refreshServerNames();
+ }
} catch (Exception e) {
LogChannel.UI.logError("Error refreshing execution metadata lists", e);
}
@@ -581,6 +596,11 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
wPluginSpecificComp,
PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID);
}
+ if (loadBalancingWidgets != null
+ && workingConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration loadBalancing) {
+ loadBalancingWidgets.setServers(loadBalancing);
+ }
} else {
if (!wPluginType.getText().isEmpty()) {
wPluginType.setText("");
@@ -623,6 +643,11 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
guiCompositeWidgets.getWidgetsContents(
meta.getEngineRunConfiguration(), PipelineRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID);
}
+ if (loadBalancingWidgets != null
+ && meta.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration loadBalancing) {
+ loadBalancingWidgets.getServers(loadBalancing);
+ }
meta.setExecutionDataProfileName(wProfile.getText());
diff --git a/ui/src/main/java/org/apache/hop/ui/server/loadbalance/LoadBalancingRunConfigurationWidgets.java b/ui/src/main/java/org/apache/hop/ui/server/loadbalance/LoadBalancingRunConfigurationWidgets.java
new file mode 100644
index 0000000000..15bc5ee5ec
--- /dev/null
+++ b/ui/src/main/java/org/apache/hop/ui/server/loadbalance/LoadBalancingRunConfigurationWidgets.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.server.loadbalance;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.server.HopServerMeta;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
+import org.apache.hop.server.loadbalance.LoadBalancingServerEntry;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.GuiResource;
+import org.apache.hop.ui.core.widget.ColumnInfo;
+import org.apache.hop.ui.core.widget.TableView;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.events.ModifyListener;
+import org.eclipse.swt.layout.FormAttachment;
+import org.eclipse.swt.layout.FormData;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.TableItem;
+
+/** Hop Servers table for a load-balancing run configuration (shown as an extra widget group). */
+public class LoadBalancingRunConfigurationWidgets {
+
+ private static final Class<?> PKG = LoadBalancingRunConfigurationWidgets.class;
+
+ private final IVariables variables;
+ private final IHopMetadataProvider metadataProvider;
+ private TableView wServers;
+ private ColumnInfo serverColumn;
+
+ public LoadBalancingRunConfigurationWidgets(
+ IVariables variables, IHopMetadataProvider metadataProvider) {
+ this.variables = variables;
+ this.metadataProvider = metadataProvider;
+ }
+
+ public void registerServersGroup(GuiCompositeWidgets widgets, ModifyListener modifyListener) {
+ widgets.registerExtraGroup(
+ BaseMessages.getString(PKG, "LoadBalancingRunConfiguration.Tab.HopServers"),
+ "30",
+ GuiResource.getInstance().getImageServer(),
+ parent -> addServersTable(parent, modifyListener));
+ }
+
+ public void setServers(ILoadBalancingRunConfiguration config) {
+ if (wServers == null || wServers.isDisposed() || config == null) {
+ return;
+ }
+ refreshServerNames();
+ wServers.clearAll();
+ List<LoadBalancingServerEntry> servers = config.getServers();
+ if (servers != null) {
+ for (LoadBalancingServerEntry entry : servers) {
+ TableItem item = new TableItem(wServers.table, SWT.NONE);
+ item.setText(1, Const.NVL(entry.getHopServerName(), ""));
+ item.setText(2, entry.isEnabled() ? "Y" : "N");
+ item.setText(3, Const.NVL(entry.getMaxConcurrent(), ""));
+ }
+ }
+ if (wServers.table.getItemCount() == 0) {
+ new TableItem(wServers.table, SWT.NONE);
+ }
+ wServers.removeEmptyRows();
+ wServers.setRowNums();
+ wServers.optWidth(true);
+ }
+
+ public void getServers(ILoadBalancingRunConfiguration config) {
+ if (wServers == null || wServers.isDisposed() || config == null) {
+ return;
+ }
+ List<LoadBalancingServerEntry> servers = new ArrayList<>();
+ for (int i = 0; i < wServers.nrNonEmpty(); i++) {
+ TableItem item = wServers.getNonEmpty(i);
+ String name = item.getText(1);
+ if (name.isEmpty()) {
+ continue;
+ }
+ LoadBalancingServerEntry entry = new LoadBalancingServerEntry();
+ entry.setHopServerName(name);
+ entry.setEnabled(!"N".equalsIgnoreCase(item.getText(2)));
+ entry.setMaxConcurrent(item.getText(3));
+ servers.add(entry);
+ }
+ config.setServers(servers);
+ }
+
+ public void refreshServerNames() {
+ if (serverColumn == null) {
+ return;
+ }
+ serverColumn.setComboValues(loadServerNames());
+ }
+
+ private void addServersTable(Composite parent, ModifyListener modifyListener) {
+ PropsUi props = PropsUi.getInstance();
+ int margin = PropsUi.getMargin();
+
+ serverColumn =
+ new ColumnInfo(
+ BaseMessages.getString(PKG, "LoadBalancingRunConfiguration.Servers.Column.Server"),
+ ColumnInfo.COLUMN_TYPE_CCOMBO,
+ loadServerNames(),
+ false);
+ serverColumn.setUsingVariables(true);
+
+ ColumnInfo enabledColumn =
+ new ColumnInfo(
+ BaseMessages.getString(PKG, "LoadBalancingRunConfiguration.Servers.Column.Enabled"),
+ ColumnInfo.COLUMN_TYPE_CCOMBO,
+ new String[] {"Y", "N"},
+ true);
+
+ ColumnInfo maxColumn =
+ new ColumnInfo(
+ BaseMessages.getString(
+ PKG, "LoadBalancingRunConfiguration.Servers.Column.MaxConcurrent"),
+ ColumnInfo.COLUMN_TYPE_TEXT,
+ false);
+ maxColumn.setUsingVariables(true);
+ maxColumn.setToolTip(
+ BaseMessages.getString(
+ PKG, "LoadBalancingRunConfiguration.Servers.Column.MaxConcurrent.ToolTip"));
+
+ wServers =
+ new TableView(
+ variables,
+ parent,
+ SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
+ new ColumnInfo[] {serverColumn, enabledColumn, maxColumn},
+ 1,
+ modifyListener,
+ props);
+ PropsUi.setLook(wServers);
+ FormData fdServers = new FormData();
+ fdServers.left = new FormAttachment(0, margin);
+ fdServers.top = new FormAttachment(0, margin);
+ fdServers.right = new FormAttachment(100, -margin);
+ fdServers.bottom = new FormAttachment(100, -margin);
+ wServers.setLayoutData(fdServers);
+ }
+
+ private String[] loadServerNames() {
+ try {
+ List<String> names = metadataProvider.getSerializer(HopServerMeta.class).listObjectNames();
+ return names.toArray(new String[0]);
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error listing Hop servers for the load-balancing table", e);
+ return new String[0];
+ }
+ }
+}
diff --git a/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java b/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
index 1b765ef2bb..c8699fd00a 100644
--- a/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
@@ -30,6 +30,7 @@ import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.execution.ExecutionInfoLocation;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.server.loadbalance.ILoadBalancingRunConfiguration;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
@@ -39,6 +40,7 @@ import org.apache.hop.ui.core.metadata.MetadataManager;
import org.apache.hop.ui.core.widget.ComboVar;
import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.server.loadbalance.LoadBalancingRunConfigurationWidgets;
import org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration;
import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.apache.hop.workflow.engine.IWorkflowEngine;
@@ -75,6 +77,7 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
private Composite wPluginSpecificComp;
private GuiCompositeWidgets guiCompositeWidgets;
+ private LoadBalancingRunConfigurationWidgets loadBalancingWidgets;
private Map<String, IWorkflowEngineRunConfiguration> metaMap;
@@ -275,11 +278,20 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
for (Control child : wPluginSpecificComp.getChildren()) {
child.dispose();
}
+ guiCompositeWidgets = null;
+ loadBalancingWidgets = null;
// Now add the run configuration plugin specific widgets
//
if (workingConfiguration.getEngineRunConfiguration() != null) {
guiCompositeWidgets = new GuiCompositeWidgets(manager.getVariables());
+ if (workingConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration) {
+ loadBalancingWidgets =
+ new LoadBalancingRunConfigurationWidgets(
+ manager.getVariables(), manager.getMetadataProvider());
+ loadBalancingWidgets.registerServersGroup(guiCompositeWidgets, e -> setChanged());
+ }
guiCompositeWidgets.createCompositeWidgets(
workingConfiguration.getEngineRunConfiguration(),
null,
@@ -348,6 +360,9 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
public void refreshOnDialogActivate() {
try {
wExecutionInfoLocation.fillItems();
+ if (loadBalancingWidgets != null) {
+ loadBalancingWidgets.refreshServerNames();
+ }
} catch (Exception e) {
LogChannel.UI.logError("Error refreshing execution information locations", e);
}
@@ -370,10 +385,17 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
if (workingConfiguration.getEngineRunConfiguration() != null) {
wPluginType.setText(
Const.NVL(workingConfiguration.getEngineRunConfiguration().getEnginePluginName(), ""));
- guiCompositeWidgets.setWidgetsContents(
- workingConfiguration.getEngineRunConfiguration(),
- wPluginSpecificComp,
- WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ if (guiCompositeWidgets != null) {
+ guiCompositeWidgets.setWidgetsContents(
+ workingConfiguration.getEngineRunConfiguration(),
+ wPluginSpecificComp,
+ WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID);
+ }
+ if (loadBalancingWidgets != null
+ && workingConfiguration.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration loadBalancing) {
+ loadBalancingWidgets.setServers(loadBalancing);
+ }
} else {
wPluginType.setText("");
}
@@ -395,6 +417,11 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
guiCompositeWidgets.getWidgetsContents(
meta.getEngineRunConfiguration(), WorkflowRunConfiguration.GUI_PLUGIN_ELEMENT_PARENT_ID);
}
+ if (loadBalancingWidgets != null
+ && meta.getEngineRunConfiguration()
+ instanceof ILoadBalancingRunConfiguration loadBalancing) {
+ loadBalancingWidgets.getServers(loadBalancing);
+ }
}
private void changeWorkingEngineConfiguration(WorkflowRunConfiguration meta) {
diff --git a/ui/src/main/resources/org/apache/hop/ui/core/gui/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/core/gui/messages/messages_en_US.properties
new file mode 100644
index 0000000000..f2c80d1546
--- /dev/null
+++ b/ui/src/main/resources/org/apache/hop/ui/core/gui/messages/messages_en_US.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+GuiCompositeWidgets.Group.General=General
diff --git a/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
index f6177a0635..72c16f02b1 100644
--- a/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
+++ b/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
@@ -55,3 +55,17 @@ PipelineRunConfigurationDialog.MitigateBufferDeadlocks.Label=Mitigate buffer dea
PipelineRunConfigurationDialog.MitigateBufferDeadlocks.ToolTip=When risks are detected, use spilling rowsets only on the recommended hops so producers do not block when a buffer fills. Enabled by default so pipelines make progress instead of hanging; excess rows spill to disk (I/O cost). Disable if you prefer strict bounded memory only.
PipelineRunConfigurationDialog.BufferDeadlockSpillDirectory.Label=Buffer deadlock spill directory
PipelineRunConfigurationDialog.BufferDeadlockSpillDirectory.ToolTip=Directory for spilled rowset temp files. Leave empty to use the system temporary directory.
+PipelineRunConfigurationDialog.LoadBalancingAlgorithm.Label=Distribution algorithm
+PipelineRunConfigurationDialog.LoadBalancingAlgorithm.ToolTip=Even load spreads work across servers. Pack keeps as few servers busy as possible so idle ones can be disabled and turned off.
+PipelineRunConfigurationDialog.LoadBalancingMaxRetries.Label=Maximum retries
+PipelineRunConfigurationDialog.LoadBalancingMaxRetries.ToolTip=How many extra times to try another server after a submit or capacity failure. Use '${MAX_RETRIES}' if you want this in a variable.
+PipelineRunConfigurationDialog.LoadBalancingRetryWindow.Label=Retry window (ms)
+PipelineRunConfigurationDialog.LoadBalancingRetryWindow.ToolTip=Give up retrying after this many milliseconds from the first attempt. 0 means retry count only. Use '${RETRY_WINDOW_MS}' if you want this in a variable.
+PipelineRunConfigurationDialog.LoadBalancingRetryOnFailure.Label=Retry failed executions
+PipelineRunConfigurationDialog.LoadBalancingRetryOnFailure.ToolTip=When enabled, also retry if the remote pipeline runs and fails. Off by default because a failed run may already have written data.
+PipelineRunConfigurationDialog.LoadBalancingProbeTimeout.Label=Server probe timeout (ms)
+PipelineRunConfigurationDialog.LoadBalancingProbeTimeout.ToolTip=How long to wait for each Hop server status check. Default 3000.
+PipelineRunConfigurationDialog.LoadBalancingConfigRefresh.Label=Configuration refresh interval (ms)
+PipelineRunConfigurationDialog.LoadBalancingConfigRefresh.ToolTip=How often to re-read this run configuration from metadata so enable/disable and max concurrent changes take effect. Default 10000.
+PipelineRunConfigurationDialog.LoadBalancingStateFolder.Label=Assignment state folder
+PipelineRunConfigurationDialog.LoadBalancingStateFolder.ToolTip=Optional VFS folder for assignment ledger files, for example '${PROJECT_HOME}/loadbalance'. Leave empty to skip files.
diff --git a/ui/src/main/resources/org/apache/hop/ui/server/loadbalance/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/server/loadbalance/messages/messages_en_US.properties
new file mode 100644
index 0000000000..c13781cab3
--- /dev/null
+++ b/ui/src/main/resources/org/apache/hop/ui/server/loadbalance/messages/messages_en_US.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+LoadBalancingRunConfiguration.Tab.Management=Management
+LoadBalancingRunConfiguration.Tab.LoadBalancing=Load-balancing
+LoadBalancingRunConfiguration.Tab.HopServers=Hop Servers
+LoadBalancingRunConfiguration.Servers.Label=Hop servers
+LoadBalancingRunConfiguration.Servers.Column.Server=Hop server
+LoadBalancingRunConfiguration.Servers.Column.Enabled=Enabled
+LoadBalancingRunConfiguration.Servers.Column.MaxConcurrent=Max concurrent
+LoadBalancingRunConfiguration.Servers.Column.MaxConcurrent.ToolTip=Maximum number of unfinished pipelines and workflows on this server. Variables like '${MAX_SLOTS}' are allowed.
diff --git a/ui/src/main/resources/org/apache/hop/ui/workflow/config/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/workflow/config/messages/messages_en_US.properties
index a51ed8b571..02fbfbc427 100644
--- a/ui/src/main/resources/org/apache/hop/ui/workflow/config/messages/messages_en_US.properties
+++ b/ui/src/main/resources/org/apache/hop/ui/workflow/config/messages/messages_en_US.properties
@@ -37,3 +37,17 @@ WorkflowRunConfigurationDialog.Transactional.Label=Make this workflow transactio
WorkflowRunConfigurationDialog.Transactional.ToolTip=If you enable this there will always be just one connection used per database. At the end of the workflow there will be a commit or rollback (in case of error) on all named connections at the same time.
WorkflowRunConfigurationDialog.WaitUntilFinished.Label=Wait until finished
WorkflowRunConfigurationDialog.WaitUntilFinished.ToolTip=Wait until a remotely executed workflow finishes
+WorkflowRunConfigurationDialog.LoadBalancingAlgorithm.Label=Distribution algorithm
+WorkflowRunConfigurationDialog.LoadBalancingAlgorithm.ToolTip=Even load spreads work across servers. Pack keeps as few servers busy as possible so idle ones can be disabled and turned off.
+WorkflowRunConfigurationDialog.LoadBalancingMaxRetries.Label=Maximum retries
+WorkflowRunConfigurationDialog.LoadBalancingMaxRetries.ToolTip=How many extra times to try another server after a submit or capacity failure. Use '${MAX_RETRIES}' if you want this in a variable.
+WorkflowRunConfigurationDialog.LoadBalancingRetryWindow.Label=Retry window (ms)
+WorkflowRunConfigurationDialog.LoadBalancingRetryWindow.ToolTip=Give up retrying after this many milliseconds from the first attempt. 0 means retry count only. Use '${RETRY_WINDOW_MS}' if you want this in a variable.
+WorkflowRunConfigurationDialog.LoadBalancingRetryOnFailure.Label=Retry failed executions
+WorkflowRunConfigurationDialog.LoadBalancingRetryOnFailure.ToolTip=When enabled, also retry if the remote workflow runs and fails. Off by default because a failed run may already have written data.
+WorkflowRunConfigurationDialog.LoadBalancingProbeTimeout.Label=Server probe timeout (ms)
+WorkflowRunConfigurationDialog.LoadBalancingProbeTimeout.ToolTip=How long to wait for each Hop server status check. Default 3000.
+WorkflowRunConfigurationDialog.LoadBalancingConfigRefresh.Label=Configuration refresh interval (ms)
+WorkflowRunConfigurationDialog.LoadBalancingConfigRefresh.ToolTip=How often to re-read this run configuration from metadata so enable/disable and max concurrent changes take effect. Default 10000.
+WorkflowRunConfigurationDialog.LoadBalancingStateFolder.Label=Assignment state folder
+WorkflowRunConfigurationDialog.LoadBalancingStateFolder.ToolTip=Optional VFS folder for assignment ledger files, for example '${PROJECT_HOME}/loadbalance'. Leave empty to skip files.
diff --git a/ui/src/test/java/org/apache/hop/ui/hopgui/PaletteEngineFilterTest.java b/ui/src/test/java/org/apache/hop/ui/hopgui/PaletteEngineFilterTest.java
index 7d0752f888..f7b34e0f68 100644
--- a/ui/src/test/java/org/apache/hop/ui/hopgui/PaletteEngineFilterTest.java
+++ b/ui/src/test/java/org/apache/hop/ui/hopgui/PaletteEngineFilterTest.java
@@ -51,6 +51,9 @@ class PaletteEngineFilterTest {
assertEquals(
PaletteEngineFilter.HOP_ENGINE_GROUP_ID,
PaletteEngineFilter.getPipelineEngineLabelForId("Remote"));
+ assertEquals(
+ PaletteEngineFilter.HOP_ENGINE_GROUP_ID,
+ PaletteEngineFilter.getPipelineEngineLabelForId("LoadBalancing"));
assertEquals(
PaletteEngineFilter.NO_FILTER_LABEL, PaletteEngineFilter.getPipelineEngineLabelForId(""));
}
@@ -74,7 +77,8 @@ class PaletteEngineFilterTest {
assertFalse(filter.isPluginAllowed(plugin(new String[] {"Beam*"}, new String[0])));
assertFalse(
filter.isPluginAllowed(
- plugin(new String[0], new String[] {"Local", "Remote", "LocalSingle"})));
+ plugin(
+ new String[0], new String[] {"Local", "Remote", "LoadBalancing", "LocalSingle"})));
}
/** A stored single-member id from an older release still filters as the whole group. */
@@ -97,7 +101,8 @@ class PaletteEngineFilterTest {
PaletteEngineFilter.forWorkflowEngineId(PaletteEngineFilter.HOP_ENGINE_GROUP_ID);
assertFalse(
- filter.isPluginAllowed(plugin(new String[0], new String[] {"Local", "Remote"})),
- "action excluded on both Hop workflow engines must stay hidden");
+ filter.isPluginAllowed(
+ plugin(new String[0], new String[] {"Local", "Remote", "LoadBalancing"})),
+ "action excluded on every Hop workflow engine must stay hidden");
}
}