You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by mc...@apache.org on 2022/10/27 09:54:49 UTC
[hop] branch master updated: HOP-4516 : Allow a default run configuration to be defined
This is an automated email from the ASF dual-hosted git repository.
mcasters pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new 395376f557 HOP-4516 : Allow a default run configuration to be defined
new edacc122c5 Merge pull request #1760 from mattcasters/cypher-builder
395376f557 is described below
commit 395376f557214b9ed0a8d1f5e6c2bd2fbb410be3
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Thu Oct 27 11:53:18 2022 +0200
HOP-4516 : Allow a default run configuration to be defined
---
.../metadata/pipeline-run-configuration/local.json | 3 +-
.../metadata/workflow-run-configuration/local.json | 3 +-
.../pipeline/config/PipelineRunConfiguration.java | 65 +++++++++++++++++++---
.../engines/local/LocalPipelineEngine.java | 19 +++++--
.../src/main/java/org/apache/hop/run/HopRun.java | 33 +++++++++--
.../workflow/config/WorkflowRunConfiguration.java | 58 ++++++++++++++++++-
.../engines/local/LocalWorkflowEngine.java | 17 ++++--
.../direct/BeamDirectPipelineEngineTest.java | 3 +-
.../engines/flink/BeamFlinkPipelineEngineTest.java | 3 +-
.../engines/spark/BeamSparkPipelineEngineTest.java | 4 +-
.../hop/beam/transform/PipelineTestBase.java | 20 +++----
.../apache/hop/projects/gui/ProjectsGuiPlugin.java | 2 +-
.../config/PipelineRunConfigurationEditor.java | 23 +++++++-
.../PipelineExecutionConfigurationDialog.java | 32 ++++++++---
.../config/WorkflowRunConfigurationEditor.java | 23 +++++++-
.../WorkflowExecutionConfigurationDialog.java | 38 ++++++++++---
.../config/messages/messages_en_US.properties | 1 +
17 files changed, 288 insertions(+), 59 deletions(-)
diff --git a/assemblies/static/src/main/resources/config/projects/default/metadata/pipeline-run-configuration/local.json b/assemblies/static/src/main/resources/config/projects/default/metadata/pipeline-run-configuration/local.json
index 49c4a99803..efa9f57c94 100644
--- a/assemblies/static/src/main/resources/config/projects/default/metadata/pipeline-run-configuration/local.json
+++ b/assemblies/static/src/main/resources/config/projects/default/metadata/pipeline-run-configuration/local.json
@@ -15,5 +15,6 @@
"configurationVariables": [],
"description": "",
"executionInfoLocationName": "local-audit",
- "dataProfile": "first-last"
+ "dataProfile": "first-last",
+ "defaultSelection": true
}
diff --git a/assemblies/static/src/main/resources/config/projects/default/metadata/workflow-run-configuration/local.json b/assemblies/static/src/main/resources/config/projects/default/metadata/workflow-run-configuration/local.json
index 5c44d45cba..8db10866ce 100644
--- a/assemblies/static/src/main/resources/config/projects/default/metadata/workflow-run-configuration/local.json
+++ b/assemblies/static/src/main/resources/config/projects/default/metadata/workflow-run-configuration/local.json
@@ -6,5 +6,6 @@
},
"name": "local",
"description": "",
- "executionInfoLocationName": "local-audit"
+ "executionInfoLocationName": "local-audit",
+ "defaultSelection": true
}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/config/PipelineRunConfiguration.java b/engine/src/main/java/org/apache/hop/pipeline/config/PipelineRunConfiguration.java
index 88360d97b0..c7cf73f4d4 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/config/PipelineRunConfiguration.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/config/PipelineRunConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.hop.pipeline.config;
import org.apache.commons.lang.StringUtils;
+import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.variables.DescribedVariable;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.execution.profiling.ExecutionDataProfile;
@@ -25,6 +26,7 @@ import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import java.util.ArrayList;
import java.util.List;
@@ -42,9 +44,7 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
@HopMetadataProperty private String description;
- /**
- * The name of the location to send execution information to
- */
+ /** The name of the location to send execution information to */
@HopMetadataProperty private String executionInfoLocationName;
@HopMetadataProperty private List<DescribedVariable> configurationVariables;
@@ -52,9 +52,11 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
@HopMetadataProperty private IPipelineEngineRunConfiguration engineRunConfiguration;
/** The name of an {@link ExecutionDataProfile} */
- @HopMetadataProperty(key="dataProfile")
+ @HopMetadataProperty(key = "dataProfile")
protected String executionDataProfileName;
+ @HopMetadataProperty protected boolean defaultSelection;
+
public PipelineRunConfiguration() {
configurationVariables = new ArrayList<>();
}
@@ -65,13 +67,15 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
String executionInfoLocationName,
List<DescribedVariable> configurationVariables,
IPipelineEngineRunConfiguration engineRunConfiguration,
- String executionDataProfileName) {
+ String executionDataProfileName,
+ boolean defaultSelection) {
this.name = name;
this.description = description;
this.executionInfoLocationName = executionInfoLocationName;
this.configurationVariables = configurationVariables;
this.engineRunConfiguration = engineRunConfiguration;
this.executionDataProfileName = executionDataProfileName;
+ this.defaultSelection = defaultSelection;
}
public PipelineRunConfiguration(PipelineRunConfiguration runConfiguration) {
@@ -84,6 +88,7 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
this.engineRunConfiguration = runConfiguration.engineRunConfiguration.clone();
}
this.executionDataProfileName = runConfiguration.executionDataProfileName;
+ this.defaultSelection = runConfiguration.defaultSelection;
}
/**
@@ -95,7 +100,9 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
return description;
}
- /** @param description The description to set */
+ /**
+ * @param description The description to set
+ */
public void setDescription(String description) {
this.description = description;
}
@@ -127,7 +134,9 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
return configurationVariables;
}
- /** @param configurationVariables The configurationVariables to set */
+ /**
+ * @param configurationVariables The configurationVariables to set
+ */
public void setConfigurationVariables(List<DescribedVariable> configurationVariables) {
this.configurationVariables = configurationVariables;
}
@@ -141,7 +150,9 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
return engineRunConfiguration;
}
- /** @param engineRunConfiguration The engineRunConfiguration to set */
+ /**
+ * @param engineRunConfiguration The engineRunConfiguration to set
+ */
public void setEngineRunConfiguration(IPipelineEngineRunConfiguration engineRunConfiguration) {
this.engineRunConfiguration = engineRunConfiguration;
}
@@ -164,6 +175,24 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
this.executionDataProfileName = executionDataProfileName;
}
+ /**
+ * Gets defaultSelection
+ *
+ * @return value of defaultSelection
+ */
+ public boolean isDefaultSelection() {
+ return defaultSelection;
+ }
+
+ /**
+ * Sets defaultSelection
+ *
+ * @param defaultSelection value of defaultSelection
+ */
+ public void setDefaultSelection(boolean defaultSelection) {
+ this.defaultSelection = defaultSelection;
+ }
+
public void applyToVariables(IVariables variables) {
for (DescribedVariable vvd : configurationVariables) {
if (StringUtils.isNotEmpty(vvd.getName())) {
@@ -171,4 +200,24 @@ public class PipelineRunConfiguration extends HopMetadataBase implements Cloneab
}
}
}
+
+ /**
+ * Find the first default run configuration in the metadata and return it.
+ * Return null if there's no default.
+ *
+ * @param metadataProvider
+ * @return The default run configuration or null if none is specified.
+ * @throws HopException
+ */
+ public static final PipelineRunConfiguration findDefault(
+ IHopMetadataProvider metadataProvider) throws HopException {
+ for (PipelineRunConfiguration runConfiguration :
+ metadataProvider.getSerializer(PipelineRunConfiguration.class).loadAll()) {
+ if (runConfiguration.isDefaultSelection()) {
+ return runConfiguration;
+ }
+ }
+
+ return null;
+ }
}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
index 55adb63cc6..83489ef6db 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
@@ -29,7 +29,12 @@ import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.parameters.INamedParameters;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.execution.*;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionDataBuilder;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.IExecutionInfoLocation;
import org.apache.hop.execution.profiling.ExecutionDataProfile;
import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
@@ -40,14 +45,19 @@ import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.config.IPipelineEngineRunConfiguration;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
-import org.apache.hop.pipeline.engine.IEngineComponent;
import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.pipeline.engine.PipelineEngineCapabilities;
import org.apache.hop.pipeline.engine.PipelineEnginePlugin;
import org.apache.hop.pipeline.transform.IRowListener;
import org.apache.hop.pipeline.transform.TransformMetaDataCombi;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.UUID;
@PipelineEnginePlugin(
id = "Local",
@@ -98,7 +108,8 @@ public class LocalPipelineEngine extends Pipeline implements IPipelineEngine<Pip
"",
new ArrayList<>(),
createDefaultPipelineEngineRunConfiguration(),
- null));
+ null,
+ false));
}
@Override
diff --git a/engine/src/main/java/org/apache/hop/run/HopRun.java b/engine/src/main/java/org/apache/hop/run/HopRun.java
index 060a74e550..de60015bab 100644
--- a/engine/src/main/java/org/apache/hop/run/HopRun.java
+++ b/engine/src/main/java/org/apache/hop/run/HopRun.java
@@ -48,11 +48,13 @@ import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataUtil;
import org.apache.hop.pipeline.PipelineExecutionConfiguration;
import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.pipeline.engine.IPipelineEngine;
import org.apache.hop.pipeline.engine.PipelineEngineFactory;
import org.apache.hop.server.HopServer;
import org.apache.hop.workflow.WorkflowExecutionConfiguration;
import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.apache.hop.workflow.engine.IWorkflowEngine;
import org.apache.hop.workflow.engine.WorkflowEngineFactory;
import picocli.CommandLine;
@@ -60,7 +62,6 @@ import picocli.CommandLine.ExecutionException;
import picocli.CommandLine.Option;
import picocli.CommandLine.ParameterException;
-import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -160,7 +161,7 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
SerializableMetadataProvider exportedProvider = new SerializableMetadataProvider(json);
metadataProvider.getProviders().add(exportedProvider);
- System.out.println("===> Metadata provider is now: "+metadataProvider.getDescription());
+ System.out.println("===> Metadata provider is now: " + metadataProvider.getDescription());
}
}
@@ -229,6 +230,17 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
//
parseOptions(cmd, configuration, pipelineMeta);
+ // Do we have a default run configuration?
+ // That way the user doesn't have to specify the run configuration name
+ //
+ if (StringUtils.isEmpty(configuration.getRunConfiguration())) {
+ PipelineRunConfiguration defaultRunConfiguration =
+ PipelineRunConfiguration.findDefault(metadataProvider);
+ if (defaultRunConfiguration != null) {
+ configuration.setRunConfiguration(defaultRunConfiguration.getName());
+ }
+ }
+
// Before running, do we print the options?
//
if (printingOptions) {
@@ -305,6 +317,17 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
//
parseOptions(cmd, configuration, workflowMeta);
+ // Do we have a default run configuration?
+ // That way the user doesn't have to specify the run configuration name
+ //
+ if (StringUtils.isEmpty(configuration.getRunConfiguration())) {
+ WorkflowRunConfiguration defaultRunConfiguration =
+ WorkflowRunConfiguration.findDefault(metadataProvider);
+ if (defaultRunConfiguration != null) {
+ configuration.setRunConfiguration(defaultRunConfiguration.getName());
+ }
+ }
+
// Certain Hop plugins rely on this. Meh.
//
ExtensionPointHandler.callExtensionPoint(
@@ -473,7 +496,7 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
//
variables.setVariables(configuration.getVariablesMap());
- // By default we use the value from the current variables map:
+ // By default, we use the value from the current variables map:
//
for (String key : namedParams.listParameters()) {
String value = variables.getVariable(key);
@@ -509,8 +532,8 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
if (StringUtils.isNotEmpty(realFilename)) {
log.logMinimal("OPTION: filename : '" + realFilename + "'");
}
- if (StringUtils.isNotEmpty(realRunConfigurationName)) {
- log.logMinimal("OPTION: run configuration : '" + realRunConfigurationName + "'");
+ if (StringUtils.isNotEmpty(configuration.getRunConfiguration())) {
+ log.logMinimal("OPTION: run configuration : '" + configuration.getRunConfiguration() + "'");
}
log.logMinimal("OPTION: Logging level : " + configuration.getLogLevel().getDescription());
diff --git a/engine/src/main/java/org/apache/hop/workflow/config/WorkflowRunConfiguration.java b/engine/src/main/java/org/apache/hop/workflow/config/WorkflowRunConfiguration.java
index 1b8b30a644..0b0f743482 100644
--- a/engine/src/main/java/org/apache/hop/workflow/config/WorkflowRunConfiguration.java
+++ b/engine/src/main/java/org/apache/hop/workflow/config/WorkflowRunConfiguration.java
@@ -17,10 +17,12 @@
package org.apache.hop.workflow.config;
+import org.apache.hop.core.exception.HopException;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
@HopMetadata(
key = "workflow-run-configuration",
@@ -39,15 +41,22 @@ public class WorkflowRunConfiguration extends HopMetadataBase implements Cloneab
@HopMetadataProperty private String executionInfoLocationName;
+ @HopMetadataProperty protected boolean defaultSelection;
+
public WorkflowRunConfiguration() {}
public WorkflowRunConfiguration(
- String name, String description, String executionInfoLocationName, IWorkflowEngineRunConfiguration engineRunConfiguration) {
+ String name,
+ String description,
+ String executionInfoLocationName,
+ IWorkflowEngineRunConfiguration engineRunConfiguration,
+ boolean defaultSelection) {
this();
this.name = name;
this.description = description;
this.executionInfoLocationName = executionInfoLocationName;
this.engineRunConfiguration = engineRunConfiguration;
+ this.defaultSelection = defaultSelection;
}
public WorkflowRunConfiguration(WorkflowRunConfiguration c) {
@@ -57,6 +66,7 @@ public class WorkflowRunConfiguration extends HopMetadataBase implements Cloneab
if (c.engineRunConfiguration != null) {
this.engineRunConfiguration = c.engineRunConfiguration.clone();
}
+ this.defaultSelection = c.defaultSelection;
}
@Override
@@ -73,7 +83,9 @@ public class WorkflowRunConfiguration extends HopMetadataBase implements Cloneab
return description;
}
- /** @param description The description to set */
+ /**
+ * @param description The description to set
+ */
public void setDescription(String description) {
this.description = description;
}
@@ -87,7 +99,9 @@ public class WorkflowRunConfiguration extends HopMetadataBase implements Cloneab
return engineRunConfiguration;
}
- /** @param engineRunConfiguration The engineRunConfiguration to set */
+ /**
+ * @param engineRunConfiguration The engineRunConfiguration to set
+ */
public void setEngineRunConfiguration(IWorkflowEngineRunConfiguration engineRunConfiguration) {
this.engineRunConfiguration = engineRunConfiguration;
}
@@ -109,4 +123,42 @@ public class WorkflowRunConfiguration extends HopMetadataBase implements Cloneab
public void setExecutionInfoLocationName(String executionInfoLocationName) {
this.executionInfoLocationName = executionInfoLocationName;
}
+
+ /**
+ * Gets defaultSelection
+ *
+ * @return value of defaultSelection
+ */
+ public boolean isDefaultSelection() {
+ return defaultSelection;
+ }
+
+ /**
+ * Sets defaultSelection
+ *
+ * @param defaultSelection value of defaultSelection
+ */
+ public void setDefaultSelection(boolean defaultSelection) {
+ this.defaultSelection = defaultSelection;
+ }
+
+ /**
+ * Find the first default run configuration in the metadata and return it. Return null if there's
+ * no default.
+ *
+ * @param metadataProvider
+ * @return The default run configuration or null if none is specified.
+ * @throws HopException
+ */
+ public static final WorkflowRunConfiguration findDefault(IHopMetadataProvider metadataProvider)
+ throws HopException {
+ for (WorkflowRunConfiguration runConfiguration :
+ metadataProvider.getSerializer(WorkflowRunConfiguration.class).loadAll()) {
+ if (runConfiguration.isDefaultSelection()) {
+ return runConfiguration;
+ }
+ }
+
+ return null;
+ }
}
diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/local/LocalWorkflowEngine.java b/engine/src/main/java/org/apache/hop/workflow/engines/local/LocalWorkflowEngine.java
index acbc67b2ff..0ec7f43875 100644
--- a/engine/src/main/java/org/apache/hop/workflow/engines/local/LocalWorkflowEngine.java
+++ b/engine/src/main/java/org/apache/hop/workflow/engines/local/LocalWorkflowEngine.java
@@ -29,8 +29,13 @@ import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
-import org.apache.hop.execution.*;
-import org.apache.hop.execution.profiling.ExecutionDataProfile;
+import org.apache.hop.execution.ExecutionBuilder;
+import org.apache.hop.execution.ExecutionDataBuilder;
+import org.apache.hop.execution.ExecutionInfoLocation;
+import org.apache.hop.execution.ExecutionState;
+import org.apache.hop.execution.ExecutionStateBuilder;
+import org.apache.hop.execution.ExecutionType;
+import org.apache.hop.execution.IExecutionInfoLocation;
import org.apache.hop.workflow.IActionListener;
import org.apache.hop.workflow.Workflow;
import org.apache.hop.workflow.WorkflowMeta;
@@ -41,7 +46,11 @@ import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.apache.hop.workflow.engine.IWorkflowEngine;
import org.apache.hop.workflow.engine.WorkflowEnginePlugin;
-import java.util.*;
+import java.util.Date;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@WorkflowEnginePlugin(
@@ -76,7 +85,7 @@ public class LocalWorkflowEngine extends Workflow implements IWorkflowEngine<Wor
private void setDefaultRunConfiguration() {
setWorkflowRunConfiguration(
new WorkflowRunConfiguration(
- "local", "", "", createDefaultWorkflowEngineRunConfiguration()));
+ "local", "", "", createDefaultWorkflowEngineRunConfiguration(), false));
}
@Override
diff --git a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/direct/BeamDirectPipelineEngineTest.java b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/direct/BeamDirectPipelineEngineTest.java
index 84b36d6e2f..20fbb1c6a3 100644
--- a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/direct/BeamDirectPipelineEngineTest.java
+++ b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/direct/BeamDirectPipelineEngineTest.java
@@ -44,7 +44,8 @@ public class BeamDirectPipelineEngineTest extends BeamBasePipelineEngineTest {
"",
Arrays.asList(new DescribedVariable("VAR1", "value1", "description1")),
configuration,
- null);
+ null,
+ false);
metadataProvider.getSerializer(PipelineRunConfiguration.class).save(pipelineRunConfiguration);
diff --git a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineEngineTest.java b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineEngineTest.java
index 22aecbee1c..b359adb040 100644
--- a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineEngineTest.java
+++ b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineEngineTest.java
@@ -45,7 +45,8 @@ public class BeamFlinkPipelineEngineTest extends BeamBasePipelineEngineTest {
"",
Arrays.asList(new DescribedVariable("VAR1", "flink1", "description1")),
configuration,
- null);
+ null,
+ false);
// Save the metadata
metadataProvider.getSerializer(PipelineRunConfiguration.class).save(pipelineRunConfiguration);
diff --git a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
index d205c30bc5..c36e64caf9 100644
--- a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
+++ b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineEngineTest.java
@@ -23,7 +23,6 @@ import org.apache.hop.core.variables.DescribedVariable;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.pipeline.engine.IPipelineEngine;
-import org.junit.Ignore;
import org.junit.Test;
import java.util.Arrays;
@@ -46,7 +45,8 @@ public class BeamSparkPipelineEngineTest extends BeamBasePipelineEngineTest {
"",
Arrays.asList(new DescribedVariable("VAR1", "spark1", "description1")),
configuration,
- null);
+ null,
+ false);
metadataProvider.getSerializer(PipelineRunConfiguration.class).save(pipelineRunConfiguration);
diff --git a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/PipelineTestBase.java b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/PipelineTestBase.java
index 687c6dd5cf..42f682e382 100644
--- a/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/PipelineTestBase.java
+++ b/plugins/engines/beam/src/test/java/org/apache/hop/beam/transform/PipelineTestBase.java
@@ -17,13 +17,6 @@
package org.apache.hop.beam.transform;
-import org.apache.beam.sdk.PipelineResult;
-import org.apache.beam.sdk.metrics.MetricQueryResults;
-import org.apache.beam.sdk.metrics.MetricResult;
-import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.metrics.MetricsFilter;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.commons.io.FileUtils;
import org.apache.hop.beam.core.BeamHop;
import org.apache.hop.beam.engines.direct.BeamDirectPipelineEngine;
@@ -31,7 +24,6 @@ import org.apache.hop.beam.engines.direct.BeamDirectPipelineRunConfiguration;
import org.apache.hop.beam.engines.flink.BeamFlinkPipelineEngine;
import org.apache.hop.beam.engines.spark.BeamSparkPipelineEngine;
import org.apache.hop.beam.metadata.FileDefinition;
-import org.apache.hop.beam.pipeline.HopPipelineMetaToBeamPipelineConverter;
import org.apache.hop.beam.transforms.bq.BeamBQInputMeta;
import org.apache.hop.beam.transforms.bq.BeamBQOutputMeta;
import org.apache.hop.beam.transforms.io.BeamInputMeta;
@@ -42,7 +34,6 @@ import org.apache.hop.beam.transforms.pubsub.BeamPublishMeta;
import org.apache.hop.beam.transforms.pubsub.BeamSubscribeMeta;
import org.apache.hop.beam.transforms.window.BeamTimestampMeta;
import org.apache.hop.beam.transforms.window.BeamWindowMeta;
-import org.apache.hop.beam.util.BeamConst;
import org.apache.hop.core.Const;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.variables.IVariables;
@@ -63,7 +54,13 @@ import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
-import org.apache.hop.pipeline.engine.*;
+import org.apache.hop.pipeline.engine.EngineMetrics;
+import org.apache.hop.pipeline.engine.IEngineComponent;
+import org.apache.hop.pipeline.engine.IEngineMetric;
+import org.apache.hop.pipeline.engine.IPipelineEngine;
+import org.apache.hop.pipeline.engine.PipelineEngineFactory;
+import org.apache.hop.pipeline.engine.PipelineEnginePlugin;
+import org.apache.hop.pipeline.engine.PipelineEnginePluginType;
import org.apache.hop.pipeline.transforms.constant.ConstantMeta;
import org.apache.hop.pipeline.transforms.filterrows.FilterRowsMeta;
import org.apache.hop.pipeline.transforms.memgroupby.MemoryGroupByMeta;
@@ -188,7 +185,8 @@ public class PipelineTestBase {
NAME_LOCATION,
Collections.emptyList(),
directRunConfiguration,
- NAME_DATA_PROFILE);
+ NAME_DATA_PROFILE,
+ true);
metadataProvider.getSerializer(PipelineRunConfiguration.class).save(runConfiguration);
File inputFolder = new File("/tmp/customers/io");
diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
index 350d468a1a..2ede7ff63c 100644
--- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
+++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
@@ -378,7 +378,7 @@ public class ProjectsGuiPlugin {
BaseMessages.getString(
PKG, "ProjectGuiPlugin.LocalWFRunConfigDescription.Text"),
null,
- localWorkflowRunConfiguration);
+ localWorkflowRunConfiguration, true);
wrcSerializer.save(local);
}
}
diff --git a/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java b/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
index 2ddf974c39..0effdfe99c 100644
--- a/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/pipeline/config/PipelineRunConfigurationEditor.java
@@ -73,6 +73,7 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
private Text wName;
private Text wDescription;
+ private Button wDefault;
private MetaSelectionLine<ExecutionInfoLocation> wExecutionInfoLocation;
private MetaSelectionLine<ExecutionDataProfile> wProfile;
private ComboVar wPluginType;
@@ -203,6 +204,24 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
wDescription.setLayoutData(fdDescription);
lastControl = wDescription;
+ Label wlDefault = new Label(wMainComp, SWT.RIGHT);
+ props.setLook(wlDefault);
+ wlDefault.setText(
+ BaseMessages.getString(PKG, "PipelineRunConfigurationDialog.label.Default"));
+ FormData fdlDefault = new FormData();
+ fdlDefault.top = new FormAttachment(lastControl, margin * 2);
+ fdlDefault.left = new FormAttachment(0, 0); // First one in the left top corner
+ fdlDefault.right = new FormAttachment(middle, 0);
+ wlDefault.setLayoutData(fdlDefault);
+ wDefault = new Button(wMainComp, SWT.CHECK | SWT.LEFT );
+ props.setLook(wDefault);
+ FormData fdDefault = new FormData();
+ fdDefault.top = new FormAttachment(wlDefault, 0, SWT.CENTER);
+ fdDefault.left = new FormAttachment(middle, margin); // To the right of the label
+ fdDefault.right = new FormAttachment(100, 0);
+ wDefault.setLayoutData(fdDefault);
+ lastControl = wlDefault;
+
// Which location should the execution information go to?
//
wExecutionInfoLocation =
@@ -218,7 +237,7 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
PKG, "PipelineRunConfigurationDialog.toolTip.ExecutionInfoLocation"));
props.setLook(wExecutionInfoLocation);
FormData fdExecutionInfoLocation = new FormData();
- fdExecutionInfoLocation.top = new FormAttachment(lastControl, margin);
+ fdExecutionInfoLocation.top = new FormAttachment(lastControl, 2*margin);
fdExecutionInfoLocation.left = new FormAttachment(0, 0); // To the right of the label
fdExecutionInfoLocation.right = new FormAttachment(100, 0);
wExecutionInfoLocation.setLayoutData(fdExecutionInfoLocation);
@@ -482,6 +501,7 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
wName.setText(Const.NVL(workingConfiguration.getName(), ""));
wDescription.setText(Const.NVL(workingConfiguration.getDescription(), ""));
+ wDefault.setSelection(workingConfiguration.isDefaultSelection());
try {
wExecutionInfoLocation.fillItems();
} catch (Exception e) {
@@ -526,6 +546,7 @@ public class PipelineRunConfigurationEditor extends MetadataEditor<PipelineRunCo
meta.setName(wName.getText());
meta.setDescription(wDescription.getText());
+ meta.setDefaultSelection(wDefault.getSelection());
meta.setExecutionInfoLocationName(wExecutionInfoLocation.getText());
// Get the plugin specific information from the widgets on the screen
diff --git a/ui/src/main/java/org/apache/hop/ui/pipeline/dialog/PipelineExecutionConfigurationDialog.java b/ui/src/main/java/org/apache/hop/ui/pipeline/dialog/PipelineExecutionConfigurationDialog.java
index 15f44901ca..fb9dbafd74 100644
--- a/ui/src/main/java/org/apache/hop/ui/pipeline/dialog/PipelineExecutionConfigurationDialog.java
+++ b/ui/src/main/java/org/apache/hop/ui/pipeline/dialog/PipelineExecutionConfigurationDialog.java
@@ -41,7 +41,11 @@ import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
-import org.eclipse.swt.widgets.*;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.TableItem;
import java.util.ArrayList;
import java.util.Collections;
@@ -195,17 +199,30 @@ public class PipelineExecutionConfigurationDialog extends ConfigurationDialog {
Map<String, String> pipelineUsageMap = null;
String lastGlobalRunConfig =
AuditManagerGuiUtil.getLastUsedValue(AUDIT_LIST_TYPE_LAST_USED_RUN_CONFIGURATIONS);
- String lastPipelineRunConfig = null;
+ String selectedRunConfig = null;
if (StringUtils.isNotEmpty(abstractMeta.getName())) {
pipelineUsageMap = AuditManagerGuiUtil.getUsageMap(MAP_TYPE_PIPELINE_RUN_CONFIG_USAGE);
- lastPipelineRunConfig = pipelineUsageMap.get(abstractMeta.getName());
+ selectedRunConfig = pipelineUsageMap.get(abstractMeta.getName());
}
- wRunConfiguration.setText(Const.NVL(lastPipelineRunConfig, ""));
+ if (StringUtils.isEmpty(selectedRunConfig)) {
+ // What is the default?
+ PipelineRunConfiguration defaultRunConfig = null;
+ try {
+ defaultRunConfig = PipelineRunConfiguration.findDefault(hopGui.getMetadataProvider());
+ } catch (HopException e) {
+ LogChannel.UI.logError("Error finding default pipeline run configuration", e);
+ }
+ if (defaultRunConfig != null) {
+ selectedRunConfig = defaultRunConfig.getName();
+ }
+ }
+
+ wRunConfiguration.setText(Const.NVL(selectedRunConfig, ""));
- if (StringUtils.isNotEmpty(lastPipelineRunConfig)
+ if (StringUtils.isNotEmpty(selectedRunConfig)
&& StringUtils.isNotEmpty(lastGlobalRunConfig)
- && !lastPipelineRunConfig.equals(lastGlobalRunConfig)) {
+ && !selectedRunConfig.equals(lastGlobalRunConfig)) {
wRunConfiguration
.getLabelWidget()
.setBackground(GuiResource.getInstance().getColorLightBlue());
@@ -355,7 +372,8 @@ public class PipelineExecutionConfigurationDialog extends ConfigurationDialog {
PKG, "PipelineExecutionConfigurationDialog.LocalRunConfiguration.Description"),
new ArrayList<>(),
localPipelineRunConfiguration,
- null);
+ null,
+ true);
prcSerializer.save(local);
return local.getName();
diff --git a/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java b/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
index 913fcd0689..0d4c87ff62 100644
--- a/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/workflow/config/WorkflowRunConfigurationEditor.java
@@ -66,6 +66,7 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
private Text wName;
private Text wDescription;
+ private Button wDefault;
private MetaSelectionLine<ExecutionInfoLocation> wExecutionInfoLocation;
private ComboVar wPluginType;
@@ -172,6 +173,24 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
wDescription.setLayoutData(fdDescription);
lastControl = wDescription;
+ Label wlDefault = new Label(parent, SWT.RIGHT);
+ props.setLook(wlDefault);
+ wlDefault.setText(
+ BaseMessages.getString(PKG, "PipelineRunConfigurationDialog.label.Default"));
+ FormData fdlDefault = new FormData();
+ fdlDefault.top = new FormAttachment(lastControl, margin * 2);
+ fdlDefault.left = new FormAttachment(0, 0); // First one in the left top corner
+ fdlDefault.right = new FormAttachment(middle, 0);
+ wlDefault.setLayoutData(fdlDefault);
+ wDefault = new Button(parent, SWT.CHECK | SWT.LEFT );
+ props.setLook(wDefault);
+ FormData fdDefault = new FormData();
+ fdDefault.top = new FormAttachment(wlDefault, 0, SWT.CENTER);
+ fdDefault.left = new FormAttachment(middle, margin); // To the right of the label
+ fdDefault.right = new FormAttachment(100, 0);
+ wDefault.setLayoutData(fdDefault);
+ lastControl = wlDefault;
+
// Which location should the execution information go to?
//
wExecutionInfoLocation =
@@ -187,7 +206,7 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
PKG, "WorkflowRunConfigurationDialog.toolTip.ExecutionInfoLocation"));
props.setLook(wExecutionInfoLocation);
FormData fdExecutionInfoLocation = new FormData();
- fdExecutionInfoLocation.top = new FormAttachment(lastControl, margin);
+ fdExecutionInfoLocation.top = new FormAttachment(lastControl, 2*margin);
fdExecutionInfoLocation.left = new FormAttachment(0, 0); // To the right of the label
fdExecutionInfoLocation.right = new FormAttachment(100, 0);
wExecutionInfoLocation.setLayoutData(fdExecutionInfoLocation);
@@ -317,6 +336,7 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
wName.setText(Const.NVL(workingConfiguration.getName(), ""));
wDescription.setText(Const.NVL(workingConfiguration.getDescription(), ""));
+ wDefault.setSelection(workingConfiguration.isDefaultSelection());
try {
wExecutionInfoLocation.fillItems();
} catch(Exception e) {
@@ -340,6 +360,7 @@ public class WorkflowRunConfigurationEditor extends MetadataEditor<WorkflowRunCo
meta.setName(wName.getText());
meta.setDescription(wDescription.getText());
+ meta.setDefaultSelection(wDefault.getSelection());
meta.setExecutionInfoLocationName(wExecutionInfoLocation.getText());
// Get the plugin specific information from the widgets on the screen
diff --git a/ui/src/main/java/org/apache/hop/ui/workflow/dialog/WorkflowExecutionConfigurationDialog.java b/ui/src/main/java/org/apache/hop/ui/workflow/dialog/WorkflowExecutionConfigurationDialog.java
index 385db07530..27f382c8f6 100644
--- a/ui/src/main/java/org/apache/hop/ui/workflow/dialog/WorkflowExecutionConfigurationDialog.java
+++ b/ui/src/main/java/org/apache/hop/ui/workflow/dialog/WorkflowExecutionConfigurationDialog.java
@@ -22,11 +22,13 @@ import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.extension.ExtensionPointHandler;
import org.apache.hop.core.extension.HopExtensionPoint;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.logging.LogLevel;
import org.apache.hop.core.util.StringUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.ui.core.dialog.ConfigurationDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.gui.GuiResource;
@@ -42,7 +44,11 @@ import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
-import org.eclipse.swt.widgets.*;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.MessageBox;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.TableItem;
import java.util.ArrayList;
import java.util.Collections;
@@ -228,17 +234,30 @@ public class WorkflowExecutionConfigurationDialog extends ConfigurationDialog {
Map<String, String> workflowUsageMap = null;
String lastGlobalRunConfig =
AuditManagerGuiUtil.getLastUsedValue(AUDIT_LIST_TYPE_LAST_USED_RUN_CONFIGURATIONS);
- String lastWorkflowRunConfig = null;
+ String selectedRunConfig = null;
if (StringUtils.isNotEmpty(abstractMeta.getName())) {
workflowUsageMap = AuditManagerGuiUtil.getUsageMap(MAP_TYPE_WORKFLOW_RUN_CONFIG_USAGE);
- lastWorkflowRunConfig = workflowUsageMap.get(abstractMeta.getName());
+ selectedRunConfig = workflowUsageMap.get(abstractMeta.getName());
}
- wRunConfiguration.setText(Const.NVL(lastWorkflowRunConfig, ""));
+ if (StringUtils.isEmpty(selectedRunConfig)) {
+ // What is the default?
+ WorkflowRunConfiguration defaultRunConfig = null;
+ try {
+ defaultRunConfig = WorkflowRunConfiguration.findDefault(hopGui.getMetadataProvider());
+ } catch (HopException e) {
+ LogChannel.UI.logError("Error finding default workflow run configuration", e);
+ }
+ if (defaultRunConfig != null) {
+ selectedRunConfig = defaultRunConfig.getName();
+ }
+ }
+
+ wRunConfiguration.setText(Const.NVL(selectedRunConfig, ""));
- if (StringUtils.isNotEmpty(lastWorkflowRunConfig)
+ if (StringUtils.isNotEmpty(selectedRunConfig)
&& StringUtils.isNotEmpty(lastGlobalRunConfig)
- && !lastWorkflowRunConfig.equals(lastGlobalRunConfig)) {
+ && !selectedRunConfig.equals(lastGlobalRunConfig)) {
wRunConfiguration
.getLabelWidget()
.setBackground(GuiResource.getInstance().getColorLightBlue());
@@ -397,7 +416,8 @@ public class WorkflowExecutionConfigurationDialog extends ConfigurationDialog {
BaseMessages.getString(
PKG, "WorkflowExecutionConfigurationDialog.LocalRunConfiguration.Description"),
null,
- localWorkflowRunConfiguration);
+ localWorkflowRunConfiguration,
+ true);
prcSerializer.save(local);
return local.getName();
@@ -414,7 +434,9 @@ public class WorkflowExecutionConfigurationDialog extends ConfigurationDialog {
return null;
}
- /** @return the configuration */
+ /**
+ * @return the configuration
+ */
public WorkflowExecutionConfiguration getConfiguration() {
return (WorkflowExecutionConfiguration) configuration;
}
diff --git a/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
index 44fd91734b..20d0da6012 100644
--- a/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
+++ b/ui/src/main/resources/org/apache/hop/ui/pipeline/config/messages/messages_en_US.properties
@@ -16,6 +16,7 @@
#
PipelineRunConfigurationDialog.label.name=Name
PipelineRunConfigurationDialog.label.Description=Description
+PipelineRunConfigurationDialog.label.Default=Make this the default selection
PipelineRunConfigurationDialog.label.ExecutionInfoLocation=Execution information location
PipelineRunConfigurationDialog.toolTip.ExecutionInfoLocation=This is the location where all information regarding the execution of pipelines is handled.
PipelineRunConfigurationDialog.label.EngineType=Engine type