You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2022/08/08 11:02:24 UTC
[hop] branch master updated: HOP-4098: Allow a specific set of workflows to be logged in Workflow Log Metadata object
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new bc62b75394 HOP-4098: Allow a specific set of workflows to be logged in Workflow Log Metadata object
new 089a3b32a3 Merge pull request #1625 from uraychang/master
bc62b75394 is described below
commit bc62b7539497b686c570a32eedd12e131684c261
Author: ray.chang <ur...@gmail.com>
AuthorDate: Mon Aug 8 17:03:19 2022 +0800
HOP-4098: Allow a specific set of workflows to be logged in Workflow Log Metadata object
---
.../hop/reflection/workflow/meta/WorkflowLog.java | 19 +++++-
.../workflow/meta/WorkflowLogEditor.java | 75 +++++++++++++++++++---
.../workflow/xp/WorkflowStartLoggingXp.java | 39 +++++++++--
.../meta/messages/messages_en_US.properties | 2 +
4 files changed, 117 insertions(+), 18 deletions(-)
diff --git a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLog.java b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLog.java
index df74a19b52..b16e3dbe32 100644
--- a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLog.java
+++ b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLog.java
@@ -22,6 +22,10 @@ import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.reflection.pipeline.meta.PipelineToLogLocation;
+
+import java.util.ArrayList;
+import java.util.List;
@HopMetadata(
key = "workflow-log",
@@ -39,6 +43,7 @@ public class WorkflowLog extends HopMetadataBase implements IHopMetadata {
@HopMetadataProperty private boolean executingPeriodically;
@HopMetadataProperty private String intervalInSeconds;
@HopMetadataProperty private boolean executingAtEnd;
+ @HopMetadataProperty private List<String> workflowToLog;
public WorkflowLog() {
enabled = true;
@@ -47,10 +52,12 @@ public class WorkflowLog extends HopMetadataBase implements IHopMetadata {
executingPeriodically = false;
intervalInSeconds = "30";
executingAtEnd = true;
+ workflowToLog = new ArrayList<>();
}
public WorkflowLog(String name) {
super(name);
+ workflowToLog = new ArrayList<>();
}
public WorkflowLog(
@@ -61,7 +68,8 @@ public class WorkflowLog extends HopMetadataBase implements IHopMetadata {
boolean executingAtStart,
boolean executingPeriodically,
String intervalInSeconds,
- boolean executingAtEnd) {
+ boolean executingAtEnd,
+ List<String> workflowToLog) {
super(name);
this.enabled = enabled;
this.loggingParentsOnly = loggingParentsOnly;
@@ -70,6 +78,7 @@ public class WorkflowLog extends HopMetadataBase implements IHopMetadata {
this.executingPeriodically = executingPeriodically;
this.intervalInSeconds = intervalInSeconds;
this.executingAtEnd = executingAtEnd;
+ this.workflowToLog = workflowToLog;
}
/**
@@ -169,4 +178,12 @@ public class WorkflowLog extends HopMetadataBase implements IHopMetadata {
public void setExecutingAtEnd(boolean executingAtEnd) {
this.executingAtEnd = executingAtEnd;
}
+
+ public List<String> getWorkflowToLog(){
+ return workflowToLog;
+ }
+
+ public void setWorkflowToLog(List<String> workflowToLog){
+ this.workflowToLog = workflowToLog;
+ }
}
diff --git a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLogEditor.java b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLogEditor.java
index a4870b8a66..1af966d99a 100644
--- a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLogEditor.java
+++ b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/meta/WorkflowLogEditor.java
@@ -25,12 +25,15 @@ import org.apache.hop.pipeline.PipelineHopMeta;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.reflection.pipeline.meta.PipelineToLogLocation;
import org.apache.hop.reflection.workflow.transform.WorkflowLoggingMeta;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.metadata.MetadataEditor;
import org.apache.hop.ui.core.metadata.MetadataManager;
+import org.apache.hop.ui.core.widget.ColumnInfo;
+import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
import org.apache.hop.ui.hopgui.file.pipeline.HopPipelineFileType;
@@ -40,6 +43,9 @@ import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.widgets.*;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Editor that allows you to change Workflow Log metadata
*
@@ -57,6 +63,7 @@ public class WorkflowLogEditor extends MetadataEditor<WorkflowLog> {
private Button wPeriodic;
private Label wlInterval;
private TextVar wInterval;
+ private TableView wWorkflows;
public WorkflowLogEditor(
HopGui hopGui, MetadataManager<WorkflowLog> manager, WorkflowLog metadata) {
@@ -267,6 +274,40 @@ public class WorkflowLogEditor extends MetadataEditor<WorkflowLog> {
wInterval.addListener(SWT.Selection, this::enableFields);
lastControl = wlInterval;
+ // The locations in a table view:
+ //
+ Label wlSources = new Label(parent, SWT.LEFT);
+ props.setLook(wlSources);
+ wlSources.setText(BaseMessages.getString(PKG, "WorkflowLoggingEditor.Sources.Label"));
+ FormData fdlSources = new FormData();
+ fdlSources.left = new FormAttachment(0, 0);
+ fdlSources.right = new FormAttachment(100, 0);
+ fdlSources.top = new FormAttachment(lastControl, 2 * margin);
+ wlSources.setLayoutData(fdlSources);
+ lastControl = wlSources;
+ ColumnInfo[] columns = {
+ new ColumnInfo(
+ BaseMessages.getString(PKG, "WorkflowLoggingEditor.SourcesTable.Column.Pipeline"),
+ ColumnInfo.COLUMN_TYPE_TEXT,
+ false,
+ false),
+ };
+ wWorkflows =
+ new TableView(
+ manager.getVariables(),
+ parent,
+ SWT.BORDER,
+ columns,
+ 0,
+ e -> setChanged(),
+ props);
+ FormData fdSources = new FormData();
+ fdSources.left = new FormAttachment(0, 0);
+ fdSources.top = new FormAttachment(lastControl, margin);
+ fdSources.right = new FormAttachment(100, 0);
+ fdSources.bottom = new FormAttachment(100, 0);
+ wWorkflows.setLayoutData(fdSources);
+
setWidgetsContent();
// Add listener to detect change after loading data
@@ -382,16 +423,24 @@ public class WorkflowLogEditor extends MetadataEditor<WorkflowLog> {
@Override
public void setWidgetsContent() {
- WorkflowLog pl = getMetadata();
-
- wName.setText(Const.NVL(pl.getName(), ""));
- wEnabled.setSelection(pl.isEnabled());
- wLoggingParentsOnly.setSelection(pl.isLoggingParentsOnly());
- wFilename.setText(Const.NVL(pl.getPipelineFilename(), ""));
- wAtStart.setSelection(pl.isExecutingAtStart());
- wAtEnd.setSelection(pl.isExecutingAtEnd());
- wPeriodic.setSelection(pl.isExecutingPeriodically());
- wInterval.setText(Const.NVL(pl.getIntervalInSeconds(), ""));
+ WorkflowLog wl = getMetadata();
+
+ wName.setText(Const.NVL(wl.getName(), ""));
+ wEnabled.setSelection(wl.isEnabled());
+ wLoggingParentsOnly.setSelection(wl.isLoggingParentsOnly());
+ wFilename.setText(Const.NVL(wl.getPipelineFilename(), ""));
+ wAtStart.setSelection(wl.isExecutingAtStart());
+ wAtEnd.setSelection(wl.isExecutingAtEnd());
+ wPeriodic.setSelection(wl.isExecutingPeriodically());
+ wInterval.setText(Const.NVL(wl.getIntervalInSeconds(), ""));
+ wWorkflows.removeAll();
+ List<String> workflowsToLog = wl.getWorkflowToLog();
+ for(String workflowToLog : workflowsToLog){
+ TableItem item = new TableItem(wWorkflows.table, SWT.NONE);
+ item.setText(1, Const.NVL(workflowToLog, ""));
+ }
+ wWorkflows.setRowNums();
+ wWorkflows.optimizeTableView();
}
@Override
@@ -404,6 +453,12 @@ public class WorkflowLogEditor extends MetadataEditor<WorkflowLog> {
pl.setExecutingAtEnd(wAtEnd.getSelection());
pl.setExecutingPeriodically(wPeriodic.getSelection());
pl.setIntervalInSeconds(wInterval.getText());
+ List<String> locations = new ArrayList<>();
+ List<TableItem> items = wWorkflows.getNonEmptyItems();
+ for (TableItem item : items) {
+ locations.add(item.getText(1));
+ }
+ pl.setWorkflowToLog(locations);
}
@Override
diff --git a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/xp/WorkflowStartLoggingXp.java b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/xp/WorkflowStartLoggingXp.java
index 35a653ae64..f8a84d8d7e 100644
--- a/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/xp/WorkflowStartLoggingXp.java
+++ b/plugins/misc/reflection/src/main/java/org/apache/hop/reflection/workflow/xp/WorkflowStartLoggingXp.java
@@ -18,6 +18,7 @@
package org.apache.hop.reflection.workflow.xp;
import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.extension.ExtensionPoint;
@@ -67,7 +68,7 @@ public class WorkflowStartLoggingXp implements IExtensionPoint<IWorkflowEngine<W
final IWorkflowEngine<WorkflowMeta> workflow,
final IVariables variables)
throws HopException {
- try {
+
// See if we need to do anything at all...
//
@@ -83,12 +84,13 @@ public class WorkflowStartLoggingXp implements IExtensionPoint<IWorkflowEngine<W
}
}
- // Load the pipeline filename specified in the Workflow Log object...
- //
- final String loggingPipelineFilename = variables.resolve(workflowLog.getPipelineFilename());
+ // Load the pipeline filename specified in the Workflow Log object...
+ //
+ final String loggingPipelineFilename = variables.resolve(workflowLog.getPipelineFilename());
- // See if the file exists...
- FileObject loggingFileObject = HopVfs.getFileObject(loggingPipelineFilename);
+ // See if the file exists...
+ FileObject loggingFileObject = HopVfs.getFileObject(loggingPipelineFilename);
+ try {
if (!loggingFileObject.exists()) {
log.logBasic(
"WARNING: The Workflow Log pipeline file '"
@@ -96,7 +98,31 @@ public class WorkflowStartLoggingXp implements IExtensionPoint<IWorkflowEngine<W
+ "' couldn't be found to execute.");
return;
}
+ } catch (Exception e) {
+ workflow.stopExecution();
+ throw new HopException(
+ "Error handling Workflow Log metadata object '"
+ + workflowLog.getName()
+ + "' at the start of pipeline: "
+ + workflow,
+ e);
+ }
+
+ if (workflowLog.getWorkflowToLog().isEmpty()) {
+ logWorkflow(workflowLog, workflow, variables, loggingPipelineFilename);
+ } else {
+ for (String workflowToLog : workflowLog.getWorkflowToLog()) {
+ String workflowUri = HopVfs.getFileObject(workflow.getFilename()).getPublicURIString();
+ String workflowToLogUri = HopVfs.getFileObject(variables.resolve(workflowToLog)).getPublicURIString();
+ if (workflowUri.equals(workflowToLogUri)) {
+ logWorkflow(workflowLog, workflow, variables, loggingPipelineFilename);
+ }
+ }
+ }
+ }
+ private void logWorkflow(WorkflowLog workflowLog, IWorkflowEngine<WorkflowMeta> workflow, IVariables variables, String loggingPipelineFilename) throws HopException {
+ try {
final Timer timer = new Timer();
if (workflowLog.isExecutingAtStart()) {
@@ -135,7 +161,6 @@ public class WorkflowStartLoggingXp implements IExtensionPoint<IWorkflowEngine<W
timer.schedule(timerTask, intervalInSeconds * 1000L, intervalInSeconds * 1000L);
}
}
-
} catch (Exception e) {
workflow.stopExecution();
throw new HopException(
diff --git a/plugins/misc/reflection/src/main/resources/org/apache/hop/reflection/workflow/meta/messages/messages_en_US.properties b/plugins/misc/reflection/src/main/resources/org/apache/hop/reflection/workflow/meta/messages/messages_en_US.properties
index aee1eb910a..d7992ec1a6 100644
--- a/plugins/misc/reflection/src/main/resources/org/apache/hop/reflection/workflow/meta/messages/messages_en_US.properties
+++ b/plugins/misc/reflection/src/main/resources/org/apache/hop/reflection/workflow/meta/messages/messages_en_US.properties
@@ -23,3 +23,5 @@ WorkflowLoggingEditor.AtStart.Label=Execute at the start of a pipeline?
WorkflowLoggingEditor.AtEnd.Label=Execute at the end of a pipeline?
WorkflowLoggingEditor.Periodic.Label=Execute periodically during execution?
WorkflowLoggingEditor.Interval.Label=Interval in seconds
+WorkflowLoggingEditor.Sources.Label=Capture output of the following workflows
+WorkflowLoggingEditor.SourcesTable.Column.Pipeline=Workflow