You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/07/28 12:14:28 UTC
(hop) branch main updated: do not block project load with invalid metadata objects, fixes #7655 (#7659)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 8e29319989 do not block project load with invalid metadata objects, fixes #7655 (#7659)
8e29319989 is described below
commit 8e293199893e6312212a54ccd127daa5fb9558c1
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Tue Jul 28 14:14:22 2026 +0200
do not block project load with invalid metadata objects, fixes #7655 (#7659)
---
core/src/main/java/org/apache/hop/core/Const.java | 20 +
.../serializer/json/JsonMetadataSerializer.java | 30 +-
.../test/java/org/apache/hop/core/ConstTest.java | 22 +
.../json/JsonMetadataSerializerTest.java | 53 +++
.../apache/hop/projects/gui/ProjectsGuiPlugin.java | 76 ++++
.../gui/messages/messages_en_US.properties | 3 +
.../perspective/metadata/MetadataPerspective.java | 484 ++++++++++++++++++++-
.../metadata/messages/messages_en_US.properties | 15 +-
8 files changed, 689 insertions(+), 14 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Const.java b/core/src/main/java/org/apache/hop/core/Const.java
index fa25fec12d..882e1b1ea3 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -2802,6 +2802,26 @@ public class Const {
+ ExceptionUtils.getRootCauseMessage(aThrowable);
}
+ /**
+ * Digs up the message of the deepest cause of an error and puts it on a single line. Use it to
+ * explain a problem in a dialog or in a list of problems: Hop exception messages are nested and
+ * spread over several lines, and only the deepest one says what actually went wrong.
+ *
+ * @param aThrowable the error to explain
+ * @return the message of the root cause, or the name of its class if it doesn't have one
+ */
+ public static String getRootCauseMessage(Throwable aThrowable) {
+ Throwable rootCause = ExceptionUtils.getRootCause(aThrowable);
+ if (rootCause == null) {
+ rootCause = aThrowable;
+ }
+ String message = rootCause.getMessage();
+ if (StringUtils.isEmpty(message)) {
+ return rootCause.getClass().getSimpleName();
+ }
+ return message.replaceAll("\\s+", " ").trim();
+ }
+
/**
* Create a valid filename using a name We remove all special characters, spaces, etc.
*
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
index fe82ae6267..a4b95333fd 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
+++ b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
@@ -34,7 +34,10 @@ import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
+import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.metadata.api.IHopMetadata;
@@ -82,7 +85,32 @@ public class JsonMetadataSerializer<T extends IHopMetadata> implements IHopMetad
List<String> names = listObjectNames();
Collections.sort(names);
for (String name : names) {
- list.add(load(name));
+ try {
+ T t = load(name);
+ if (t != null) {
+ list.add(t);
+ }
+ } catch (Exception e) {
+ // One object we can't load (a missing plugin, corrupt JSON, ...) shouldn't keep all the
+ // other objects of this type from being loaded. Log the problem and skip the object.
+ // Loading the object by name still reports the error to whoever really needs it.
+ //
+ String message =
+ "Error loading metadata object '"
+ + name
+ + "' of type "
+ + managedClass.getSimpleName()
+ + " from folder '"
+ + baseFolder
+ + "'. The object is ignored.";
+ if (HopLogStore.isInitialized()) {
+ LogChannel.GENERAL.logError(message, e);
+ } else {
+ // Serializers are also used before any logging is available.
+ //
+ System.err.println(message + Const.CR + Const.getStackTracker(e));
+ }
+ }
}
return list;
}
diff --git a/core/src/test/java/org/apache/hop/core/ConstTest.java b/core/src/test/java/org/apache/hop/core/ConstTest.java
index f94a6bb170..92b9278bcb 100644
--- a/core/src/test/java/org/apache/hop/core/ConstTest.java
+++ b/core/src/test/java/org/apache/hop/core/ConstTest.java
@@ -89,6 +89,28 @@ class ConstTest {
assertEquals("test1", Const.NVL(null, "test1"));
}
+ @Test
+ void testGetRootCauseMessage() {
+ // The deepest cause explains the problem, on a single line.
+ //
+ HopException nested =
+ new HopException(
+ "Error loading the object",
+ new HopException("Unable to find the plugin with id: BeamDirectPipelineEngine"));
+ assertEquals(
+ "Unable to find the plugin with id: BeamDirectPipelineEngine",
+ Const.getRootCauseMessage(nested));
+
+ // Without a cause the error itself is the root cause.
+ //
+ assertEquals(
+ "Something went wrong", Const.getRootCauseMessage(new Exception("Something went wrong")));
+
+ // Without a message we fall back to the class name.
+ //
+ assertEquals("NullPointerException", Const.getRootCauseMessage(new NullPointerException()));
+ }
+
@Test
void testNullToEmpty_NVL() {
assertEquals(Const.NVL(null, ""), Const.nullToEmpty(null));
diff --git a/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializerTest.java b/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializerTest.java
index 771a3eacbe..a2606d931a 100644
--- a/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializerTest.java
+++ b/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializerTest.java
@@ -17,11 +17,16 @@
package org.apache.hop.metadata.serializer.json;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hop.core.Const;
import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
@@ -37,6 +42,7 @@ import org.apache.hop.metadata.serializer.json.person.interest.Music;
import org.apache.hop.metadata.serializer.json.person.interest.Running;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
/** Unit test for {@link JsonMetadataSerializer} */
class JsonMetadataSerializerTest {
@@ -85,4 +91,51 @@ class JsonMetadataSerializerTest {
assertEquals(person, anotherPerson);
}
+
+ /**
+ * An object we can't load, for example because the plugin which created it isn't installed,
+ * shouldn't keep the other objects from being loaded. See also the "Unable to find the plugin"
+ * errors which prevented projects from being opened.
+ */
+ @Test
+ void testLoadAllIgnoresObjectsWeCantLoad(@TempDir Path folder) throws Exception {
+ JsonMetadataProvider provider =
+ new JsonMetadataProvider(
+ new HopTwoWayPasswordEncoder(),
+ folder.toString(),
+ Variables.getADefaultVariableSpace());
+
+ Occupation occupation = new Occupation("Occupation1", "Description of occupation1", 2001);
+ provider.getSerializer(Occupation.class).save(occupation);
+
+ IHopMetadataSerializer<Person> personSerializer = provider.getSerializer(Person.class);
+ personSerializer.save(
+ new Person(
+ "Person",
+ "51",
+ new Address("Street", "123", new City("12345", "McCity")),
+ new Music("Music", "Love Music"),
+ Arrays.asList(new Cooking("Cooking", "Cooking is great")),
+ new HashMap<>(),
+ occupation));
+
+ // Copy the person to a second file but with an interest which our object factory doesn't know.
+ // This is what happens when a plugin used to create an object isn't installed.
+ //
+ Path personFolder = folder.resolve("person");
+ String json = new String(Files.readAllBytes(personFolder.resolve("Person.json")), UTF_8);
+ Files.write(
+ personFolder.resolve("Gardener.json"),
+ json.replace("\"music\"", "\"gardening\"").getBytes(UTF_8));
+
+ // Loading all the objects simply skips the one we can't load...
+ //
+ List<Person> persons = personSerializer.loadAll();
+ assertEquals(1, persons.size());
+ assertEquals("Person", persons.get(0).getName());
+
+ // ... while loading it by name still reports the problem.
+ //
+ assertThrows(HopException.class, () -> personSerializer.load("Gardener"));
+ }
}
diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
index 94aee6ce3f..a2da9a7270 100644
--- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
+++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
@@ -51,6 +51,7 @@ import org.apache.hop.core.gui.plugin.toolbar.GuiToolbarElementType;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.metadata.SerializableMetadataProvider;
+import org.apache.hop.core.util.TranslateUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.DescribedVariable;
import org.apache.hop.core.variables.IVariables;
@@ -60,6 +61,8 @@ import org.apache.hop.history.AuditEvent;
import org.apache.hop.history.AuditList;
import org.apache.hop.history.AuditManager;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
@@ -305,6 +308,11 @@ public class ProjectsGuiPlugin {
// Reset VFS filesystem to load additional configurations
HopVfs.reset();
+
+ // Finally, warn about metadata elements in this project which we can't load.
+ // They're ignored so the project itself opens just fine.
+ //
+ reportUnloadableMetadata(hopGui);
} catch (Exception e) {
MissingProjectInfo missing = extractMissingProjectInfo(e);
if (missing != null) {
@@ -325,6 +333,74 @@ public class ProjectsGuiPlugin {
}
}
+ /**
+ * Metadata elements which can't be loaded (a missing plugin, invalid JSON, ...) are ignored when
+ * we load a project. We check for those here and show one dialog listing all of them so the user
+ * knows which elements are not available and why.
+ *
+ * @param hopGui the Hop GUI instance with the metadata provider of the project
+ */
+ private static void reportUnloadableMetadata(HopGui hopGui) {
+ List<String> problems = new ArrayList<>();
+
+ IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
+ for (Class<IHopMetadata> metadataClass : metadataProvider.getMetadataClasses()) {
+ try {
+ IHopMetadataSerializer<IHopMetadata> serializer =
+ metadataProvider.getSerializer(metadataClass);
+ HopMetadata annotation = metadataClass.getAnnotation(HopMetadata.class);
+ String typeName =
+ annotation == null
+ ? metadataClass.getSimpleName()
+ : TranslateUtil.translate(annotation.name(), metadataClass);
+
+ for (String name : serializer.listObjectNames()) {
+ try {
+ serializer.load(name);
+ } catch (Exception e) {
+ hopGui
+ .getLog()
+ .logError("Error loading " + typeName + " metadata element '" + name + "'", e);
+ problems.add(
+ " - "
+ + typeName
+ + " : "
+ + name
+ + Const.CR
+ + " "
+ + Const.getRootCauseMessage(e)
+ + Const.CR);
+ }
+ }
+ } catch (Exception e) {
+ // Checking for problems shouldn't cause any of its own: just log it and move on.
+ //
+ hopGui
+ .getLog()
+ .logError("Error checking metadata elements of type " + metadataClass.getName(), e);
+ }
+ }
+
+ if (problems.isEmpty()) {
+ return;
+ }
+
+ StringBuilder message =
+ new StringBuilder(
+ BaseMessages.getString(PKG, "ProjectGuiPlugin.MetadataNotLoaded.Dialog.Message"));
+ message.append(Const.CR).append(Const.CR);
+ problems.forEach(message::append);
+ message
+ .append(Const.CR)
+ .append(BaseMessages.getString(PKG, "ProjectGuiPlugin.MetadataNotLoaded.Dialog.Hint"));
+
+ MessageBox box = new MessageBox(hopGui.getActiveShell(), SWT.OK | SWT.ICON_WARNING);
+ box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.MetadataNotLoaded.Dialog.Header"));
+ box.setMessage(message.toString());
+ box.setMinimumSize(500, -1);
+ box.open();
+ }
+
/** Result when the failure is due to a project home folder that does not exist. */
public static class MissingProjectInfo {
public final String projectName;
diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties
index e829e8e681..481988830a 100644
--- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties
+++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties
@@ -106,6 +106,9 @@ ProjectGuiPlugin.Lifecycle.Dialog.Message2=With it you can manage the specific s
ProjectGuiPlugin.LocalWFRunConfig.Dialog.Header=Create local workflow run configuration?
ProjectGuiPlugin.LocalWFRunConfig.Dialog.Message=Do you want to have a local workflow run configuration for this project?
ProjectGuiPlugin.LocalWFRunConfigDescription.Text=Runs your workflows locally with the standard local Hop workflow engine
+ProjectGuiPlugin.MetadataNotLoaded.Dialog.Header=Metadata elements could not be loaded
+ProjectGuiPlugin.MetadataNotLoaded.Dialog.Message=The following metadata elements of this project could not be loaded and are ignored. This usually means that a plugin is missing.
+ProjectGuiPlugin.MetadataNotLoaded.Dialog.Hint=You can inspect or delete these elements in the Metadata perspective, under "Unknown".
ProjectGuiPlugin.ProjectExists.Dialog.Header=Configuration File Exists!
ProjectGuiPlugin.ProjectExists.Dialog.Message=An existing project configuration file was found in the specified folder. It will be used for this project.
ProjectGuiPlugin.ReloadProject.Dialog.Header=Reload project?
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
index e7e1859298..f0876ea64f 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.Getter;
+import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
import org.apache.hop.core.Props;
import org.apache.hop.core.exception.HopException;
@@ -45,6 +46,7 @@ import org.apache.hop.core.search.ISearchResult;
import org.apache.hop.core.search.ISearchable;
import org.apache.hop.core.util.TranslateUtil;
import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.history.AuditList;
import org.apache.hop.history.AuditManager;
import org.apache.hop.history.IAuditManager;
@@ -58,6 +60,8 @@ import org.apache.hop.metadata.plugin.MetadataPluginType;
import org.apache.hop.metadata.refactor.MetadataObjectReference;
import org.apache.hop.metadata.refactor.MetadataReferenceFinder;
import org.apache.hop.metadata.refactor.MetadataReferenceResult;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataUtil;
import org.apache.hop.ui.core.ConstUi;
import org.apache.hop.ui.core.FormDataBuilder;
@@ -193,14 +197,32 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
/** A category header node grouping several metadata types. */
public static final String CATEGORY = "Category";
+ /** The header node collecting everything in the project we can't load. */
+ public static final String UNKNOWN_CATEGORY = "UnknownCategory";
+
+ /** A type node under {@link #UNKNOWN_CATEGORY}: one metadata folder on disk. */
+ public static final String UNKNOWN_TYPE = "UnknownType";
+
+ /** An element under {@link #UNKNOWN_TYPE}: one JSON file we can't turn into a metadata object. */
+ public static final String UNKNOWN_FILE = "UnknownFile";
+
public static final String VIRTUAL_PATH = "virtualPath";
public static final String ERROR = "Error";
+ /** The full filename of the JSON file behind an {@link #UNKNOWN_FILE} node. */
+ private static final String KEY_FILENAME = "filename";
+
+ /** The reason why an {@link #UNKNOWN_FILE} node can't be loaded. */
+ private static final String KEY_REASON = "reason";
+
/** Icons for the "show empty types" toggle button (icon reflects the action it will perform). */
private static final String IMAGE_SHOW_ALL = "ui/images/show-all.svg";
private static final String IMAGE_SHOW_SELECTED = "ui/images/show-selected.svg";
+ /** Icon for the "Unknown" category and the metadata elements we can't load. */
+ private static final String IMAGE_UNKNOWN = "ui/images/warning.svg";
+
private static final int FILTER_DEBOUNCE_MS = 250;
@Getter private static MetadataPerspective instance;
@@ -227,6 +249,13 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
*/
private final List<MetadataTypeModel> typeModels = new ArrayList<>();
+ /**
+ * Everything in the project's metadata folders which we can't load, keyed by metadata type: JSON
+ * files of a type no installed plugin provides and elements of a known type which fail to load,
+ * usually because the plugin they were created with is missing. Loaded by {@link #reloadModel()}.
+ */
+ private final List<UnknownTypeModel> unknownTypeModels = new ArrayList<>();
+
/** Stable node ids whose default expand state has been seeded into TreeMemory this session. */
private final Set<String> treeStateSeeded = new HashSet<>();
@@ -404,8 +433,14 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
SWT.DefaultSelection,
event -> {
TreeItem treeItem = tree.getSelection()[0];
- if (treeItem != null && FILE.equals(treeItem.getData(KEY_TYPE))) {
+ if (treeItem == null) {
+ return;
+ }
+ if (FILE.equals(treeItem.getData(KEY_TYPE))) {
onEditMetadata();
+ } else if (UNKNOWN_FILE.equals(treeItem.getData(KEY_TYPE))) {
+ // There's no editor for an element we can't load: explain why instead.
+ onUnknownMetadataDetails();
}
});
@@ -473,6 +508,10 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
GuiResource.getInstance().getImage("ui/images/search.svg", 16, 16));
menuItem.addListener(SWT.Selection, e -> onFindReferences());
+ menuItem = new MenuItem(menu, SWT.POP_UP);
+ menuItem.setText(BaseMessages.getString(PKG, "MetadataPerspective.Menu.OpenFile"));
+ menuItem.addListener(SWT.Selection, e -> onOpenMetadataFile());
+
new MenuItem(menu, SWT.SEPARATOR);
menuItem = new MenuItem(menu, SWT.POP_UP);
@@ -482,12 +521,31 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
new MenuItem(menu, SWT.SEPARATOR);
break;
+ case UNKNOWN_FILE:
+ // An element we can't load: all we can do is explain, show and remove it.
+ menuItem = new MenuItem(menu, SWT.POP_UP);
+ menuItem.setText(
+ BaseMessages.getString(PKG, "MetadataPerspective.Menu.ShowDetails"));
+ menuItem.setImage(GuiResource.getInstance().getImageInfo());
+ menuItem.addListener(SWT.Selection, e -> onUnknownMetadataDetails());
+
+ menuItem = new MenuItem(menu, SWT.POP_UP);
+ menuItem.setText(BaseMessages.getString(PKG, "MetadataPerspective.Menu.OpenFile"));
+ menuItem.addListener(SWT.Selection, e -> onOpenMetadataFile());
+
+ new MenuItem(menu, SWT.SEPARATOR);
+
+ menuItem = new MenuItem(menu, SWT.POP_UP);
+ menuItem.setText(BaseMessages.getString(PKG, "MetadataPerspective.Menu.Delete"));
+ menuItem.setImage(GuiResource.getInstance().getImageDelete());
+ menuItem.addListener(SWT.Selection, e -> onDeleteUnknownMetadata());
+ break;
default:
break;
}
- // Help is not meaningful on a category header.
- if (!CATEGORY.equals(treeItem.getData(KEY_TYPE))) {
+ // Help is not meaningful on a category header or on anything we can't load.
+ if (!CATEGORY.equals(treeItem.getData(KEY_TYPE)) && !isUnknownNode(treeItem)) {
menuItem = new MenuItem(menu, SWT.POP_UP);
menuItem.setText(BaseMessages.getString(PKG, "MetadataPerspective.Menu.Help"));
menuItem.setImage(GuiResource.getInstance().getImageHelp());
@@ -499,6 +557,12 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
appendPluginContextMenuItems(menu);
}
+ // Nodes without any action (an unknown category or type header) get no popup at all.
+ if (menu.getItemCount() == 0) {
+ menu.dispose();
+ return;
+ }
+
tree.setMenu(menu);
menu.setVisible(true);
}
@@ -526,6 +590,14 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
createTreeDropTarget(tree);
}
+ /** True for the "Unknown" category and everything below it: metadata we can't load. */
+ private static boolean isUnknownNode(TreeItem item) {
+ String nodeType = (String) item.getData(KEY_TYPE);
+ return UNKNOWN_CATEGORY.equals(nodeType)
+ || UNKNOWN_TYPE.equals(nodeType)
+ || UNKNOWN_FILE.equals(nodeType);
+ }
+
/**
* Returns the metadata type key (objectKey) for the given tree item by walking up to the root
* class item.
@@ -620,8 +692,9 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
}
TreeItem targetItem = (TreeItem) event.item;
String targetType = (String) targetItem.getData(KEY_TYPE);
- // Only allow drop on folder or on class root (MetadataItem); not on another file
- if (FILE.equals(targetType)) {
+ // Only allow drop on folder or on class root (MetadataItem); not on another file and
+ // not on the "Unknown" category, which isn't a place to move metadata to.
+ if (FILE.equals(targetType) || isUnknownNode(targetItem)) {
event.detail = DND.DROP_NONE;
event.feedback = DND.FEEDBACK_NONE;
return;
@@ -1286,6 +1359,10 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
TreeItem item = tree.getSelection()[0];
if (item != null) {
if (item.getParentItem() == null) return;
+ // Elements we can't load have no metadata object to rename.
+ if (isUnknownNode(item)) {
+ return;
+ }
String objectKey = (String) item.getParentItem().getData();
String objectName = item.getText(0);
@@ -1816,6 +1893,11 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
onDeleteFolder();
return;
}
+ // Elements we can't load are deleted as plain files: there's no object to load or reference.
+ if (UNKNOWN_FILE.equals(treeItem.getData(KEY_TYPE))) {
+ onDeleteUnknownMetadata();
+ return;
+ }
if (!FILE.equals(treeItem.getData(KEY_TYPE))) {
return;
}
@@ -1881,6 +1963,160 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
}
}
+ /**
+ * Deletes the selected element we can't load. There is no metadata object to load, so no
+ * references can be looked up either: after confirmation we simply delete the JSON file.
+ */
+ public void onDeleteUnknownMetadata() {
+ if (tree.getSelectionCount() != 1) {
+ return;
+ }
+ TreeItem treeItem = tree.getSelection()[0];
+ if (!UNKNOWN_FILE.equals(treeItem.getData(KEY_TYPE))) {
+ return;
+ }
+ String name = treeItem.getText(0);
+ String filename = (String) treeItem.getData(KEY_FILENAME);
+ if (Utils.isEmpty(filename)) {
+ showFileNotFound(name);
+ return;
+ }
+
+ MessageBox confirmBox = new MessageBox(getShell(), SWT.ICON_QUESTION | SWT.YES | SWT.NO);
+ confirmBox.setText(BaseMessages.getString(PKG, "MetadataPerspective.DeleteMetadata.Title"));
+ confirmBox.setMessage(
+ BaseMessages.getString(
+ PKG,
+ "MetadataPerspective.DeleteUnknownMetadata.Confirm",
+ name,
+ toDisplayPath(filename, hopGui.getVariables().resolve(Const.VAR_PROJECT_HOME))));
+ if ((confirmBox.open() & SWT.YES) == 0) {
+ return;
+ }
+
+ try {
+ FileObject file = HopVfs.getFileObject(filename);
+ FileObject typeFolder = file.getParent();
+ if (!file.delete()) {
+ throw new HopException("The file couldn't be deleted");
+ }
+ deleteEmptyOrphanFolder((String) treeItem.getData(), typeFolder);
+
+ refresh();
+ updateSelection();
+ hopGui.getEventsHandler().fire(HopGuiEvents.MetadataDeleted.name());
+ } catch (Exception e) {
+ new ErrorDialog(
+ getShell(),
+ ERROR,
+ BaseMessages.getString(
+ PKG, "MetadataPerspective.DeleteUnknownMetadata.Error.Message", filename),
+ e);
+ }
+ }
+
+ /**
+ * Removes the metadata folder of a type no plugin provides once its last element is deleted, so
+ * cleaning up after a missing plugin doesn't leave an empty folder behind. Folders of known types
+ * are left alone: they are in use. Failing to clean up is logged, never shown.
+ *
+ * @param typeKey the metadata type key (the folder name)
+ * @param typeFolder the folder holding the elements of that type
+ */
+ private void deleteEmptyOrphanFolder(String typeKey, FileObject typeFolder) {
+ if (PluginRegistry.getInstance().findPluginWithId(MetadataPluginType.class, typeKey) != null) {
+ return;
+ }
+ try {
+ if (typeFolder.exists() && typeFolder.getChildren().length == 0) {
+ typeFolder.delete();
+ }
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error deleting empty metadata folder " + typeFolder.getName(), e);
+ }
+ }
+
+ /** Tells the user we don't know which file holds a metadata element. */
+ private void showFileNotFound(String name) {
+ MessageBox box = new MessageBox(getShell(), SWT.OK | SWT.ICON_WARNING);
+ box.setText(BaseMessages.getString(PKG, "MetadataPerspective.OpenFile.NotFound.Header"));
+ box.setMessage(
+ BaseMessages.getString(PKG, "MetadataPerspective.OpenFile.NotFound.Message", name));
+ box.open();
+ }
+
+ /** Shows the file and the error behind the selected element we can't load. */
+ public void onUnknownMetadataDetails() {
+ if (tree.getSelectionCount() != 1) {
+ return;
+ }
+ TreeItem treeItem = tree.getSelection()[0];
+ if (!UNKNOWN_FILE.equals(treeItem.getData(KEY_TYPE))) {
+ return;
+ }
+ String name = treeItem.getText(0);
+ String filename = Const.NVL((String) treeItem.getData(KEY_FILENAME), "");
+ String reason = Const.NVL((String) treeItem.getData(KEY_REASON), "");
+
+ new DetailsDialog(
+ getShell(),
+ BaseMessages.getString(PKG, "MetadataPerspective.UnknownDetails.Title", name),
+ GuiResource.getInstance().getImageHop(),
+ BaseMessages.getString(PKG, "MetadataPerspective.UnknownDetails.Message", name),
+ BaseMessages.getString(PKG, "MetadataPerspective.UnknownDetails.File")
+ + Const.CR
+ + toDisplayPath(filename, hopGui.getVariables().resolve(Const.VAR_PROJECT_HOME))
+ + Const.CR
+ + Const.CR
+ + BaseMessages.getString(PKG, "MetadataPerspective.UnknownDetails.Error")
+ + Const.CR
+ + reason)
+ .open();
+ }
+
+ /**
+ * Opens the JSON file behind the selected metadata element in a text editor. Works for regular
+ * elements as well as for the ones we can't load, which is often the only way to see what a
+ * broken element contains.
+ */
+ public void onOpenMetadataFile() {
+ if (tree.getSelectionCount() != 1) {
+ return;
+ }
+ TreeItem treeItem = tree.getSelection()[0];
+ String nodeType = (String) treeItem.getData(KEY_TYPE);
+
+ String filename;
+ if (UNKNOWN_FILE.equals(nodeType)) {
+ filename = (String) treeItem.getData(KEY_FILENAME);
+ } else if (FILE.equals(nodeType)) {
+ filename = findMetadataFilename(getObjectKey(treeItem), treeItem.getText(0));
+ } else {
+ return;
+ }
+
+ if (Utils.isEmpty(filename)) {
+ showFileNotFound(treeItem.getText(0));
+ return;
+ }
+
+ try {
+ if (hopGui.fileDelegate.fileOpen(filename) != null) {
+ // The file opens in a tab of the Explorer perspective, so let's go there. Opening a file
+ // only switches perspective by itself for the file types a perspective claims as its own
+ // and a JSON file isn't one of those.
+ //
+ ExplorerPerspective.getInstance().activate();
+ }
+ } catch (Exception e) {
+ new ErrorDialog(
+ getShell(),
+ ERROR,
+ BaseMessages.getString(PKG, "MetadataPerspective.OpenFile.Error.Message", filename),
+ e);
+ }
+ }
+
/**
* Replaces the resolved {@code projectHome} prefix in {@code path} with {@code ${PROJECT_HOME}}
* so displayed paths are project-relative and shorter.
@@ -2154,6 +2390,10 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
*/
private void reloadModel() {
typeModels.clear();
+ unknownTypeModels.clear();
+ // Metadata types and elements we can't load, keyed by metadata type, filled in below.
+ Map<String, UnknownTypeModel> unknownByKey = new LinkedHashMap<>();
+ Set<String> knownKeys = new HashSet<>();
try {
IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
for (Class<IHopMetadata> metadataClass : metadataProvider.getMetadataClasses()) {
@@ -2170,6 +2410,8 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
annotation.image(),
metadataClass);
+ knownKeys.add(annotation.key());
+
IHopMetadataSerializer<IHopMetadata> serializer =
metadataProvider.getSerializer(metadataClass);
List<String> names = serializer.listObjectNames();
@@ -2178,15 +2420,37 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
String virtualPath;
try {
virtualPath = Const.NVL(serializer.load(name).getVirtualPath(), "");
- } catch (HopException e) {
- // Ignore missing/corrupt metadata items
- LogChannel.GENERAL.logError("Error loading metadata object:" + name);
+ } catch (Exception e) {
+ // We can't load this one: a missing plugin, corrupt JSON, ... . Rather than hiding it,
+ // we list it under the "Unknown" category where it can be inspected and deleted.
+ //
+ LogChannel.GENERAL.logError("Error loading metadata object:" + name, e);
+ unknownByKey
+ .computeIfAbsent(
+ annotation.key(), key -> new UnknownTypeModel(key, typeModel.typeName))
+ .items
+ .add(
+ new UnknownItemModel(
+ name,
+ findMetadataFilename(annotation.key(), name),
+ Const.getRootCauseMessage(e)));
continue;
}
typeModel.items.add(new MetadataItemModel(name, virtualPath));
}
typeModels.add(typeModel);
}
+
+ // Metadata folders for which no plugin provides the type at all: without a metadata class
+ // there's no serializer, so these are invisible unless we go and look for them ourselves.
+ //
+ collectOrphanMetadataFolders(metadataProvider, knownKeys, unknownByKey);
+
+ unknownTypeModels.addAll(unknownByKey.values());
+ unknownTypeModels.sort(Comparator.comparing(unknownType -> unknownType.typeName));
+ for (UnknownTypeModel unknownType : unknownTypeModels) {
+ unknownType.items.sort(Comparator.comparing(item -> item.name));
+ }
// Attach explicitly-created (persisted) virtual folders so empty ones survive
// refresh/restart.
loadPersistedFolders();
@@ -2201,6 +2465,82 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
}
}
+ /**
+ * Looks for metadata folders in the project which no installed plugin claims: every JSON file in
+ * such a folder is an element we can't load, so they are collected as unknown elements of an
+ * unknown type. Anything we can't read here is logged and skipped: this is a best-effort scan.
+ *
+ * @param metadataProvider the provider(s) of the project
+ * @param knownKeys the metadata type keys provided by the installed plugins
+ * @param unknownByKey collects the unknown elements, keyed by metadata type
+ */
+ private void collectOrphanMetadataFolders(
+ IHopMetadataProvider metadataProvider,
+ Set<String> knownKeys,
+ Map<String, UnknownTypeModel> unknownByKey) {
+ for (JsonMetadataProvider jsonProvider : getJsonProviders(metadataProvider)) {
+ try {
+ FileObject baseFolder = HopVfs.getFileObject(jsonProvider.getBaseFolder());
+ if (!baseFolder.exists()) {
+ continue;
+ }
+ for (FileObject typeFolder : baseFolder.getChildren()) {
+ String key = typeFolder.getName().getBaseName();
+ if (!typeFolder.isFolder() || knownKeys.contains(key)) {
+ continue;
+ }
+ String reason =
+ BaseMessages.getString(PKG, "MetadataPerspective.Unknown.NoPluginForType", key);
+ UnknownTypeModel unknownType =
+ unknownByKey.computeIfAbsent(key, k -> new UnknownTypeModel(k, k));
+ for (FileObject jsonFile : HopVfs.findFiles(typeFolder, "json", false)) {
+ String name = jsonFile.getName().getBaseName().replaceAll("\\.json$", "");
+ // The same element can live in a parent project as well: like anywhere else the first
+ // provider which has it wins, so we don't list it twice.
+ if (unknownType.items.stream().noneMatch(item -> item.name.equals(name))) {
+ unknownType.items.add(
+ new UnknownItemModel(name, HopVfs.getFilename(jsonFile), reason));
+ }
+ }
+ }
+ } catch (Exception e) {
+ LogChannel.UI.logError(
+ "Error looking for unknown metadata in folder " + jsonProvider.getBaseFolder(), e);
+ }
+ }
+ }
+
+ /** The JSON (file based) providers behind the given provider, which can be a multi-provider. */
+ private static List<JsonMetadataProvider> getJsonProviders(IHopMetadataProvider provider) {
+ List<JsonMetadataProvider> jsonProviders = new ArrayList<>();
+ if (provider instanceof MultiMetadataProvider multiProvider) {
+ for (IHopMetadataProvider childProvider : multiProvider.getProviders()) {
+ jsonProviders.addAll(getJsonProviders(childProvider));
+ }
+ } else if (provider instanceof JsonMetadataProvider jsonProvider) {
+ jsonProviders.add(jsonProvider);
+ }
+ return jsonProviders;
+ }
+
+ /**
+ * The file behind a metadata element: {@code <base folder>/<type key>/<name>.json} in the first
+ * provider which has it. Returns null if no file was found (or the metadata isn't file based).
+ */
+ private String findMetadataFilename(String typeKey, String name) {
+ for (JsonMetadataProvider jsonProvider : getJsonProviders(hopGui.getMetadataProvider())) {
+ String filename = jsonProvider.getBaseFolder() + "/" + typeKey + "/" + name + ".json";
+ try {
+ if (HopVfs.fileExists(filename)) {
+ return filename;
+ }
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error checking metadata file " + filename, e);
+ }
+ }
+ return null;
+ }
+
/**
* Reads the project's persisted virtual folders from the audit trail and attaches each path to
* its metadata type model, so explicitly-created (and possibly empty) folders are rendered.
@@ -2446,6 +2786,10 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
}
}
+ // Everything we can't load goes into its own category at the bottom of the tree.
+ //
+ totalMatches += buildUnknownCategory(filtering);
+
updateResultCount(filtering, totalMatches);
TreeUtil.setOptimalWidthOnColumns(tree);
@@ -2494,6 +2838,15 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
+ "\t"
+ item.getText(0);
}
+ if (UNKNOWN_CATEGORY.equals(type)) {
+ return "UC";
+ }
+ if (UNKNOWN_TYPE.equals(type)) {
+ return "UT\t" + item.getData();
+ }
+ if (UNKNOWN_FILE.equals(type)) {
+ return "UI\t" + item.getData() + "\t" + item.getText(0);
+ }
return null;
}
@@ -2603,6 +2956,73 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
}
}
+ /**
+ * Builds the "Unknown" category at the bottom of the tree: the metadata elements in the project
+ * which we can't load, grouped by their metadata type. Nothing is rendered when there are none.
+ *
+ * @param filtering true if a search filter is active
+ * @return the number of elements shown (to be counted as search matches)
+ */
+ private int buildUnknownCategory(boolean filtering) {
+ if (unknownTypeModels.isEmpty()) {
+ return 0;
+ }
+ int shownCount = 0;
+ TreeItem categoryItem = null;
+
+ for (UnknownTypeModel unknownType : unknownTypeModels) {
+ boolean typeMatches =
+ filtering && (contains(unknownType.typeName) || contains(unknownType.key));
+ List<UnknownItemModel> shownItems;
+ if (!filtering || typeMatches) {
+ shownItems = unknownType.items;
+ } else {
+ shownItems = new ArrayList<>();
+ for (UnknownItemModel item : unknownType.items) {
+ if (contains(item.name)) {
+ shownItems.add(item);
+ }
+ }
+ }
+ if (filtering && shownItems.isEmpty()) {
+ continue;
+ }
+ shownCount += shownItems.size();
+
+ if (categoryItem == null) {
+ categoryItem = new TreeItem(tree, SWT.NONE);
+ categoryItem.setText(
+ BaseMessages.getString(PKG, "MetadataPerspective.Category.Unknown.Label"));
+ categoryItem.setImage(
+ GuiResource.getInstance()
+ .getImage(IMAGE_UNKNOWN, ConstUi.SMALL_ICON_SIZE, ConstUi.SMALL_ICON_SIZE));
+ categoryItem.setData(UNKNOWN_CATEGORY);
+ categoryItem.setData(KEY_TYPE, UNKNOWN_CATEGORY);
+ categoryItem.setData(VIRTUAL_PATH, "");
+ }
+
+ TreeItem typeItem = new TreeItem(categoryItem, SWT.NONE);
+ typeItem.setText(0, unknownType.typeName + " (" + shownItems.size() + ")");
+ typeItem.setImage(
+ GuiResource.getInstance()
+ .getImage(IMAGE_UNKNOWN, ConstUi.SMALL_ICON_SIZE, ConstUi.SMALL_ICON_SIZE));
+ typeItem.setData(unknownType.key);
+ typeItem.setData(KEY_TYPE, UNKNOWN_TYPE);
+ typeItem.setData(VIRTUAL_PATH, "");
+
+ for (UnknownItemModel item : shownItems) {
+ TreeItem fileItem = new TreeItem(typeItem, SWT.NONE);
+ fileItem.setText(0, item.name);
+ fileItem.setData(unknownType.key);
+ fileItem.setData(KEY_TYPE, UNKNOWN_FILE);
+ fileItem.setData(VIRTUAL_PATH, "");
+ fileItem.setData(KEY_FILENAME, item.filename);
+ fileItem.setData(KEY_REASON, item.reason);
+ }
+ }
+ return shownCount;
+ }
+
/**
* Ensures the folder chain for {@code virtualPath} exists under {@code typeItem}, creating folder
* nodes as needed, and returns the deepest folder node (or {@code typeItem} for an empty path).
@@ -2655,6 +3075,12 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
if (FOLDER.equals(type)) {
return new String[] {"F", (String) item.getData(), (String) item.getData(VIRTUAL_PATH)};
}
+ if (UNKNOWN_CATEGORY.equals(type)) {
+ return new String[] {"UC"};
+ }
+ if (UNKNOWN_TYPE.equals(type)) {
+ return new String[] {"UT", (String) item.getData()};
+ }
return null;
}
@@ -2672,9 +3098,10 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
// While searching, expand everything so matches are visible (not recorded as a choice).
item.setExpanded(true);
} else {
- // Categories expand by default; types and folders collapse by default. Seed each
- // default-expanded node once per session so the default holds until the user changes it.
- boolean defaultExpanded = "C".equals(path[0]);
+ // Categories (including the "Unknown" one) expand by default; types and folders collapse
+ // by default. Seed each default-expanded node once per session so the default holds until
+ // the user changes it.
+ boolean defaultExpanded = "C".equals(path[0]) || "UC".equals(path[0]);
if (defaultExpanded && treeStateSeeded.add(String.join(" ", path))) {
TreeMemory.getInstance().storeExpanded(METADATA_PERSPECTIVE_TREE, path, true);
}
@@ -2759,6 +3186,7 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
boolean isMetadataSelected = false;
boolean isFolderSelected = false;
+ boolean isUnknownSelected = false;
boolean canCreateHere = false;
if (tree.getSelectionCount() > 0) {
@@ -2766,6 +3194,8 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
String nodeType = (String) treeItem.getData(KEY_TYPE);
isMetadataSelected = FILE.equals(nodeType);
isFolderSelected = FOLDER.equals(nodeType);
+ // An element we can't load can only be deleted.
+ isUnknownSelected = UNKNOWN_FILE.equals(nodeType);
// The context "New" applies to a type, folder or file (all resolve to a type key), but not
// to a category header or a plain label.
canCreateHere = getObjectKey(treeItem) != null;
@@ -2775,7 +3205,8 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
toolBarWidgets.enableToolbarItem(TOOLBAR_ITEM_EDIT, isMetadataSelected);
toolBarWidgets.enableToolbarItem(TOOLBAR_ITEM_RENAME, isMetadataSelected);
toolBarWidgets.enableToolbarItem(TOOLBAR_ITEM_DUPLICATE, isMetadataSelected);
- toolBarWidgets.enableToolbarItem(TOOLBAR_ITEM_DELETE, isMetadataSelected || isFolderSelected);
+ toolBarWidgets.enableToolbarItem(
+ TOOLBAR_ITEM_DELETE, isMetadataSelected || isFolderSelected || isUnknownSelected);
}
@Override
@@ -3033,4 +3464,33 @@ public class MetadataPerspective implements IHopPerspective, TabClosable, IMetad
this.virtualPath = virtualPath;
}
}
+
+ /**
+ * One metadata folder on disk holding elements we can't load: either no installed plugin provides
+ * the type at all (there's not even a serializer for it, and {@code typeName} is the folder name)
+ * or the type is known but some of its elements fail to load.
+ */
+ private static final class UnknownTypeModel {
+ private final String key;
+ private final String typeName;
+ private final List<UnknownItemModel> items = new ArrayList<>();
+
+ private UnknownTypeModel(String key, String typeName) {
+ this.key = key;
+ this.typeName = typeName;
+ }
+ }
+
+ /** A single metadata file we can't load: its name, where it lives and what's wrong with it. */
+ private static final class UnknownItemModel {
+ private final String name;
+ private final String filename;
+ private final String reason;
+
+ private UnknownItemModel(String name, String filename, String reason) {
+ this.name = name;
+ this.filename = filename;
+ this.reason = reason;
+ }
+ }
}
diff --git a/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/metadata/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/metadata/messages/messages_en_US.properties
index 9f30cd3102..a527d32b15 100644
--- a/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/metadata/messages/messages_en_US.properties
+++ b/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/metadata/messages/messages_en_US.properties
@@ -99,4 +99,17 @@ MetadataPerspective.Category.Variables=Variables
MetadataPerspective.Category.Other=Other
MetadataPerspective.Overview.Title=Metadata
MetadataPerspective.Overview.Subtitle=Select a type to create a new item, or pick an existing item from the tree on the left.
-MetadataPerspective.Overview.New.Button=New
\ No newline at end of file
+MetadataPerspective.Overview.New.Button=New
+MetadataPerspective.Category.Unknown.Label=Unknown
+MetadataPerspective.Menu.OpenFile=Open JSON file
+MetadataPerspective.Menu.ShowDetails=Show details
+MetadataPerspective.Unknown.NoPluginForType=There is no plugin installed which handles metadata type ''{0}''.
+MetadataPerspective.UnknownDetails.Title=Unknown metadata: {0}
+MetadataPerspective.UnknownDetails.Message=Metadata element ''{0}'' can''t be loaded and is ignored. This usually means that a plugin is missing.
+MetadataPerspective.UnknownDetails.File=File:
+MetadataPerspective.UnknownDetails.Error=Reason:
+MetadataPerspective.DeleteUnknownMetadata.Confirm=Are you sure you want to delete ''{0}''?\n\nThis removes file {1} from the project.
+MetadataPerspective.DeleteUnknownMetadata.Error.Message=Error deleting file {0}
+MetadataPerspective.OpenFile.NotFound.Header=File not found
+MetadataPerspective.OpenFile.NotFound.Message=The JSON file of metadata element ''{0}'' could not be found. It may be stored somewhere else than in a metadata folder.
+MetadataPerspective.OpenFile.Error.Message=Error opening file {0}
\ No newline at end of file