You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/03/24 16:20:24 UTC
(hop) branch main updated: Issues #1882 (Meta inject XML cleanup, HopMetadataInjector rewrite) (#6849)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 4c51db7edc Issues #1882 (Meta inject XML cleanup, HopMetadataInjector rewrite) (#6849)
4c51db7edc is described below
commit 4c51db7edcf96a9f80ed2752b90e59ab580781b5
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Tue Mar 24 17:20:17 2026 +0100
Issues #1882 (Meta inject XML cleanup, HopMetadataInjector rewrite) (#6849)
* Issues #1882 (Meta inject XML cleanup, HopMetadataInjector rewrite)
* Issues #1882 (checkstyle)
* Issues #1882 (warning fixes)
* Issues #1882 (checkstyle)
* Issues #1882 (UDJE IT test fix)
* Issues #1882 (Meta Inject IT test)
* Issues #1882 (Developer docs)
* Issues #1882 (Developer docs + deprecation of unused annotation property)
---
core/src/main/java/org/apache/hop/core/Const.java | 21 +
.../hop/metadata/api/HopMetadataProperty.java | 14 +-
.../hop/metadata/inject/HopMetadataInjector.java | 574 +++++++++++++++
.../xml/MetadataPropertyKeyFunction.java | 2 +-
.../metadata/serializer/xml/XmlMetadataUtil.java | 379 ++++++----
.../metadata/inject/HopMetadataInjectorTest.java | 218 ++++++
.../hop/metadata/inject/company/Company.java} | 28 +-
.../hop/metadata/inject/company/Employee.java} | 30 +-
.../hop/metadata/inject/sample/Additional.java} | 29 +-
.../apache/hop/metadata/inject/sample/Field.java | 57 ++
.../hop/metadata/inject/sample/SampleMeta.java | 64 ++
.../hop/metadata/inject/sample/SampleType.java} | 28 +-
.../serializer/xml/XmlMetadataUtilTest.java | 30 +
.../metadata/serializer/xml/classes/WithMap.java | 105 +++
docs/hop-dev-manual/modules/ROOT/nav.adoc | 2 +
.../modules/ROOT/pages/metadata-injection.adoc | 61 ++
.../modules/ROOT/pages/metadata-serialization.adoc | 402 +++++++++++
.../azure-key-vault-variable-resolver.adoc | 4 +-
.../hop/core/injection/bean/BeanInjector.java | 11 +-
.../hop/core/injection/bean/BeanLevelInfo.java | 2 +-
integration-tests/mdi/0004-csv-input-parent.hpl | 468 ++++++------
integration-tests/mdi/0031-get-data-from-xml.hpl | 3 +-
.../mdi/0042-user-defined-java-class.hpl | 199 ++----
.../mdi/0042-user-defined-java-expression.hpl | 157 ++---
.../mdi/0045-meta-inject-template.hpl | 69 ++
integration-tests/mdi/0045-meta-inject-valiate.hpl | 561 +++++++++++++++
...defined-java-class.hpl => 0045-meta-inject.hpl} | 559 ++++++---------
.../mdi/datasets/golden-meta-inject-fields.csv | 3 +
.../mdi/datasets/golden-meta-inject-mappings.csv | 39 +
.../mdi/datasets/golden-meta-inject.csv | 2 +
.../mdi/files/excelwriter-mdi-test.xlsx | Bin 3311 -> 3311 bytes
integration-tests/mdi/files/janino.xml | 2 +-
integration-tests/mdi/files/meta-inject.xml | 312 ++++++++
.../mdi/main-0042-user-defined-java-expression.hwf | 57 ++
integration-tests/mdi/main-0045-meta-inject.hwf | 113 +++
integration-tests/mdi/main-0087-javascript.hwf | 58 --
.../dataset/golden-meta-inject--fields.json | 40 ++
.../dataset/golden-meta-inject-mappings.json | 48 ++
.../mdi/metadata/dataset/golden-meta-inject.json | 80 +++
.../unit-test/0045-meta-inject-valiate UNIT.json | 129 ++++
.../hop/pipeline/transforms/abort/AbortMeta.java | 2 +-
.../calculator/CalculatorMetaFunction.java | 13 +
.../transforms/datagrid/DataGridDataMeta.java | 10 +-
.../transforms/denormaliser/DenormaliserMeta.java | 4 +
.../denormaliser/DenormaliserMetaTest.java} | 32 +-
.../transforms/filterrows/FilterRowsMeta.java | 4 +
plugins/transforms/getvariable/pom.xml | 8 +
.../transforms/getvariable/GetVariableMeta.java | 20 +-
.../hop/pipeline/transforms/janino/JaninoMeta.java | 6 -
.../UserDefinedJavaClassDialog.java | 4 +-
.../UserDefinedJavaClassMeta.java | 15 +-
.../transforms/jsoninput/JsonInputMeta.java | 4 -
.../transforms/jsonoutput/JsonOutputField.java | 2 -
.../transforms/jsonoutput/JsonOutputMetaTest.java | 12 +
.../transforms/mergejoin/MergeJoinMetaTest.java | 12 +
.../pipeline/transforms/metainject/MetaInject.java | 785 +++++++++++++--------
.../transforms/metainject/MetaInjectData.java | 2 +
.../transforms/metainject/MetaInjectDialog.java | 251 +++----
.../transforms/metainject/MetaInjectMapping.java | 101 +--
.../transforms/metainject/MetaInjectMeta.java | 516 +++-----------
.../transforms/metainject/MetaInjectMigration.java | 41 --
.../metainject/MetaInjectOutputField.java | 80 +--
.../metainject/SourceTransformField.java | 83 ---
.../metainject/TargetTransformAttribute.java | 90 ---
.../metainject/messages/messages_en_US.properties | 13 +-
.../metainject/MetaInjectMetaInjectionTest.java | 64 --
.../metainject/MetaInjectMetaLoadSaveTest.java | 193 -----
.../transforms/metainject/MetaInjectMetaTest.java | 200 +++---
.../transforms/metainject/MetaInjectTest.java | 116 ---
.../metainject/src/test/resources/meta-inject.xml | 312 ++++++++
.../transforms/normaliser/NormaliserMeta.java | 32 +-
.../transforms/rowgenerator/RowGeneratorMeta.java | 104 +--
.../hop/pipeline/transforms/csvinput/CsvInput.java | 6 +-
.../transforms/csvinput/CsvInputDialog.java | 8 +-
.../transforms/csvinput/CsvInputField.java | 452 ++++++++++++
.../pipeline/transforms/csvinput/CsvInputMeta.java | 138 +---
.../transforms/csvinput/BaseCsvParsingTest.java | 3 +-
.../csvinput/CsvInputContentParsingTest.java | 37 +-
.../csvinput/CsvInputDoubleLineEndTest.java | 3 +-
.../transforms/csvinput/CsvInputEnclosureTest.java | 3 +-
.../transforms/csvinput/CsvInputMetaTest.java | 17 +-
.../csvinput/CsvInputMultiCharDelimiterTest.java | 3 +-
.../transforms/csvinput/CsvInputRowNumberTest.java | 3 +-
.../pipeline/transforms/csvinput/CsvInputTest.java | 5 +-
.../transforms/csvinput/CsvInputUnicodeTest.java | 3 +-
.../transforms/csvinput/CsvInputUnitTestBase.java | 11 +-
.../csvinput/CsvProcessRowInParallelTest.java | 3 +-
.../transforms/xml/getxmldata/GetXmlDataMeta.java | 2 +-
.../transforms/xml/getxmldata/GetXmlFileItem.java | 1 +
.../xml/getxmldata/GetXmlDataMetaTest.java | 13 +
90 files changed, 5696 insertions(+), 3106 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Const.java b/core/src/main/java/org/apache/hop/core/Const.java
index 018276cd4e..4157e24118 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -1888,6 +1888,27 @@ public class Const {
return source;
}
+ /**
+ * Implements Oracle style NVL function. The first value of the ones provided, that isn't null or
+ * empty is returned.
+ *
+ * @param values The values to test for null/empty.
+ * @return null if no arguments given or null. Otherwise, returns the first value of the ones
+ * provided, that isn't null or empty is returned.
+ */
+ public static String coalesce(String... values) {
+ if (values == null || values.length == 0) {
+ return null;
+ }
+ for (int i = 0; i < values.length - 1; i++) {
+ String value = values[i];
+ if (value != null && !value.isEmpty()) {
+ return value;
+ }
+ }
+ return values[values.length - 1];
+ }
+
/**
* Return empty string "" in case the given parameter is null, otherwise return the same value.
*
diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java
index 991da4dff7..c183bf37e4 100644
--- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java
+++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataProperty.java
@@ -28,7 +28,6 @@ import org.apache.hop.core.injection.InjectionTypeConverter;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD})
public @interface HopMetadataProperty {
-
/**
* The optional key to store this metadata property under. By the default the name of the field is
* taken.
@@ -71,6 +70,11 @@ public @interface HopMetadataProperty {
/** What is the name of the enum to pick in case the code in the metadata can't be found? * */
String enumNameWhenNotFound() default "";
+ /**
+ * @return Prevents the item from being serialized or inflated. Default value: false
+ */
+ boolean isExcludedFromSerialization() default false;
+
/**
* @return Prevents the item to be considered in injection. Default value: false
*/
@@ -102,7 +106,9 @@ public @interface HopMetadataProperty {
* A description of the field. Right now this is used only for metadata injection purposes
*
* @return The description of the property
+ * @deprecated Not used anywhere anymore.
*/
+ @Deprecated(since = "2.18.0")
String description() default "";
/**
@@ -126,7 +132,7 @@ public @interface HopMetadataProperty {
*
* <p>In this scenario we would specify the tags "key" and "value" to populate the list correctly.
*
- * @return
+ * @return The tags to put inline
*/
String[] inlineListTags() default {};
@@ -174,4 +180,8 @@ public @interface HopMetadataProperty {
* @return the keys to hide from serialization and metadata injection.
*/
String[] childKeysToIgnore() default {};
+
+ Class<?> mapKeyClass() default Object.class;
+
+ Class<?> mapValueClass() default Object.class;
}
diff --git a/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
new file mode 100644
index 0000000000..5467d7c00f
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/metadata/inject/HopMetadataInjector.java
@@ -0,0 +1,574 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.metadata.inject;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.injection.InjectionTypeConverter;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IEnumHasCode;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.api.IIntCodeConverter;
+import org.apache.hop.metadata.api.IStringObjectConverter;
+import org.apache.hop.metadata.serializer.xml.MetadataPropertyKeyFunction;
+import org.apache.hop.metadata.util.ReflectionUtil;
+
+public class HopMetadataInjector {
+ private HopMetadataInjector() {
+ // Hidden constructor
+ }
+
+ /**
+ * For the given object we retrieve the mappings between injection groups and their keys.
+ *
+ * @param objectClass The class to search
+ * @return The mapping giving you the set of keys for each injection group.
+ */
+ public static Map<String, Set<String>> findInjectionGroupKeys(Class<?> objectClass)
+ throws HopException {
+ if (objectClass == null) {
+ throw new HopException("The class to find injection group key mappings for is null");
+ }
+ Map<String, Set<String>> map = new HashMap<>();
+ findInjectionGroupKeys(objectClass, map);
+ return map;
+ }
+
+ /**
+ * For the given object we retrieve the mappings between injection keys and the group to which
+ * they belong.
+ *
+ * @param objectClass The class to search
+ * @param map the map to populate
+ */
+ public static void findInjectionGroupKeys(Class<?> objectClass, Map<String, Set<String>> map)
+ throws HopException {
+ try {
+ List<Field> fields =
+ ReflectionUtil.findAllFields(objectClass, new MetadataPropertyKeyFunction());
+ for (Field field : fields) {
+ Class<?> fieldClass = field.getType();
+ HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class);
+ if (property != null && !property.isExcludedFromInjection()) {
+ // Does this have an injection group key?
+ //
+ String injectionGroup = Const.NVL(property.injectionGroupKey(), property.groupKey());
+ if (StringUtils.isNotEmpty(injectionGroup)) {
+ if (fieldClass.equals(List.class)) {
+ // Search in the generic type of the List
+ ParameterizedType type = (ParameterizedType) field.getGenericType();
+ Class<?> listItemClass = (Class<?>) type.getActualTypeArguments()[0];
+ if (listItemClass.equals(String.class)) {
+ // We simply map the single injection key with the group key
+ //
+ map.put(injectionGroup, new HashSet<>(Set.of(property.injectionKey())));
+ } else {
+ findInjectionGroupKeys(listItemClass, injectionGroup, map);
+ }
+ }
+ } else if (StringUtils.isEmpty(property.injectionGroupKey())) {
+ // No injection key or group: we want to look further down.
+ // If the class is no primitive, we look into it.
+ //
+ findInjectionGroupKeys(fieldClass, map);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new HopException("Error injecting source data into object", e);
+ }
+ }
+
+ /**
+ * For the given class, find the metadata properties with injection keys below it.
+ *
+ * @param fieldClass The class to search for injection keys
+ * @param injectionGroup The parent injection group key
+ * @param map The map to add the keys to
+ */
+ private static void findInjectionGroupKeys(
+ Class<?> fieldClass, String injectionGroup, Map<String, Set<String>> map) {
+ List<Field> fields =
+ ReflectionUtil.findAllFields(fieldClass, new MetadataPropertyKeyFunction());
+ for (Field field : fields) {
+ HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class);
+ if (property != null) {
+ String injectionKey =
+ Const.coalesce(property.injectionKey(), property.key(), field.getName());
+ if (StringUtils.isNotEmpty(injectionKey)) {
+ // Add to the set for this injection group
+ Set<String> keySet = map.computeIfAbsent(injectionGroup, k -> new HashSet<>());
+ keySet.add(injectionKey);
+ } else {
+ // Perhaps this is a subclass and the properties are located deeper.
+ //
+ Class<?> childClass = field.getType();
+ findInjectionGroupKeys(childClass, injectionGroup, map);
+ }
+ }
+ }
+ }
+
+ /**
+ * Inject data into an object using HopMetadataProperty annotations on the fields in the class.
+ *
+ * @param metadataProvider The metadata provider to resolve injected object names.
+ * @param object The object to inject data into.
+ * @param injectionKeyMap A map between the injection key and the data where it should end up.
+ * @param groupKeyMap a mapping between the group key and the data for the underlying list items.
+ * The rows contain values where the names correspond to the injection keys.
+ * @throws HopException In case there was an error injecting data.
+ */
+ public static void inject(
+ IHopMetadataProvider metadataProvider,
+ Object object,
+ Map<String, Object> injectionKeyMap,
+ Map<String, RowBuffer> groupKeyMap)
+ throws HopException {
+ // We find all the with HopMetadataProperty annotated fields in the parent object.
+ // For these fields we see if we can find injection instructions.
+ // If this is the case, we inject the metadata.
+ //
+ if (object == null) {
+ return;
+ }
+ Class<?> objectClass = object.getClass();
+ try {
+ List<Field> fields =
+ ReflectionUtil.findAllFields(objectClass, new MetadataPropertyKeyFunction());
+ for (Field field : fields) {
+ injectField(metadataProvider, object, injectionKeyMap, groupKeyMap, field, objectClass);
+ }
+ } catch (Exception e) {
+ throw new HopException("Error injecting source data into object", e);
+ }
+ }
+
+ private static void injectField(
+ IHopMetadataProvider metadataProvider,
+ Object object,
+ Map<String, Object> injectionKeyMap,
+ Map<String, RowBuffer> groupKeyMap,
+ Field field,
+ Class<?> objectClass)
+ throws Exception {
+ HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class);
+ if (property != null && !property.isExcludedFromInjection()) {
+ Class<?> fieldClass = field.getType();
+
+ // Sometimes the injection key is identical to the key.
+ // This is some legacy or misunderstood functionality.
+ // The key is the same as the field if none is given.
+ String injectionKey =
+ Const.coalesce(property.injectionKey(), property.key(), field.getName());
+ String groupKey = Const.coalesce(property.injectionGroupKey(), property.groupKey());
+
+ boolean hasKey = StringUtils.isNotEmpty(injectionKey);
+ boolean hasKeyData = hasKey && injectionKeyMap.containsKey(injectionKey);
+ boolean hasGroupKey = StringUtils.isNotEmpty(groupKey);
+ boolean hasGroupData = hasGroupKey && groupKeyMap.containsKey(groupKey);
+
+ Object valueToSet =
+ StringUtils.isEmpty(injectionKey) ? null : injectionKeyMap.get(injectionKey);
+
+ if (hasKey && hasKeyData && !hasGroupData) {
+ injectValue(property, metadataProvider, object, field, valueToSet);
+ } else if (hasGroupData) {
+ injectList(metadataProvider, object, groupKeyMap, field, objectClass, groupKey);
+ } else if (!fieldClass.isEnum()
+ && !fieldClass.isPrimitive()
+ && !fieldClass.equals(String.class)) {
+ // Typically this is an inline object or one with a group element around it.
+ // The class is organized like that to facilitate "correct" XML generation and parsing.
+ // In this scenario we simply navigate into the object given by the field getter.
+ //
+ // We already know that there is no injection key set so we can't inject into primitives
+ // or enums at this point. The developer chose not to want this.
+ //
+ injectIntoInlineField(
+ metadataProvider, object, injectionKeyMap, groupKeyMap, field, objectClass);
+ }
+ }
+ }
+
+ private static void injectIntoInlineField(
+ IHopMetadataProvider metadataProvider,
+ Object object,
+ Map<String, Object> dataMap,
+ Map<String, RowBuffer> listItemData,
+ Field field,
+ Class<?> objectClass)
+ throws IllegalAccessException,
+ InvocationTargetException,
+ NoSuchMethodException,
+ HopException {
+ // Grab the value of this field and inject into this object
+ //
+ Object fieldObject = findGetter(objectClass, field).invoke(object);
+ if (fieldObject == null) {
+ throw new HopException(
+ "Make sure that field "
+ + field.getName()
+ + " in class "
+ + objectClass.getName()
+ + " has a value to inject into");
+ }
+ inject(metadataProvider, fieldObject, dataMap, listItemData);
+ }
+
+ private static void injectValue(
+ HopMetadataProperty property,
+ IHopMetadataProvider metadataProvider,
+ Object object,
+ Field field,
+ Object valueToSet)
+ throws Exception {
+ Object fieldValue = valueToSet;
+ // Is there a converter defined so that we know how to convert from String to integer?
+ //
+ Class<? extends IIntCodeConverter> intConverterClass = getIntConverter(property);
+ Class<? extends IStringObjectConverter> stringObjectConverterClass =
+ getStringObjectConverter(property);
+
+ if (intConverterClass != null && (valueToSet instanceof String string)) {
+ // There is an intConverter specified on the HopMetadataProperty.
+ // We can use it to convert the String to an integer.
+ // For example: String --> 2 (IValueMeta.TYPE_STRING)
+ //
+ fieldValue = convertStringWithIntConverter(string, intConverterClass);
+ } else if (field.getType().isEnum()
+ && property.storeWithCode()
+ && (valueToSet instanceof String code)) {
+ // This is an enum and we need to find its value with the code given.
+ // The enum implements IEnumHasCode
+ //
+ fieldValue = convertCodeToEnumeration(field, code);
+ } else if (field.getType().isEnum() && (fieldValue instanceof String name)) {
+ // A simple enum stored by its name.
+ //
+ fieldValue = convertEnumNameToValue(field, name);
+ } else if (property.storeWithName() && (valueToSet instanceof String name)) {
+ // A reference to a metadata element is stored with its name.
+ // We look it up using the metadata provider.
+ //
+ fieldValue = convertNameToMetadataElement(metadataProvider, field, name);
+ } else if (stringObjectConverterClass != null && (valueToSet instanceof String string)) {
+ // There is a specific string converter to use.
+ //
+ fieldValue = convertWithStringToObjectConverter(stringObjectConverterClass, field, string);
+ } else if (valueToSet != null) {
+ // We'll do some data type conversions for convenience.
+ // This allows simple String to Int or Long to int conversions don't cause any issues.
+ //
+ fieldValue = convertWithInjectionConverter(property, field, valueToSet, fieldValue);
+ }
+
+ // We need to set the data on the field
+ //
+ setValue(object, field, fieldValue);
+ }
+
+ private static Object convertWithStringToObjectConverter(
+ Class<? extends IStringObjectConverter> stringObjectConverterClass,
+ Field field,
+ String string)
+ throws HopException {
+ try {
+ IStringObjectConverter converter = stringObjectConverterClass.getConstructor().newInstance();
+ return converter.getObject(string);
+ } catch (Exception e) {
+ throw new HopException(
+ "Error converting string to object of field "
+ + field.getName()
+ + " of type "
+ + field.getType().getName()
+ + " using string: "
+ + string,
+ e);
+ }
+ }
+
+ private static Object convertEnumNameToValue(Field field, String name) throws HopException {
+ try {
+ Enum<?>[] enumConstants = (Enum<?>[]) field.getType().getEnumConstants();
+
+ // Backward compatibility: if the name is an integer, get the value at the given index.
+ //
+ int index = Const.toInt(name, -1);
+ if (index < 0 || index >= enumConstants.length) {
+ for (Enum<?> enumConstant : enumConstants) {
+ if (enumConstant.name().equals(name)) {
+ return enumConstant;
+ }
+ }
+ } else {
+ return enumConstants[index];
+ }
+ throw new HopException(
+ "Unrecognized enum name: " + name + " for class " + field.getType().getName());
+ } catch (Exception e) {
+ throw new HopException(
+ "Error converting enumeration with name '"
+ + name
+ + "' to its value for Enum class "
+ + field.getType().getName());
+ }
+ }
+
+ private static Object convertNameToMetadataElement(
+ IHopMetadataProvider metadataProvider, Field field, String name) throws HopException {
+ Object fieldValue;
+ Class<?> metadataClass = field.getType();
+ IHopMetadataSerializer<? extends IHopMetadata> serializer =
+ metadataProvider.getSerializer((Class<? extends IHopMetadata>) metadataClass);
+ fieldValue = serializer.load(name);
+ return fieldValue;
+ }
+
+ private static Object convertWithInjectionConverter(
+ HopMetadataProperty property, Field field, Object valueToSet, Object fieldValue)
+ throws InstantiationException,
+ IllegalAccessException,
+ InvocationTargetException,
+ NoSuchMethodException,
+ HopValueException {
+ Class<? extends InjectionTypeConverter> injectionConverter = property.injectionConverter();
+ InjectionTypeConverter converter = injectionConverter.getConstructor().newInstance();
+
+ Class<?> fieldClass = field.getType();
+ Class<?> valueClass = valueToSet.getClass();
+
+ // String to boolean
+ if (valueClass.equals(String.class)
+ && (fieldClass.equals(Boolean.class) || fieldClass.equals(boolean.class))) {
+ fieldValue = converter.string2booleanPrimitive((String) fieldValue);
+ } else
+ // String to integer
+ if (valueClass.equals(String.class)
+ && (fieldClass.equals(Integer.class) || fieldClass.equals(int.class))) {
+ fieldValue = converter.string2intPrimitive((String) fieldValue);
+ } else
+ // String to long
+ if (valueClass.equals(String.class)
+ && (fieldClass.equals(Long.class) || fieldClass.equals(long.class))) {
+ fieldValue = converter.string2longPrimitive((String) fieldValue);
+ } else
+ // Integer to String
+ //
+ if (fieldClass.equals(String.class) && (valueClass.equals(Integer.class))) {
+ fieldValue = Integer.toString((int) fieldValue);
+ } else
+ // long to int
+ if (fieldClass.equals(int.class) && (valueClass.equals(Long.class))) {
+ //noinspection ReassignedVariable
+ fieldValue = Long.valueOf((long) fieldValue).intValue();
+ } else
+ // int to long
+ if (fieldClass.equals(Long.class) && (valueClass.equals(int.class))) {
+ //noinspection UnnecessaryBoxing
+ fieldValue = Long.valueOf((int) fieldValue);
+ } else // anything to String
+ if (fieldClass.equals(String.class)) {
+ fieldValue = fieldValue.toString();
+ }
+ return fieldValue;
+ }
+
+ private static Class<? extends IIntCodeConverter> getIntConverter(HopMetadataProperty property) {
+ Class<? extends IIntCodeConverter> converter = property.intCodeConverter();
+ if (converter.equals(IIntCodeConverter.None.class)) {
+ return null;
+ }
+ return converter;
+ }
+
+ private static Class<? extends IStringObjectConverter> getStringObjectConverter(
+ HopMetadataProperty property) {
+ Class<? extends IStringObjectConverter> converter = property.injectionStringObjectConverter();
+ if (converter.equals(IStringObjectConverter.None.class)) {
+ return null;
+ }
+ return converter;
+ }
+
+ private static Object convertCodeToEnumeration(Field field, String code) throws HopException {
+ Object fieldValue;
+ final Class<? extends IEnumHasCode> enumerationClass =
+ (Class<? extends IEnumHasCode>) field.getType();
+ fieldValue = IEnumHasCode.lookupCode(enumerationClass, code, null);
+ if (fieldValue == null) {
+ // For backward compatibility, retry with the name as well
+ //
+ try {
+ fieldValue = convertEnumNameToValue(field, code);
+ } catch (HopException e) {
+ throw new HopException(
+ "Enumeration code '"
+ + code
+ + "' was not recognized for class "
+ + enumerationClass.getName(),
+ e);
+ }
+ }
+ return fieldValue;
+ }
+
+ private static Object convertStringWithIntConverter(
+ String string, Class<? extends IIntCodeConverter> intConverterClass)
+ throws InstantiationException,
+ IllegalAccessException,
+ InvocationTargetException,
+ NoSuchMethodException {
+ Object fieldValue;
+ IIntCodeConverter converter = intConverterClass.getConstructor().newInstance();
+ fieldValue = converter.getType(string);
+ return fieldValue;
+ }
+
+ private static void injectList(
+ IHopMetadataProvider metadataProvider,
+ Object object,
+ Map<String, RowBuffer> listItemData,
+ Field field,
+ Class<?> objectClass,
+ String groupKey)
+ throws HopException {
+ try {
+ // What is the list?
+ //
+ List<Object> list = (List<Object>) findGetter(objectClass, field).invoke(object);
+ list.clear();
+
+ // Which class is being listed?
+ //
+ ParameterizedType type = (ParameterizedType) field.getGenericType();
+ Class<?> listItemClass = (Class<?>) type.getActualTypeArguments()[0];
+
+ // We're given a row buffer for the group specified
+ //
+ RowBuffer rowBuffer = listItemData.get(groupKey);
+ if (rowBuffer == null) {
+ rowBuffer = new RowBuffer();
+ }
+ // Loop over the rows in the buffer, then over the keys.
+ // The keys (and their types) are in the row metadata.
+ //
+ IRowMeta rowMeta = rowBuffer.getRowMeta();
+ for (Object[] row : rowBuffer.getBuffer()) {
+ // Is this a primitive type we're dealing with?
+ //
+ if (listItemClass.isPrimitive() || listItemClass.equals(String.class)) {
+ // The row contains a single value that we simply need to add to the list
+ //
+ list.add(row[0]);
+ } else {
+ // Create a new list item for every row.
+ //
+ Object listItemObject = listItemClass.getConstructor().newInstance();
+
+ // Add it to the list
+ list.add(listItemObject);
+
+ // Now set values on the list item object:
+ //
+ for (int valueIndex = 0; valueIndex < rowMeta.size(); valueIndex++) {
+ IValueMeta valueMeta = rowMeta.getValueMeta(valueIndex);
+ String valueKey = valueMeta.getName();
+ Object value = row[valueIndex];
+
+ // The map will contain just a single value matching the expected key in the underlying
+ // object
+ // class field annotation. Only that field will be set.
+ //
+ if (StringUtils.isEmpty(valueKey)) {
+ throw new HopException("No value key found for list item " + listItemClass.getName());
+ } else {
+ Map<String, Object> listItemValueMap = new HashMap<>();
+ listItemValueMap.put(valueKey, value);
+ inject(metadataProvider, listItemObject, listItemValueMap, Map.of());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new HopException(
+ "Error while injecting list items in class "
+ + object.getClass().getName()
+ + ", field: "
+ + field.getName(),
+ e);
+ }
+ }
+
+ private static void setValue(Object object, Field field, Object valueToSet) throws Exception {
+ try {
+ if (valueToSet == null) {
+ return;
+ }
+ Method setter = findSetter(object.getClass(), field);
+ setter.invoke(object, valueToSet);
+ } catch (Exception e) {
+ throw new HopException(
+ "Error while injecting value in class "
+ + object.getClass().getName()
+ + ", field "
+ + field.getName()
+ + " with value '"
+ + valueToSet
+ + " ("
+ + valueToSet.getClass()
+ + ")",
+ e);
+ }
+ }
+
+ private static Method findGetter(Class<?> clazz, Field field) throws NoSuchMethodException {
+ String name = field.getName();
+ String methodName;
+ if (field.getGenericType().equals(Boolean.class)
+ || field.getGenericType().equals(boolean.class)) {
+ methodName = "is" + StringUtils.capitalize(name);
+ } else {
+ methodName = "get" + StringUtils.capitalize(name);
+ }
+ return clazz.getMethod(methodName);
+ }
+
+ private static Method findSetter(Class<?> clazz, Field field) throws NoSuchMethodException {
+ String name = field.getName();
+ String methodName = "set" + StringUtils.capitalize(name);
+ return clazz.getMethod(methodName, field.getType());
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java b/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
index b0b9716e33..ea354e3bd7 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
+++ b/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
@@ -30,6 +30,6 @@ public class MetadataPropertyKeyFunction implements Function<Field, String> {
if (annotation == null) {
return null;
}
- return Const.NVL(annotation.groupKey(), Const.NVL(annotation.key(), field.getName()));
+ return Const.coalesce(annotation.groupKey(), annotation.key(), field.getName());
}
}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtil.java b/core/src/main/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtil.java
index 9079c3ca9b..30334bd4e9 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtil.java
+++ b/core/src/main/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtil.java
@@ -22,8 +22,10 @@ import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.ArrayList;
import java.util.Date;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.hop.core.Const;
@@ -212,7 +214,6 @@ public class XmlMetadataUtil {
xml.append(XmlHandler.addTagValue(tag, ((Enum) parentObject).name()));
}
} else if (parentObject instanceof java.util.List listItems) {
-
// Serialize a list of values
// Use the key on the annotation to open a new block
// Store the items in that block
@@ -230,7 +231,27 @@ public class XmlMetadataUtil {
if (StringUtils.isNotEmpty(groupKey)) {
xml.append(XmlHandler.closeTag(groupKey)).append(Const.CR);
}
-
+ } else //noinspection rawtypes
+ if (parentObject instanceof java.util.Map map) {
+ // For Map we loop over all the keys and get the values.
+ // We serialize this way:
+ // <groupKey>
+ // <key> key-and-value-elements </key>
+ // <key> key-and-value-elements </key>
+ // ...
+ // </groupKey>
+
+ xml.append(XmlHandler.openTag(parentProperty.groupKey())).append(Const.CR);
+ Set<Map.Entry<Object, Object>> entrySet = map.entrySet();
+ for (Map.Entry<Object, Object> entry : entrySet) {
+ xml.append(XmlHandler.openTag(parentProperty.key())).append(Const.CR);
+ Object keyObject = entry.getKey();
+ Object valueObject = entry.getValue();
+ xml.append(serializeObjectToXml(keyObject, parentProperty, groupKey, tag));
+ xml.append(serializeObjectToXml(valueObject, parentProperty, groupKey, tag));
+ xml.append(XmlHandler.closeTag(parentProperty.key())).append(Const.CR);
+ }
+ xml.append(XmlHandler.closeTag(parentProperty.groupKey())).append(Const.CR);
} else {
// POJO : serialize to XML...
// We only take the fields of the POJO class that are annotated
@@ -502,13 +523,11 @@ public class XmlMetadataUtil {
throws HopXmlException {
String elementString = XmlHandler.getNodeValue(elementNode);
- boolean password = parentProperty.password();
boolean defaultBoolean = parentProperty.defaultBoolean();
boolean storeWithName = parentProperty.storeWithName();
boolean storeWithCode = parentProperty.storeWithCode();
String[] inlineListTags = parentProperty.inlineListTags();
Class<? extends IIntCodeConverter> intCodeConverterClass = parentProperty.intCodeConverter();
- Class<? extends IStringEncoder> stringEncoderClass = parentProperty.stringEncoder();
if (storeWithName) {
// No name: return null
@@ -533,46 +552,10 @@ public class XmlMetadataUtil {
// Convert to other data type?
//
if (fieldType.equals(String.class)) {
- if (elementNode != null) {
- if (password) {
- return Encr.decryptPasswordOptionallyEncrypted(elementString);
- }
- if (!EmptyStringEncoder.class.equals(stringEncoderClass)) {
- // Decode the encoded string
- //
- try {
- IStringEncoder encoder = stringEncoderClass.getConstructor().newInstance();
- return encoder.decode(elementString);
- } catch (Exception e) {
- throw new HopXmlException(
- "Error decoding string '"
- + elementString
- + "' with string encoder class "
- + stringEncoderClass.getName(),
- e);
- }
- } else {
- return elementString;
- }
- }
+ return deSerializeString(
+ elementString, parentProperty, elementNode, fieldType, metadataProvider);
} else if (fieldType.equals(Integer.class) || fieldType.equals(int.class)) {
- if (elementNode != null) {
- if (intCodeConverterClass.equals(IIntCodeConverter.None.class)) {
- return Integer.valueOf(elementString);
- } else {
- try {
- IIntCodeConverter converter = intCodeConverterClass.getConstructor().newInstance();
- return converter.getType(elementString);
- } catch (Exception e) {
- throw new HopXmlException(
- "Error converting String code "
- + elementString
- + " to integer using converter class "
- + intCodeConverterClass,
- e);
- }
- }
- }
+ return deSerializeInteger(elementNode, intCodeConverterClass, elementString);
} else if (fieldType.equals(Long.class) || fieldType.equals(long.class)) {
if (elementNode != null) {
return Long.valueOf(elementString);
@@ -592,113 +575,243 @@ public class XmlMetadataUtil {
return defaultBoolean;
}
} else if (fieldType.isEnum()) {
- final Class<? extends Enum> enumerationClass = (Class<? extends Enum>) field.getType();
- if (StringUtils.isNotEmpty(elementString)) {
- if (storeWithCode) {
- try {
- IEnumHasCode[] values = (IEnumHasCode[]) enumerationClass.getEnumConstants();
- for (IEnumHasCode value : values) {
- if (value.getCode().equals(elementString)) {
- return value;
- }
+ Object value = deSerializeEnum(field, elementString, storeWithCode);
+ if (value != null) return value;
+ } else if (fieldType.equals(java.util.List.class)) {
+ return deSerializeList(
+ parentObject, parentProperty, groupNode, tag, field, metadataProvider, inlineListTags);
+ } else if (fieldType.equals(java.util.Map.class)) {
+ return deSerializeMap(parentObject, parentProperty, groupNode, tag, metadataProvider);
+ } else {
+ // Load the metadata for this node...
+ //
+ return deSerializeFromXml(
+ parentObject, parentProperty, elementNode, fieldType, metadataProvider);
+ }
+
+ // No value found for the given arguments: return the default value
+ //
+ return null;
+ }
+
+ private static Object deSerializeString(
+ String elementString,
+ HopMetadataProperty parentProperty,
+ Node elementNode,
+ Class<?> fieldType,
+ IHopMetadataProvider metadataProvider)
+ throws HopXmlException {
+ if (elementNode != null) {
+ if (parentProperty.password()) {
+ return Encr.decryptPasswordOptionallyEncrypted(elementString);
+ }
+ if (!EmptyStringEncoder.class.equals(parentProperty.stringEncoder())) {
+ // Decode the encoded string
+ //
+ try {
+ IStringEncoder encoder = parentProperty.stringEncoder().getConstructor().newInstance();
+ return encoder.decode(elementString);
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Error decoding string '"
+ + elementString
+ + "' with string encoder class "
+ + parentProperty.stringEncoder().getName(),
+ e);
+ }
+ } else {
+ return elementString;
+ }
+ }
+ return null;
+ }
+
+ private static Integer deSerializeInteger(
+ Node elementNode,
+ Class<? extends IIntCodeConverter> intCodeConverterClass,
+ String elementString)
+ throws HopXmlException {
+ if (elementNode == null) {
+ return null;
+ }
+ if (intCodeConverterClass.equals(IIntCodeConverter.None.class)) {
+ return Integer.valueOf(elementString);
+ } else {
+ try {
+ IIntCodeConverter converter = intCodeConverterClass.getConstructor().newInstance();
+ return converter.getType(elementString);
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Error converting String code "
+ + elementString
+ + " to integer using converter class "
+ + intCodeConverterClass,
+ e);
+ }
+ }
+ }
+
+ private static Object deSerializeEnum(Field field, String elementString, boolean storeWithCode)
+ throws HopXmlException {
+ final Class<? extends Enum> enumerationClass = (Class<? extends Enum>) field.getType();
+ if (StringUtils.isNotEmpty(elementString)) {
+ if (storeWithCode) {
+ try {
+ IEnumHasCode[] values = (IEnumHasCode[]) enumerationClass.getEnumConstants();
+ for (IEnumHasCode value : values) {
+ if (value.getCode().equals(elementString)) {
+ return value;
}
- // Retry with the description for compatibility with older versions
- //
- if (enumerationClass.isInstance(IEnumHasCodeAndDescription.class)) {
- IEnumHasCodeAndDescription[] vals =
- (IEnumHasCodeAndDescription[]) enumerationClass.getEnumConstants();
- for (IEnumHasCodeAndDescription value : vals) {
- if (value.getDescription().equals(elementString)) {
- return value;
- }
+ }
+ // Retry with the description for compatibility with older versions
+ //
+ if (enumerationClass.isInstance(IEnumHasCodeAndDescription.class)) {
+ IEnumHasCodeAndDescription[] vals =
+ (IEnumHasCodeAndDescription[]) enumerationClass.getEnumConstants();
+ for (IEnumHasCodeAndDescription value : vals) {
+ if (value.getDescription().equals(elementString)) {
+ return value;
}
}
- } catch (Exception e) {
- throw new HopXmlException(
- "Unable to get values() of enumeration to look up code value " + elementString, e);
}
- } else {
- try {
- return Enum.valueOf(enumerationClass, elementString);
- } catch (IllegalArgumentException e) {
- String nameNotFound =
- field.getAnnotation(HopMetadataProperty.class).enumNameWhenNotFound();
- if (StringUtils.isEmpty(nameNotFound)) {
- throw e;
- } else {
- return Enum.valueOf(enumerationClass, nameNotFound);
- }
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Unable to get values() of enumeration to look up code value " + elementString, e);
+ }
+ } else {
+ try {
+ return Enum.valueOf(enumerationClass, elementString);
+ } catch (IllegalArgumentException e) {
+ String nameNotFound =
+ field.getAnnotation(HopMetadataProperty.class).enumNameWhenNotFound();
+ if (StringUtils.isEmpty(nameNotFound)) {
+ throw e;
+ } else {
+ return Enum.valueOf(enumerationClass, nameNotFound);
}
}
}
- } else if (fieldType.equals(java.util.List.class)) {
- // So if we have a List<Field> as per the unit test example
- // We'll now have a String with a bunch of <field></field> groups
+ }
+ return null;
+ }
+
+ private static List<Object> deSerializeList(
+ Object parentObject,
+ HopMetadataProperty parentProperty,
+ Node groupNode,
+ String tag,
+ Field field,
+ IHopMetadataProvider metadataProvider,
+ String[] inlineListTags)
+ throws HopXmlException {
+ // So if we have a List<Field> as per the unit test example
+ // We'll now have a String with a bunch of <field></field> groups
+ //
+ List<Object> list = new ArrayList<>();
+ List<Node> itemNodes = XmlHandler.getNodes(groupNode, tag);
+ if (inlineListTags.length > 0 && !itemNodes.isEmpty()) {
+ // Old XML serialization format where everything is just dumped into the same tag.
+ // See also HopMetadataProperty.inlineListTags
//
- List<Object> list = new ArrayList<>();
- List<Node> itemNodes = XmlHandler.getNodes(groupNode, tag);
- if (inlineListTags.length > 0 && !itemNodes.isEmpty()) {
- // Old XML serialization format where everything is just dumped into the same tag.
- // See also HopMetadataProperty.inlineListTags
- //
- Node parentNode = itemNodes.get(0);
- int listSize = XmlHandler.countNodes(parentNode, inlineListTags[0]);
- if (listSize > 1) {
- itemNodes.clear();
- for (int i = 0; i < listSize; i++) {
- Node node = parentNode.getOwnerDocument().createElement(tag);
- for (String inlineTag : inlineListTags) {
- Node n = XmlHandler.getSubNodeByNr(parentNode, inlineTag, i);
- if (n != null) {
- node.appendChild(n);
- }
+ Node parentNode = itemNodes.get(0);
+ int listSize = XmlHandler.countNodes(parentNode, inlineListTags[0]);
+ if (listSize > 1) {
+ itemNodes.clear();
+ for (int i = 0; i < listSize; i++) {
+ Node node = parentNode.getOwnerDocument().createElement(tag);
+ for (String inlineTag : inlineListTags) {
+ Node n = XmlHandler.getSubNodeByNr(parentNode, inlineTag, i);
+ if (n != null) {
+ node.appendChild(n);
}
- itemNodes.add(node);
}
+ itemNodes.add(node);
}
}
- for (Node itemNode : itemNodes) {
- // We assume that the constructor of the parent class created the List object
- // so that we can simply add items to the list here.
+ }
+ for (Node itemNode : itemNodes) {
+ // We assume that the constructor of the parent class created the List object
+ // so that we can simply add items to the list here.
+ //
+ ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType();
+ Class<?> listClass = (Class<?>) parameterizedType.getActualTypeArguments()[0];
+ try {
+ Object newItem =
+ deSerializeFromXml(
+ parentObject,
+ parentProperty,
+ listClass,
+ null,
+ itemNode,
+ null,
+ null,
+ metadataProvider);
+
+ // Add it to the list
//
- ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType();
- Class<?> listClass = (Class<?>) parameterizedType.getActualTypeArguments()[0];
- try {
- Object newItem =
- deSerializeFromXml(
- parentObject,
- parentProperty,
- listClass,
- null,
- itemNode,
- null,
- null,
- metadataProvider);
-
- // Add it to the list
- //
- list.add(newItem);
- } catch (Exception e) {
- throw new HopXmlException(
- "Unable to instantiate a new instance of class "
- + listClass.getName()
- + ": make sure there is an empty public constructor available to allow XML de-serialization",
- e);
- }
+ list.add(newItem);
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Unable to instantiate a new instance of class "
+ + listClass.getName()
+ + ": make sure there is an empty public constructor available to allow XML de-serialization",
+ e);
}
+ }
- // We now have the list...
+ // We now have the list...
+ //
+ return list;
+ }
+
+ private static Map<Object, Object> deSerializeMap(
+ Object parentObject,
+ HopMetadataProperty parentProperty,
+ Node groupNode,
+ String tag,
+ IHopMetadataProvider metadataProvider)
+ throws HopXmlException {
+ // For the map we're looking at the following pattern:
+ //
+ // <groupKey>
+ // <key> key-and-value-elements </key>
+ // <key> key-and-value-elements </key>
+ // ...
+ // </groupKey>
+ //
+ Map<Object, Object> map = new HashMap<>();
+ List<Node> itemNodes = XmlHandler.getNodes(groupNode, tag);
+ for (Node itemNode : itemNodes) {
+ // This item node contains all items of both the key and value classes.
+ // So we can feed this node into both
//
- return list;
- } else {
- // Load the metadata for this node...
+ Class<?> keyClass = parentProperty.mapKeyClass();
+ Class<?> valueClass = parentProperty.mapValueClass();
+ Object key;
+ Object value;
+ try {
+ // Instantiate the key object
+ key = deSerializeFromXml(itemNode, keyClass, metadataProvider);
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Unable to instantiate a new instance of map key class "
+ + keyClass.getName()
+ + ": make sure there is an empty public constructor available to allow XML de-serialization",
+ e);
+ }
+ try {
+ value = deSerializeFromXml(itemNode, valueClass, metadataProvider);
+ } catch (Exception e) {
+ throw new HopXmlException(
+ "Unable to instantiate a new instance of map value class "
+ + keyClass.getName()
+ + ": make sure there is an empty public constructor available to allow XML de-serialization",
+ e);
+ }
+ // Now we can simply add key and value to the map
//
- return deSerializeFromXml(
- parentObject, parentProperty, elementNode, fieldType, metadataProvider);
+ map.put(key, value);
}
-
- // No value found for the given arguments: return the default value
- //
- return null;
+ return map;
}
}
diff --git a/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
new file mode 100644
index 0000000000..b2edc2b768
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/metadata/inject/HopMetadataInjectorTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.metadata.inject;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.database.DatabaseMetaPlugin;
+import org.apache.hop.core.database.DatabasePluginType;
+import org.apache.hop.core.database.NoneDatabaseMeta;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.core.row.RowMetaBuilder;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaPlugin;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.metadata.inject.company.Company;
+import org.apache.hop.metadata.inject.company.Employee;
+import org.apache.hop.metadata.inject.sample.Field;
+import org.apache.hop.metadata.inject.sample.SampleMeta;
+import org.apache.hop.metadata.inject.sample.SampleType;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class HopMetadataInjectorTest {
+ @BeforeAll
+ static void beforeAll() throws Exception {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ String[] classNames = {
+ ValueMetaString.class.getName(),
+ ValueMetaInteger.class.getName(),
+ ValueMetaDate.class.getName(),
+ ValueMetaNumber.class.getName(),
+ ValueMetaJson.class.getName()
+ };
+ for (String className : classNames) {
+ registry.registerPluginClass(className, ValueMetaPluginType.class, ValueMetaPlugin.class);
+ }
+ registry.registerPluginClass(
+ NoneDatabaseMeta.class.getName(), DatabasePluginType.class, DatabaseMetaPlugin.class);
+ }
+
+ @Test
+ void testCompanyInjection() throws Exception {
+ Map<String, Object> injectionKeyMap = new HashMap<>();
+ Map<String, RowBuffer> injectionGroupMap = new HashMap<>();
+
+ Company company = new Company();
+
+ // We inject the name and 2 employees
+ injectionKeyMap.put("NAME", "ACME");
+
+ RowBuffer rowBuffer = new RowBuffer();
+ rowBuffer.setRowMeta(
+ new RowMetaBuilder().addString("FIRST_NAME").addString("LAST_NAME").build());
+ rowBuffer.addRow("Donald", "Duck");
+ rowBuffer.addRow("Micky", "Mouse");
+ injectionGroupMap.put("EMPLOYEES", rowBuffer);
+
+ HopMetadataInjector.inject(
+ new MemoryMetadataProvider(), company, injectionKeyMap, injectionGroupMap);
+
+ assertEquals("ACME", company.getName());
+ assertEquals(2, company.getEmployees().size());
+ Employee e = company.getEmployees().get(0);
+ assertEquals("Donald", e.getFirstName());
+ assertEquals("Duck", e.getLastName());
+ e = company.getEmployees().get(1);
+ assertEquals("Micky", e.getFirstName());
+ assertEquals("Mouse", e.getLastName());
+ }
+
+ @Test
+ void testCompanyMapping() throws Exception {
+ Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(Company.class);
+ assertNotNull(map);
+ assertEquals(1, map.size());
+ Set<String> keys = map.get("EMPLOYEES");
+ assertEquals(2, keys.size());
+ assertTrue(keys.contains("FIRST_NAME"));
+ assertTrue(keys.contains("LAST_NAME"));
+ }
+
+ @Test
+ void testSampleMetaInjection() throws Exception {
+ Map<String, Object> injectionKeyMap = new HashMap<>();
+ Map<String, RowBuffer> injectionGroupMap = new HashMap<>();
+ MemoryMetadataProvider provider = new MemoryMetadataProvider();
+ provider
+ .getSerializer(DatabaseMeta.class)
+ .save(new DatabaseMeta("EDW", "NONE", "", "", "", "", "", ""));
+
+ SampleMeta sampleMeta = new SampleMeta();
+ injectionKeyMap.put("FILE_NAME", "filename.txt");
+ injectionKeyMap.put("LIMIT", "123");
+ injectionKeyMap.put("IGNORE_ERRORS", "Y");
+ injectionKeyMap.put("MAX_ERRORS", 999);
+ injectionKeyMap.put("SAMPLE_TYPE", "-2-");
+ injectionKeyMap.put("CONNECTION_NAME", "EDW");
+
+ RowBuffer fieldsBuffer = new RowBuffer();
+ fieldsBuffer.setRowMeta(
+ new RowMetaBuilder()
+ .addString("FIELD_NAME")
+ .addString("FIELD_TYPE")
+ .addInteger("FIELD_LENGTH")
+ .addInteger("FIELD_PRECISION")
+ .addString("FIELD_TRIM_TYPE")
+ .build());
+ fieldsBuffer.addRow("f1", "String", 100, -1, "right");
+ fieldsBuffer.addRow("f2", "Integer", 7, 0, "both");
+ fieldsBuffer.addRow("f3", "Number", 9, 2, "left");
+ injectionGroupMap.put("FIELDS", fieldsBuffer);
+ injectionGroupMap.put("MORE_FIELDS", fieldsBuffer);
+
+ RowBuffer stringsBuffer = new RowBuffer();
+ stringsBuffer.setRowMeta(new RowMetaBuilder().addString("STRING").build());
+ stringsBuffer.addRow("string1");
+ stringsBuffer.addRow("string2");
+ injectionGroupMap.put("STRINGS", stringsBuffer);
+
+ HopMetadataInjector.inject(provider, sampleMeta, injectionKeyMap, injectionGroupMap);
+
+ assertEquals("filename.txt", sampleMeta.getFileName());
+ assertEquals(123, sampleMeta.getLimit());
+ assertTrue(sampleMeta.getAdditional().isIgnoreErrors());
+ assertEquals(999, sampleMeta.getAdditional().getMaxErrors());
+ assertEquals(SampleType.TWO, sampleMeta.getSampleType());
+ assertEquals("EDW", sampleMeta.getDatabaseMeta().getName());
+
+ assertEquals(2, sampleMeta.getStrings().size());
+ assertEquals("string1", sampleMeta.getStrings().get(0));
+ assertEquals("string2", sampleMeta.getStrings().get(1));
+
+ assertEquals(3, sampleMeta.getFields().size());
+ validateSampleMetaFields(sampleMeta.getFields());
+
+ assertEquals(3, sampleMeta.getMoreFields().size());
+ validateSampleMetaFields(sampleMeta.getMoreFields());
+ }
+
+ private static void validateSampleMetaFields(List<Field> fields) {
+ Field f = fields.get(0);
+ assertEquals("f1", f.getName());
+ assertEquals(IValueMeta.TYPE_STRING, f.getHopType());
+ assertEquals(100, f.getLength());
+ assertEquals(-1, f.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_RIGHT, f.getTrimType());
+
+ f = fields.get(1);
+ assertEquals("f2", f.getName());
+ assertEquals(IValueMeta.TYPE_INTEGER, f.getHopType());
+ assertEquals(7, f.getLength());
+ assertEquals(0, f.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_BOTH, f.getTrimType());
+
+ f = fields.get(2);
+ assertEquals("f3", f.getName());
+ assertEquals(IValueMeta.TYPE_NUMBER, f.getHopType());
+ assertEquals(9, f.getLength());
+ assertEquals(2, f.getPrecision());
+ assertEquals(IValueMeta.TRIM_TYPE_LEFT, f.getTrimType());
+ }
+
+ @Test
+ void testSampleMetaMapping() throws Exception {
+ Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(SampleMeta.class);
+ assertNotNull(map);
+ assertEquals(3, map.size());
+ Set<String> strings = map.get("STRINGS");
+ assertEquals(1, strings.size());
+ assertTrue(strings.contains("STRING"));
+
+ Set<String> fields = map.get("FIELDS");
+ assertEquals(5, fields.size());
+ assertTrue(fields.contains("FIELD_NAME"));
+ assertTrue(fields.contains("FIELD_TYPE"));
+ assertTrue(fields.contains("FIELD_LENGTH"));
+ assertTrue(fields.contains("FIELD_PRECISION"));
+ assertTrue(fields.contains("FIELD_TRIM_TYPE"));
+
+ Set<String> moreFields = map.get("MORE_FIELDS");
+ assertEquals(5, moreFields.size());
+ assertTrue(moreFields.contains("FIELD_NAME"));
+ assertTrue(moreFields.contains("FIELD_TYPE"));
+ assertTrue(moreFields.contains("FIELD_LENGTH"));
+ assertTrue(moreFields.contains("FIELD_PRECISION"));
+ assertTrue(moreFields.contains("FIELD_TRIM_TYPE"));
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java b/core/src/test/java/org/apache/hop/metadata/inject/company/Company.java
similarity index 63%
copy from core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
copy to core/src/test/java/org/apache/hop/metadata/inject/company/Company.java
index b0b9716e33..367e443b78 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
+++ b/core/src/test/java/org/apache/hop/metadata/inject/company/Company.java
@@ -16,20 +16,24 @@
*
*/
-package org.apache.hop.metadata.serializer.xml;
+package org.apache.hop.metadata.inject.company;
-import java.lang.reflect.Field;
-import java.util.function.Function;
-import org.apache.hop.core.Const;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.metadata.api.HopMetadataProperty;
-public class MetadataPropertyKeyFunction implements Function<Field, String> {
- @Override
- public String apply(Field field) {
- HopMetadataProperty annotation = field.getAnnotation(HopMetadataProperty.class);
- if (annotation == null) {
- return null;
- }
- return Const.NVL(annotation.groupKey(), Const.NVL(annotation.key(), field.getName()));
+@Getter
+@Setter
+public class Company {
+ @HopMetadataProperty(key = "name", injectionKey = "NAME")
+ private String name;
+
+ @HopMetadataProperty(key = "employees", injectionGroupKey = "EMPLOYEES")
+ private List<Employee> employees;
+
+ public Company() {
+ employees = new ArrayList<>();
}
}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java b/core/src/test/java/org/apache/hop/metadata/inject/company/Employee.java
similarity index 63%
copy from core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
copy to core/src/test/java/org/apache/hop/metadata/inject/company/Employee.java
index b0b9716e33..648d3bac55 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
+++ b/core/src/test/java/org/apache/hop/metadata/inject/company/Employee.java
@@ -16,20 +16,26 @@
*
*/
-package org.apache.hop.metadata.serializer.xml;
+package org.apache.hop.metadata.inject.company;
-import java.lang.reflect.Field;
-import java.util.function.Function;
-import org.apache.hop.core.Const;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.metadata.api.HopMetadataProperty;
-public class MetadataPropertyKeyFunction implements Function<Field, String> {
- @Override
- public String apply(Field field) {
- HopMetadataProperty annotation = field.getAnnotation(HopMetadataProperty.class);
- if (annotation == null) {
- return null;
- }
- return Const.NVL(annotation.groupKey(), Const.NVL(annotation.key(), field.getName()));
+@Getter
+@Setter
+public class Employee {
+ @HopMetadataProperty(key = "firstName", injectionKey = "FIRST_NAME")
+ private String firstName;
+
+ @HopMetadataProperty(key = "lastName", injectionKey = "LAST_NAME")
+ private String lastName;
+
+ public Employee() {}
+
+ public Employee(String firstName, String lastName) {
+ this();
+ this.firstName = firstName;
+ this.lastName = lastName;
}
}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java b/core/src/test/java/org/apache/hop/metadata/inject/sample/Additional.java
similarity index 63%
copy from core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
copy to core/src/test/java/org/apache/hop/metadata/inject/sample/Additional.java
index b0b9716e33..0a51346aea 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
+++ b/core/src/test/java/org/apache/hop/metadata/inject/sample/Additional.java
@@ -16,20 +16,25 @@
*
*/
-package org.apache.hop.metadata.serializer.xml;
+package org.apache.hop.metadata.inject.sample;
-import java.lang.reflect.Field;
-import java.util.function.Function;
-import org.apache.hop.core.Const;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.metadata.api.HopMetadataProperty;
-public class MetadataPropertyKeyFunction implements Function<Field, String> {
- @Override
- public String apply(Field field) {
- HopMetadataProperty annotation = field.getAnnotation(HopMetadataProperty.class);
- if (annotation == null) {
- return null;
- }
- return Const.NVL(annotation.groupKey(), Const.NVL(annotation.key(), field.getName()));
+@Getter
+@Setter
+public class Additional {
+ @HopMetadataProperty(injectionKey = "IGNORE_ERRORS")
+ private boolean ignoreErrors;
+
+ @HopMetadataProperty(injectionKey = "MAX_ERRORS")
+ private int maxErrors;
+
+ public Additional() {}
+
+ public Additional(boolean ignoreErrors, int maxErrors) {
+ this.ignoreErrors = ignoreErrors;
+ this.maxErrors = maxErrors;
}
}
diff --git a/core/src/test/java/org/apache/hop/metadata/inject/sample/Field.java b/core/src/test/java/org/apache/hop/metadata/inject/sample/Field.java
new file mode 100644
index 0000000000..f0f3bb9b47
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/metadata/inject/sample/Field.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.metadata.inject.sample;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.row.value.ValueMetaBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+@Getter
+@Setter
+public class Field {
+ @HopMetadataProperty(injectionKey = "FIELD_NAME")
+ private String name;
+
+ @HopMetadataProperty(
+ injectionKey = "FIELD_TYPE",
+ intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class)
+ private int hopType;
+
+ @HopMetadataProperty(injectionKey = "FIELD_LENGTH")
+ private int length;
+
+ @HopMetadataProperty(injectionKey = "FIELD_PRECISION")
+ private int precision;
+
+ @HopMetadataProperty(
+ injectionKey = "FIELD_TRIM_TYPE",
+ intCodeConverter = ValueMetaBase.TrimTypeCodeConverter.class)
+ private int trimType;
+
+ public Field() {}
+
+ public Field(int hopType, int length, String name, int precision, int trimType) {
+ this.hopType = hopType;
+ this.length = length;
+ this.name = name;
+ this.precision = precision;
+ this.trimType = trimType;
+ }
+}
diff --git a/core/src/test/java/org/apache/hop/metadata/inject/sample/SampleMeta.java b/core/src/test/java/org/apache/hop/metadata/inject/sample/SampleMeta.java
new file mode 100644
index 0000000000..78ba6538d8
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/metadata/inject/sample/SampleMeta.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.metadata.inject.sample;
+
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+
+@Getter
+@Setter
+public class SampleMeta {
+ @HopMetadataProperty(injectionKey = "STRING", injectionGroupKey = "STRINGS")
+ private List<String> strings;
+
+ @HopMetadataProperty(injectionKey = "FILE_NAME")
+ private String fileName;
+
+ @HopMetadataProperty(injectionKey = "LIMIT")
+ private int limit;
+
+ @HopMetadataProperty(storeWithCode = true, injectionKey = "SAMPLE_TYPE")
+ private SampleType sampleType;
+
+ @HopMetadataProperty(
+ storeWithName = true,
+ injectionKey = "CONNECTION_NAME",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+ private DatabaseMeta databaseMeta;
+
+ @HopMetadataProperty private Additional additional;
+
+ @HopMetadataProperty(injectionGroupKey = "FIELDS")
+ private List<Field> fields;
+
+ @HopMetadataProperty(injectionGroupKey = "MORE_FIELDS")
+ private List<Field> moreFields;
+
+ public SampleMeta() {
+ this.strings = new ArrayList<>();
+ this.additional = new Additional();
+ this.fields = new ArrayList<>();
+ this.moreFields = new ArrayList<>();
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java b/core/src/test/java/org/apache/hop/metadata/inject/sample/SampleType.java
similarity index 59%
copy from core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
copy to core/src/test/java/org/apache/hop/metadata/inject/sample/SampleType.java
index b0b9716e33..c9a657a2af 100644
--- a/core/src/main/java/org/apache/hop/metadata/serializer/xml/MetadataPropertyKeyFunction.java
+++ b/core/src/test/java/org/apache/hop/metadata/inject/sample/SampleType.java
@@ -16,20 +16,22 @@
*
*/
-package org.apache.hop.metadata.serializer.xml;
+package org.apache.hop.metadata.inject.sample;
-import java.lang.reflect.Field;
-import java.util.function.Function;
-import org.apache.hop.core.Const;
-import org.apache.hop.metadata.api.HopMetadataProperty;
+import lombok.Getter;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
-public class MetadataPropertyKeyFunction implements Function<Field, String> {
- @Override
- public String apply(Field field) {
- HopMetadataProperty annotation = field.getAnnotation(HopMetadataProperty.class);
- if (annotation == null) {
- return null;
- }
- return Const.NVL(annotation.groupKey(), Const.NVL(annotation.key(), field.getName()));
+@Getter
+public enum SampleType implements IEnumHasCodeAndDescription {
+ ONE("-1-", "One"),
+ TWO("-2-", "Two"),
+ THREE("-3-", "Three"),
+ ;
+ private final String code;
+ private final String description;
+
+ SampleType(String code, String description) {
+ this.code = code;
+ this.description = description;
}
}
diff --git a/core/src/test/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtilTest.java b/core/src/test/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtilTest.java
index a25b8dbefd..738739b58b 100644
--- a/core/src/test/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/hop/metadata/serializer/xml/XmlMetadataUtilTest.java
@@ -18,6 +18,7 @@
package org.apache.hop.metadata.serializer.xml;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
import org.apache.hop.core.Const;
@@ -27,6 +28,7 @@ import org.apache.hop.metadata.serializer.xml.classes.Field;
import org.apache.hop.metadata.serializer.xml.classes.Info;
import org.apache.hop.metadata.serializer.xml.classes.MetaData;
import org.apache.hop.metadata.serializer.xml.classes.TestEnum;
+import org.apache.hop.metadata.serializer.xml.classes.WithMap;
import org.junit.jupiter.api.Test;
import org.w3c.dom.Node;
@@ -150,4 +152,32 @@ class XmlMetadataUtilTest {
assertEquals(valueTest, valueData);
}
}
+
+ @Test
+ void testMappingSerialization() throws Exception {
+ WithMap withMap = new WithMap();
+ WithMap.Key k1 = new WithMap.Key("k11", "k12");
+ WithMap.Value v1 = new WithMap.Value("v11", "v12");
+ withMap.getMappings().put(k1, v1);
+
+ WithMap.Key k2 = new WithMap.Key("k21", "k22");
+ WithMap.Value v2 = new WithMap.Value("v21", "v22");
+ withMap.getMappings().put(k2, v2);
+
+ WithMap.Key k3 = new WithMap.Key("k31", "k32");
+ WithMap.Value v3 = new WithMap.Value("v31", "v32");
+ withMap.getMappings().put(k3, v3);
+
+ String xml = XmlMetadataUtil.serializeObjectToXml(withMap);
+ Node node = XmlHandler.loadXmlString(xml);
+ WithMap withCopy =
+ XmlMetadataUtil.deSerializeFromXml(node, WithMap.class, new MemoryMetadataProvider());
+ assertEquals(withCopy.getMappings().size(), withMap.getMappings().size());
+ for (WithMap.Key key : withMap.getMappings().keySet()) {
+ WithMap.Value value = withMap.getMappings().get(key);
+ WithMap.Value valueCopy = withCopy.getMappings().get(key);
+ assertTrue(withCopy.getMappings().containsKey(key));
+ assertEquals(value, valueCopy);
+ }
+ }
}
diff --git a/core/src/test/java/org/apache/hop/metadata/serializer/xml/classes/WithMap.java b/core/src/test/java/org/apache/hop/metadata/serializer/xml/classes/WithMap.java
new file mode 100644
index 0000000000..450bb74ae5
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/metadata/serializer/xml/classes/WithMap.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.metadata.serializer.xml.classes;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+@Getter
+@Setter
+public class WithMap {
+
+ @HopMetadataProperty(
+ key = "mapping",
+ groupKey = "mappings",
+ inline = true,
+ mapKeyClass = Key.class,
+ mapValueClass = Value.class)
+ private Map<Key, Value> mappings;
+
+ public WithMap() {
+ mappings = new HashMap<>();
+ }
+
+ @Getter
+ @Setter
+ public static class Key {
+ @HopMetadataProperty(key = "f1")
+ private String f1;
+
+ @HopMetadataProperty(key = "f2")
+ private String f2;
+
+ public Key() {
+ // Nothing
+ }
+
+ public Key(String f1, String f2) {
+ this();
+ this.f1 = f1;
+ this.f2 = f2;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Key key)) return false;
+ return Objects.equals(f1, key.f1) && Objects.equals(f2, key.f2);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(f1, f2);
+ }
+ }
+
+ @Getter
+ @Setter
+ public static class Value {
+ @HopMetadataProperty(key = "v1")
+ private String v1;
+
+ @HopMetadataProperty(key = "v2")
+ private String v2;
+
+ public Value() {
+ // Nothing
+ }
+
+ public Value(String v1, String v2) {
+ this();
+ this.v1 = v1;
+ this.v2 = v2;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Value value)) return false;
+ return Objects.equals(v1, value.v1) && Objects.equals(v2, value.v2);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(v1, v2);
+ }
+ }
+}
diff --git a/docs/hop-dev-manual/modules/ROOT/nav.adoc b/docs/hop-dev-manual/modules/ROOT/nav.adoc
index c81487ca81..bb34f30958 100644
--- a/docs/hop-dev-manual/modules/ROOT/nav.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/nav.adoc
@@ -18,6 +18,8 @@ under the License.
* xref:upgrading-to-20.adoc[Plugins Upgrades to 2.x]
* xref:porting-kettle-plugins.adoc[Porting Kettle plugins]
* xref:metadata-plugins.adoc[Metadata plugins]
+* xref:metadata-serialization.adoc[Metadata Serialization]
+* xref:metadata-injection.adoc[Metadata Injection]
* xref:setup-dev-environment.adoc[Setting up your development environment]
* xref:docker-build.adoc[Docker Build Script]
* xref:integration-testing.adoc[Integration testing]
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/metadata-injection.adoc b/docs/hop-dev-manual/modules/ROOT/pages/metadata-injection.adoc
new file mode 100644
index 0000000000..2496c79ec3
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/metadata-injection.adoc
@@ -0,0 +1,61 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: How does ETL Metadata Injection work in Hop transforms?
+
+[[metadata-injection]]
+
+:toc:
+
+= ETL Metadata Injection
+
+== Introduction
+
+https://hop.apache.org//manual/latest/pipeline/transforms/metainject.html[ETL Metadata Injection] is a transform that allows you to modify the properties of a specified "template" pipeline. The transforms that will be executed are chosen, but their properties and settings are "injected" at runtime.
+
+This transform uses a mapping between information that the transform reads from source transforms in the form of one or more rows of data in source fields. These fields can then be mapped to target transform fields in the specified "template" pipeline.
+
+== Hop Metadata Property annotations
+
+How can you specify how a field to be injected shows up in the ETL Metadata Injection dialog? You can use the `@HopMetadataProperty` annotation. For more example on how this is used to serialize metadata, see the xref:metadata-serialization.adoc[Metadata Serialization] page.
+
+By default, if a field is annotated, it's going to be visible.
+You can specify that a field has a specific `injectionKey` for backward compatibility with older forms or annotations.
+
+For lists, you can use the `injectionGroupKey`.
+
+[IMPORTANT]
+The ETL Metadata Injection transform requires all keys in a single transform metadata class to be unique.
+
+Avoid giving a `File` object in a list of files a key or injectionKey value of `name` if you would do the same in a Field class in a list of fields.
+
+== Internationalization (i18n)
+
+To allow for meaningful descriptions of the injectable fields to show up in the ETL Metadata Injection transform dialog you an specify properties `injectionKeyDescription` and `injectionGroupDescription`. The specified strings reference an entry to the message bundles in the transform, e.g. `messages_en_US.properties` and their localized variants.
+
+[TIP]
+You can list the values that are acceptable in the injection key description.
+
+== Data conversions
+
+The injection system will try to use the already specified methods for XML/JSON serialization but there are also these options:
+
+* `InjectionTypeConverter` specified by `injectionConverter`
+* `IIntCodeConverter` specified by `intCodeConverter`
+* `IStringObjectConverter` specified by `injectionStringObjectConverter`
+* `IStringEncoder` specified by `stringEncoder`
+
+
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/metadata-serialization.adoc b/docs/hop-dev-manual/modules/ROOT/pages/metadata-serialization.adoc
new file mode 100644
index 0000000000..e04cbdd12a
--- /dev/null
+++ b/docs/hop-dev-manual/modules/ROOT/pages/metadata-serialization.adoc
@@ -0,0 +1,402 @@
+////
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+////
+:description: How does Hop serialize pipeline and workflow metadata?
+[[metadata-serialization]]
+
+:toc:
+
+= XML and JSON serialization
+
+== The Basics
+
+To serialize fields to XML in the `*Meta` transform classes or the `Action*` action classes, or indeed to JSON for project metadata elements, you can use the `@HopMetadataProperty` annotation. The serialization to XML or JSON will happen with the name of the field by default. For example:
+
+=== The default
+
+[source,java]
+----
+@HopMetadataProperty
+String filename;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<filename>field-content</filename>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "filename" : "field-content"
+}
+----
+====
+
+=== The key property
+
+If you want to use a different XML element name you can specify this with the `key` property of the annotation:
+
+[source,java]
+----
+@HopMetadataProperty(key="file-name")
+String filename;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<file-name>field-content</file-name>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "file-name" : "field-content"
+}
+----
+====
+
+=== Metadata references
+
+For fields of types that implement `IHopMetadata`, you can opt to store the name like this:
+
+[source,java]
+----
+@HopMetadataProperty(
+ key="connection",
+ storeWithName=true,
+ hopMetadataPropertyType=HopMetadataPropertyType.RDBMS_CONNECTION)
+DatabaseMeta databaseMeta;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<connection>element-name</connection>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "connection" : "element-name"
+}
+----
+====
+
+=== Enum fields
+
+Enumerations are stored by default using their name.
+Since it's used so often we also allow you to implement `IEnumHasCode` or even `IEnumHasCodeAndDescription`. For enums that implement either you can opt to store the value by a chosen code. This is useful for backward compatibility with older XML formats and to give you the flexibility needed.
+
+Example from `DimensionLookupMeta.java`:
+[source,java]
+----
+ @HopMetadataProperty(
+ key = "creation_method",
+ storeWithCode = true)
+ private TechnicalKeyCreationMethod creationMethod;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<creation_method>autoinc</creation_method>
+<creation_method>sequence</creation_method>
+<creation_method>tablemax</creation_method>
+<creation_method>uuid</creation_method>
+<creation_method>field</creation_method>
+----
+One of these code values will be used depending on the enum chosen.
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "creation_method" : "autoinc"
+}
+----
+====
+
+=== int codes
+
+If often happens in the codebase that field and trim types are stored as integers, typically something like `IValueMeta.TYPE_STRING` (2) or `IValueMeta.TRIM_TYPE_LEFT`.
+Since this isn't very meaningful, we offer the ability to specify an `IIntCodeConverter` to convert `2` into `String`.
+
+Example from `CsvInputField.java`:
+
+[source,java]
+----
+ @HopMetadataProperty(
+ key = "type",
+ intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class)
+ private int type;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<type>String</type>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "type" : "String"
+}
+----
+====
+
+In the same way there is a standard class for trim type: `ValueMetaBase.TrimTypeCodeConverter`.
+
+=== Passwords & secrets
+
+If you have sensitive information that you want to obfuscate or https://hop.apache.org//manual/latest/password/passwords/aespasswords.html[encrypt], you can use the `password` annotation property.
+
+[source,java]
+----
+ @HopMetadataProperty(password=true)
+ private String password;
+----
+
+This will serialize to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<password>Encrypted: SomeObfuscationGoingOn</password>
+<secret>AES2: SomeEncryptionGoingOn</secret>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "password" : "Encrypted: SomeObfuscationGoingOn",
+ "secret": "AES2: SomeEncryptionGoingOn"
+}
+----
+====
+
+=== String encoder
+
+Sometimes you want to encode Strings to Base64 or other schemes to avoid having unsupported characters or even binary content in your serialized files. You can use the `stringEncoder` property of the annotation for this.
+
+From `TextFileInputMeta.java`:
+[source,java]
+----
+ /** The string to filter on */
+ @HopMetadataProperty(
+ key = "filter_string",
+ stringEncoder = Base64StringEncoder.class)
+ private String filterString;
+----
+
+This will serialize `Apache Hop` to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<filter_string>QXBhY2hlIEhvcA==</filter_string>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "filter_string" : "QXBhY2hlIEhvcA=="
+}
+----
+====
+
+== Lists
+
+If you want to serialize List of objects, you can typically wrap them in a group. An example from `CsvInputMeta.java`:
+
+[source,java]
+----
+ @HopMetadataProperty(
+ key = "field",
+ groupKey = "fields")
+ private List<CsvInputField> inputFields;
+----
+
+Will serialize the content of the `CsvInputField` objects inside `<fields><field>` wrappers:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<fields>
+ <field>
+ <name>fieldName</name>
+ <type>String</type>
+ ...
+ </field>
+ ...
+</fields>
+----
+====
+
+.JSON
+[%collapsible]
+====
+[source,JSON]
+----
+{
+ "fields": [
+ {
+ "name": "fieldName",
+ "type": "String",
+ ...
+ },
+ ...
+ ]
+}
+----
+====
+
+== Maps
+
+You can serialize Map fields to XML as well.
+
+[IMPORTANT]
+It's not possible to inject metadata into a Map data type so don't use these constructs for transform serialization.
+
+[source,java]
+----
+ @HopMetadataProperty(
+ key = "mapping",
+ groupKey = "mappings",
+ inline = true,
+ mapKeyClass = Key.class,
+ mapValueClass = Value.class)
+ private Map<Key, Value> mappings;
+----
+
+Serializes to:
+
+.XML
+[%collapsible]
+====
+[source,XML]
+----
+<mappings>
+ <mapping>
+ <key>
+ <keyProperty1>key1</keyProperty1>
+ <keyProperty2>key2</keyProperty2>
+ </key>
+ <value>
+ <valueProperty1>value1</keyProperty1>
+ <valueProperty2>value2</keyProperty2>
+ <valueProperty3>value3</keyProperty3>
+ </value>
+ </mapping>
+ ...
+</mappings>
+----
+====
+
+.JSON
+[%collapsible]
+====
+JSON serialization of Map objects is not supported yet.
+====
+
+== Sub-elements
+
+All elements you want to put in a different sub-element you can put in a different field and give it another key.
+
+== Serializing interfaces
+
+It's possible to serialize objects with only their interface. You need to specify a `@HopMetadataObject` annotation on the interface in question.
+The following example is taken from `IValueMeta.java` where data types are plugins. We want to read the type `String` and know to get object `ValueMetaString` by specifying an Object Factory like this:
+
+[source,java]
+----
+@HopMetadataObject(objectFactory = IValueMeta.ValueMetaHopMetadataObjectFactory.class)
+public interface IValueMeta {}
+----
+
+The way that this is done is with 2 simple methods. One for creating a new object with an id, and one for getting an ID from an object interface.
+
+From `class ValueMetaHopMetadataObjectFactory`:
+[source,java]
+----
+ @Override
+ public Object createObject(String id, Object parentObject) throws HopException {
+ int type = ValueMetaFactory.getIdForValueMeta(id);
+ return ValueMetaFactory.getValueMetaName(type);
+ }
+
+ @Override
+ public String getObjectId(Object object) throws HopException {
+ return ((IValueMeta) object).getTypeDesc();
+ }
+----
+
diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/azure-key-vault-variable-resolver.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/azure-key-vault-variable-resolver.adoc
index 3acf0f5c87..23a333b26f 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/azure-key-vault-variable-resolver.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/variable-resolver/azure-key-vault-variable-resolver.adoc
@@ -109,7 +109,7 @@ Using Azure Portal:
6. Note down the **Application (client) ID** and **Directory (tenant) ID**
7. Go to **Certificates & secrets** → **New client secret**
8. Add a description and expiration period
-9. Click **Add** and immediately copy the **Value** (you won't be able to see it again!)
+9. Click **Add** and immediately copy the **V** (you won't be able to see it again!)
Using Azure CLI:
@@ -162,7 +162,7 @@ Using Azure Portal:
1. Navigate to your Key Vault
2. Go to **Secrets** → **Generate/Import**
3. Enter a **Name** (e.g., `database-password`)
-4. Enter the **Value** (the actual secret)
+4. Enter the **V** (the actual secret)
5. Click **Create**
Using Azure CLI:
diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java
index d94d2622be..3c2233c7dd 100644
--- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java
+++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanInjector.java
@@ -187,12 +187,11 @@ public class BeanInjector<Meta extends Object> {
setProperty(root, prop, i, data.get(i), dataName, dataValue);
}
} else {
- for (int i = 0; ; i++) {
- boolean found = setProperty(root, prop, i, null, null, dataValue);
- if (!found) {
- break;
- }
- }
+ boolean found;
+ int i = 0;
+ do {
+ found = setProperty(root, prop, i++, null, null, dataValue);
+ } while (found);
}
} catch (Exception ex) {
throw new HopException(
diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
index 4fd7acfada..792321dc46 100644
--- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
+++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
@@ -41,7 +41,7 @@ public class BeanLevelInfo<Meta extends Object> {
NONE,
ARRAY,
LIST,
- };
+ }
/** Parent transform or null for root. */
public BeanLevelInfo parent;
diff --git a/integration-tests/mdi/0004-csv-input-parent.hpl b/integration-tests/mdi/0004-csv-input-parent.hpl
index c10273c96d..62d06dd109 100644
--- a/integration-tests/mdi/0004-csv-input-parent.hpl
+++ b/integration-tests/mdi/0004-csv-input-parent.hpl
@@ -34,8 +34,6 @@ limitations under the License.
<created_date>2021/05/10 10:50:11.779</created_date>
<modified_user>-</modified_user>
<modified_date>2021/05/10 10:50:11.779</modified_date>
- <key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
- <is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
@@ -67,174 +65,173 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <create_parent_folder>N</create_parent_folder>
<filename>${PROJECT_HOME}/0004-csv-input-child.hpl</filename>
- <source_transform>Output</source_transform>
- <source_output_fields>
- <source_output_field>
- <source_output_field_name>id</source_output_field_name>
- <source_output_field_type>Integer</source_output_field_type>
- <source_output_field_length>15</source_output_field_length>
- <source_output_field_precision>0</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>name</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>50</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>firstname</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>50</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>zip</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>40</source_output_field_length>
- <source_output_field_precision>0</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>city</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>50</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>birthdate</source_output_field_name>
- <source_output_field_type>Date</source_output_field_type>
- <source_output_field_length>-1</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>street</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>11</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>housenr</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>20</source_output_field_length>
- <source_output_field_precision>0</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>stateCode</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>10</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- <source_output_field>
- <source_output_field_name>state</source_output_field_name>
- <source_output_field_type>String</source_output_field_type>
- <source_output_field_length>50</source_output_field_length>
- <source_output_field_precision>-1</source_output_field_precision>
- </source_output_field>
- </source_output_fields>
- <target_file/>
- <no_execution>N</no_execution>
- <stream_source_transform/>
- <stream_target_transform/>
<mappings>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>currency</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_CURRENCY</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>currency</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>decimal</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_DECIMAL</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>decimal</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>separator</source_field>
+ <source_transform>CSV Input metadata</source_transform>
<target_attribute_key>DELIMITER</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>CSV Input metadata</source_transform>
- <source_field>separator</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>header</source_field>
+ <source_transform>CSV Input metadata</source_transform>
<target_attribute_key>HEADER_PRESENT</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>CSV Input metadata</source_transform>
- <source_field>header</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>length</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_LENGTH</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>length</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>enclosure</source_field>
+ <source_transform>CSV Input metadata</source_transform>
<target_attribute_key>ENCLOSURE</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>CSV Input metadata</source_transform>
- <source_field>enclosure</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>precision</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_PRECISION</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>precision</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>format</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_FORMAT</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>format</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>lazy</source_field>
+ <source_transform>CSV Input metadata</source_transform>
<target_attribute_key>LAZY_CONVERSION</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>CSV Input metadata</source_transform>
- <source_field>lazy</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>group</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_GROUP</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>group</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>trimType</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>FIELD_TRIM_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>trimType</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>filename</source_field>
+ <source_transform>CSV Input metadata</source_transform>
<target_attribute_key>FILENAME</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>CSV Input metadata</source_transform>
- <source_field>filename</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>type</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>FIELD_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>type</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>CSV file input</target_transform_name>
+ <source_field>name</source_field>
+ <source_transform>CSV Input fields</source_transform>
<target_attribute_key>INPUT_NAME</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>CSV Input fields</source_transform>
- <source_field>name</source_field>
+ <target_transform_name>CSV file input</target_transform_name>
</mapping>
</mappings>
+ <no_execution>N</no_execution>
+ <source_output_fields>
+ <source_output_field>
+ <source_output_field_length>15</source_output_field_length>
+ <source_output_field_name>id</source_output_field_name>
+ <source_output_field_precision>0</source_output_field_precision>
+ <source_output_field_type>Integer</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>50</source_output_field_length>
+ <source_output_field_name>name</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>50</source_output_field_length>
+ <source_output_field_name>firstname</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>40</source_output_field_length>
+ <source_output_field_name>zip</source_output_field_name>
+ <source_output_field_precision>0</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>50</source_output_field_length>
+ <source_output_field_name>city</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>-1</source_output_field_length>
+ <source_output_field_name>birthdate</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>Date</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>11</source_output_field_length>
+ <source_output_field_name>street</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>20</source_output_field_length>
+ <source_output_field_name>housenr</source_output_field_name>
+ <source_output_field_precision>0</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>10</source_output_field_length>
+ <source_output_field_name>stateCode</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_length>50</source_output_field_length>
+ <source_output_field_name>state</source_output_field_name>
+ <source_output_field_precision>-1</source_output_field_precision>
+ <source_output_field_type>String</source_output_field_type>
+ </source_output_field>
+ </source_output_fields>
+ <source_transform>Output</source_transform>
<attributes/>
<GUI>
<xloc>304</xloc>
@@ -252,107 +249,6 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <fields>
- <field>
- <name>name</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>type</name>
- <type/>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>format</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>length</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>precision</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>currency</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>decimal</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>group</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- <field>
- <name>trimType</name>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
- <length>-1</length>
- <precision>-1</precision>
- <set_empty_string>N</set_empty_string>
- </field>
- </fields>
<data>
<line>
<item>id</item>
@@ -465,29 +361,76 @@ limitations under the License.
<item>none</item>
</line>
</data>
+ <fields>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>name</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>type</name>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>format</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>length</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>precision</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>currency</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>decimal</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>group</name>
+ <type>String</type>
+ </field>
+ <field>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ <name>trimType</name>
+ <type>String</type>
+ </field>
+ </fields>
<attributes/>
<GUI>
<xloc>128</xloc>
<yloc>96</yloc>
</GUI>
</transform>
- <transform>
- <name>Verify</name>
- <type>Dummy</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <attributes/>
- <GUI>
- <xloc>464</xloc>
- <yloc>96</yloc>
- </GUI>
- </transform>
<transform>
<name>CSV Input metadata</name>
<type>GetVariable</type>
@@ -501,64 +444,44 @@ limitations under the License.
</partitioning>
<fields>
<field>
- <name>filename</name>
- <variable>${PROJECT_HOME}/files/customers-100.txt</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>filename</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>${PROJECT_HOME}/files/customers-100.txt</variable>
</field>
<field>
- <name>separator</name>
- <variable>;</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>separator</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>;</variable>
</field>
<field>
- <name>enclosure</name>
- <variable>"</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>enclosure</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>"</variable>
</field>
<field>
- <name>lazy</name>
- <variable>false</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>lazy</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>false</variable>
</field>
<field>
- <name>header</name>
- <variable>true</variable>
- <type>String</type>
- <format/>
- <currency/>
- <decimal/>
- <group/>
<length>-1</length>
+ <name>header</name>
<precision>-1</precision>
<trim_type>none</trim_type>
+ <type>String</type>
+ <variable>true</variable>
</field>
</fields>
<attributes/>
@@ -567,6 +490,23 @@ limitations under the License.
<yloc>192</yloc>
</GUI>
</transform>
+ <transform>
+ <name>Verify</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>464</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
diff --git a/integration-tests/mdi/0031-get-data-from-xml.hpl b/integration-tests/mdi/0031-get-data-from-xml.hpl
index 9f12dc904b..6171e896e0 100644
--- a/integration-tests/mdi/0031-get-data-from-xml.hpl
+++ b/integration-tests/mdi/0031-get-data-from-xml.hpl
@@ -100,9 +100,10 @@ limitations under the License.
<source_output_field_precision>-1</source_output_field_precision>
</source_output_field>
</source_output_fields>
- <target_file/>
+ <target_file>/tmp/get-data-from-xml.hpl</target_file>
<create_parent_folder>Y</create_parent_folder>
<no_execution>N</no_execution>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
<stream_source_transform/>
<stream_target_transform/>
<mappings>
diff --git a/integration-tests/mdi/0042-user-defined-java-class.hpl b/integration-tests/mdi/0042-user-defined-java-class.hpl
index 0494fd062f..00e2c0c24d 100644
--- a/integration-tests/mdi/0042-user-defined-java-class.hpl
+++ b/integration-tests/mdi/0042-user-defined-java-class.hpl
@@ -65,11 +65,6 @@ limitations under the License.
</hop>
<hop>
<from>fields with Strings</from>
- <to>fields</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>fields</from>
<to>0042-user-defined-java-class-template.hpl</to>
<enabled>Y</enabled>
</hop>
@@ -85,137 +80,138 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <filename>${PROJECT_HOME}/0042-user-defined-java-class-template.hpl</filename>
- <run_configuration>local</run_configuration>
- <source_transform/>
- <source_output_fields> </source_output_fields>
- <target_file>${java.io.tmpdir}/user-defined-java-class.hpl</target_file>
- <create_parent_folder>Y</create_parent_folder>
- <no_execution>Y</no_execution>
<allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
- <stream_source_transform/>
- <stream_target_transform/>
+ <create_parent_folder>Y</create_parent_folder>
+ <filename>${PROJECT_HOME}/0042-user-defined-java-class-template.hpl</filename>
<mappings>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>class_name</source_field>
+ <source_transform>definitions</source_transform>
<target_attribute_key>CLASS_NAME</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_name</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>class_source</source_field>
+ <source_transform>definitions</source_transform>
<target_attribute_key>CLASS_SOURCE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_source</source_field>
- </mapping>
- <mapping>
<target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_LENGTH</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_length</source_field>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>parameter_value</source_field>
+ <source_transform>parameters</source_transform>
<target_attribute_key>VALUE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_value</source_field>
- </mapping>
- <mapping>
<target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_PRECISION</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_precision</source_field>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>clear_result_fields</source_field>
+ <source_transform>files/user-defined-java-class.xml</source_transform>
<target_attribute_key>CLEAR_RESULT_FIELDS</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>files/user-defined-java-class.xml</source_transform>
- <source_field>clear_result_fields</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_description</source_field>
+ <source_transform>targets</source_transform>
<target_attribute_key>TARGET_DESCRIPTION</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_description</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>class_type</source_field>
+ <source_transform>definitions</source_transform>
<target_attribute_key>CLASS_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_type</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_name</source_field>
+ <source_transform>targets</source_transform>
<target_attribute_key>TARGET_TRANSFORM_NAME</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_name</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_tag</source_field>
+ <source_transform>infos</source_transform>
<target_attribute_key>INFO_TAG</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_tag</source_field>
- </mapping>
- <mapping>
<target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_NAME</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_name</source_field>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>parameter_tag</source_field>
+ <source_transform>parameters</source_transform>
<target_attribute_key>TAG</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_tag</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_name</source_field>
+ <source_transform>infos</source_transform>
<target_attribute_key>INFO_TRANSFORM_NAME</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_name</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_description</source_field>
+ <source_transform>infos</source_transform>
<target_attribute_key>INFO_DESCRIPTION</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_description</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>parameter_description</source_field>
+ <source_transform>parameters</source_transform>
<target_attribute_key>DESCRIPTION</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_description</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
+ <source_field>transform_tag</source_field>
+ <source_transform>targets</source_transform>
<target_attribute_key>TARGET_TAG</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_tag</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
<mapping>
+ <source_field>field_name</source_field>
+ <source_transform>fields with Strings</source_transform>
+ <target_attribute_key>FIELD_NAME</target_attribute_key>
+ <target_detail>Y</target_detail>
<target_transform_name>User defined Java class</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>field_type</source_field>
+ <source_transform>fields with Strings</source_transform>
<target_attribute_key>FIELD_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_type</source_field>
+ <target_transform_name>User defined Java class</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>field_length</source_field>
+ <source_transform>fields with Strings</source_transform>
+ <target_attribute_key>FIELD_LENGTH</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java class</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>field_precision</source_field>
+ <source_transform>fields with Strings</source_transform>
+ <target_attribute_key>FIELD_PRECISION</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java class</target_transform_name>
</mapping>
</mappings>
+ <no_execution>Y</no_execution>
+ <run_configuration>local</run_configuration>
+ <source_output_fields>
+</source_output_fields>
+ <source_transform/>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <target_file>${java.io.tmpdir}/user-defined-java-class.hpl</target_file>
<attributes/>
<GUI>
<xloc>432</xloc>
@@ -320,11 +316,9 @@ limitations under the License.
<IsAFile>N</IsAFile>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<IsInFields>N</IsInFields>
- <XmlField/>
<addresultfile>N</addresultfile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
<encoding>UTF-8</encoding>
- <extensionFieldName/>
<fields>
<field>
<currency/>
@@ -388,29 +382,17 @@ limitations under the License.
</field>
</fields>
<file>
- <exclude_filemask/>
<file_required>N</file_required>
- <filemask/>
<include_subfolders>N</include_subfolders>
<name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
</file>
- <hiddenFieldName/>
<ignorecomments>N</ignorecomments>
<include>N</include>
- <include_field/>
- <lastModificationTimeFieldName/>
<limit>0</limit>
<loopxpath>/transform/fields/field</loopxpath>
<namespaceaware>N</namespaceaware>
- <pathFieldName/>
- <prunePath/>
<readurl>N</readurl>
- <rootUriNameFieldName/>
<rownum>N</rownum>
- <rownum_field/>
- <shortFileFieldName/>
- <sizeFieldName/>
- <uriNameFieldName/>
<usetoken>N</usetoken>
<validating>N</validating>
<attributes/>
@@ -725,49 +707,6 @@ limitations under the License.
<yloc>304</yloc>
</GUI>
</transform>
- <transform>
- <name>fields</name>
- <type>ValueMapper</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <field_to_use>field_type</field_to_use>
- <fields>
- <field>
- <source_value>Number</source_value>
- <target_value>1</target_value>
- </field>
- <field>
- <source_value>String</source_value>
- <target_value>2</target_value>
- </field>
- <field>
- <source_value>Date</source_value>
- <target_value>3</target_value>
- </field>
- <field>
- <source_value>Boolean</source_value>
- <target_value>4</target_value>
- </field>
- <field>
- <source_value>Integer</source_value>
- <target_value>5</target_value>
- </field>
- </fields>
- <non_match_default/>
- <target_field/>
- <target_type/>
- <attributes/>
- <GUI>
- <xloc>208</xloc>
- <yloc>176</yloc>
- </GUI>
- </transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
diff --git a/integration-tests/mdi/0042-user-defined-java-expression.hpl b/integration-tests/mdi/0042-user-defined-java-expression.hpl
index 825d26273b..e3bfda31ef 100644
--- a/integration-tests/mdi/0042-user-defined-java-expression.hpl
+++ b/integration-tests/mdi/0042-user-defined-java-expression.hpl
@@ -44,6 +44,78 @@ limitations under the License.
<enabled>Y</enabled>
</hop>
</order>
+ <transform>
+ <name>0042-user-defined-java-expression-template.hpl</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <create_parent_folder>Y</create_parent_folder>
+ <filename>${PROJECT_HOME}/0042-user-defined-java-expression-template.hpl</filename>
+ <mappings>
+ <mapping>
+ <source_field>replace_field</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>REPLACE_FIELD</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>field_name</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>FIELD_NAME</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>value_length</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>VALUE_LENGTH</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>value_precision</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>VALUE_PRECISION</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>value_type</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>VALUE_TYPE</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>formula_string</source_field>
+ <source_transform>files/janino.xml fields</source_transform>
+ <target_attribute_key>FIELD_FORMULA</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <target_transform_name>User defined Java expression</target_transform_name>
+ </mapping>
+ </mappings>
+ <no_execution>Y</no_execution>
+ <run_configuration>local</run_configuration>
+ <source_output_fields>
+</source_output_fields>
+ <source_transform/>
+ <stream_source_transform/>
+ <stream_target_transform/>
+ <target_file>${java.io.tmpdir}/janino.hpl</target_file>
+ <attributes/>
+ <GUI>
+ <xloc>416</xloc>
+ <yloc>64</yloc>
+ </GUI>
+ </transform>
<transform>
<name>files/janino.xml fields</name>
<type>getXMLData</type>
@@ -58,11 +130,9 @@ limitations under the License.
<IsAFile>N</IsAFile>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<IsInFields>N</IsInFields>
- <XmlField/>
<addresultfile>N</addresultfile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
<encoding>UTF-8</encoding>
- <extensionFieldName/>
<fields>
<field>
<currency/>
@@ -156,29 +226,17 @@ limitations under the License.
</field>
</fields>
<file>
- <exclude_filemask/>
<file_required>N</file_required>
- <filemask/>
<include_subfolders>N</include_subfolders>
<name>${PROJECT_HOME}/files/janino.xml</name>
</file>
- <hiddenFieldName/>
<ignorecomments>N</ignorecomments>
<include>N</include>
- <include_field/>
- <lastModificationTimeFieldName/>
<limit>0</limit>
<loopxpath>/transform/formula</loopxpath>
<namespaceaware>N</namespaceaware>
- <pathFieldName/>
- <prunePath/>
<readurl>N</readurl>
- <rootUriNameFieldName/>
<rownum>N</rownum>
- <rownum_field/>
- <shortFileFieldName/>
- <sizeFieldName/>
- <uriNameFieldName/>
<usetoken>N</usetoken>
<validating>N</validating>
<attributes/>
@@ -187,77 +245,6 @@ limitations under the License.
<yloc>64</yloc>
</GUI>
</transform>
- <transform>
- <name>0042-user-defined-java-expression-template.hpl</name>
- <type>MetaInject</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <filename>${PROJECT_HOME}/0042-user-defined-java-expression-template.hpl</filename>
- <run_configuration>local</run_configuration>
- <source_transform/>
- <source_output_fields> </source_output_fields>
- <target_file>${java.io.tmpdir}/janino.hpl</target_file>
- <create_parent_folder>Y</create_parent_folder>
- <no_execution>Y</no_execution>
- <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
- <stream_source_transform/>
- <stream_target_transform/>
- <mappings>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>REPLACE_FIELD</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>replace_field</source_field>
- </mapping>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>FIELD_NAME</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>field_name</source_field>
- </mapping>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>VALUE_LENGTH</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>value_length</source_field>
- </mapping>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>VALUE_PRECISION</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>value_precision</source_field>
- </mapping>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>VALUE_TYPE</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>value_type</source_field>
- </mapping>
- <mapping>
- <target_transform_name>User defined Java expression</target_transform_name>
- <target_attribute_key>FIELD_FORMULA</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>files/janino.xml fields</source_transform>
- <source_field>formula_string</source_field>
- </mapping>
- </mappings>
- <attributes/>
- <GUI>
- <xloc>320</xloc>
- <yloc>64</yloc>
- </GUI>
- </transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
diff --git a/integration-tests/mdi/0045-meta-inject-template.hpl b/integration-tests/mdi/0045-meta-inject-template.hpl
new file mode 100644
index 0000000000..e82f8551da
--- /dev/null
+++ b/integration-tests/mdi/0045-meta-inject-template.hpl
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0045-meta-inject-template</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/19 18:04:55.510</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/19 18:04:55.510</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ </order>
+ <transform>
+ <name>ETL metadata injection</name>
+ <type>MetaInject</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <create_parent_folder>Y</create_parent_folder>
+ <mappings>
+</mappings>
+ <no_execution>N</no_execution>
+ <source_output_fields>
+</source_output_fields>
+ <attributes/>
+ <GUI>
+ <xloc>192</xloc>
+ <yloc>160</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0045-meta-inject-valiate.hpl b/integration-tests/mdi/0045-meta-inject-valiate.hpl
new file mode 100644
index 0000000000..d198519044
--- /dev/null
+++ b/integration-tests/mdi/0045-meta-inject-valiate.hpl
@@ -0,0 +1,561 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0045-meta-inject-valiate</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+ <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+ <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/03/23 21:19:47.909</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/23 21:19:47.909</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>/tmp/meta-inject.hpl mappings</from>
+ <to>validate mappings</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>/tmp/meta-inject.hpl fields</from>
+ <to>validate fields</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>/tmp/meta-inject.hpl</from>
+ <to>validate</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>/tmp/meta-inject.hpl mappings</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_field</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_field</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_transform</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_transform</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>target_attribute_key</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>target_attribute_key</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>target_detail</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>target_detail</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>target_transform_name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>target_transform_name</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/meta-inject.hpl</name>
+ </file>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name/>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform/mappings/mapping</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>160</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate mappings</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>112</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>/tmp/meta-inject.hpl fields</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_output_field_length</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_output_field_length</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_output_field_name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_output_field_name</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_output_field_precision</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_output_field_precision</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_output_field_type</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_output_field_type</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/meta-inject.hpl</name>
+ </file>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name/>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform/source_output_fields/source_output_field</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>160</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate fields</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>208</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>/tmp/meta-inject.hpl</name>
+ <type>getXMLData</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <IsAFile>N</IsAFile>
+ <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
+ <IsInFields>N</IsInFields>
+ <XmlField/>
+ <addresultfile>N</addresultfile>
+ <doNotFailIfNoFile>Y</doNotFailIfNoFile>
+ <encoding>UTF-8</encoding>
+ <extensionFieldName/>
+ <fields>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>allow_empty_stream_on_execution</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>allow_empty_stream_on_execution</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>create_parent_folder</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>create_parent_folder</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>filename</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>filename</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>no_execution</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>no_execution</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>run_configuration</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>run_configuration</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_transform</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_transform</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>stream_source_transform</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>stream_source_transform</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>stream_target_transform</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>stream_target_transform</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>target_file</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>target_file</xPath>
+ </field>
+ </fields>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${java.io.tmpdir}/meta-inject.hpl</name>
+ </file>
+ <file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name/>
+ </file>
+ <hiddenFieldName/>
+ <ignorecomments>N</ignorecomments>
+ <include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
+ <limit>0</limit>
+ <loopxpath>/pipeline/transform</loopxpath>
+ <namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
+ <readurl>N</readurl>
+ <rootUriNameFieldName/>
+ <rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
+ <usetoken>N</usetoken>
+ <validating>N</validating>
+ <attributes/>
+ <GUI>
+ <xloc>160</xloc>
+ <yloc>304</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>validate</name>
+ <type>Dummy</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ <GUI>
+ <xloc>432</xloc>
+ <yloc>304</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/mdi/0042-user-defined-java-class.hpl b/integration-tests/mdi/0045-meta-inject.hpl
similarity index 51%
copy from integration-tests/mdi/0042-user-defined-java-class.hpl
copy to integration-tests/mdi/0045-meta-inject.hpl
index 0494fd062f..bae83e9cc2 100644
--- a/integration-tests/mdi/0042-user-defined-java-class.hpl
+++ b/integration-tests/mdi/0045-meta-inject.hpl
@@ -19,7 +19,7 @@ limitations under the License.
-->
<pipeline>
<info>
- <name>0042-user-defined-java-class</name>
+ <name>0045-meta-inject</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@@ -31,51 +31,31 @@ limitations under the License.
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
- <created_date>2026/03/17 11:27:28.469</created_date>
+ <created_date>2026/03/19 18:05:38.185</created_date>
<modified_user>-</modified_user>
- <modified_date>2026/03/17 11:27:28.469</modified_date>
+ <modified_date>2026/03/19 18:05:38.185</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
- <from>definitions</from>
- <to>0042-user-defined-java-class-template.hpl</to>
+ <from>files/meta-inject.xml mappings/mapping</from>
+ <to>0045-meta-inject-template.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
- <from>infos</from>
- <to>0042-user-defined-java-class-template.hpl</to>
+ <from>files/meta-inject.xml</from>
+ <to>0045-meta-inject-template.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
- <from>targets</from>
- <to>0042-user-defined-java-class-template.hpl</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>parameters</from>
- <to>0042-user-defined-java-class-template.hpl</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>files/user-defined-java-class.xml</from>
- <to>0042-user-defined-java-class-template.hpl</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>fields with Strings</from>
- <to>fields</to>
- <enabled>Y</enabled>
- </hop>
- <hop>
- <from>fields</from>
- <to>0042-user-defined-java-class-template.hpl</to>
+ <from>files/meta-inject.xml field</from>
+ <to>0045-meta-inject-template.hpl</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
- <name>0042-user-defined-java-class-template.hpl</name>
+ <name>0045-meta-inject-template.hpl</name>
<type>MetaInject</type>
<description/>
<distribute>Y</distribute>
@@ -85,145 +65,150 @@ limitations under the License.
<method>none</method>
<schema_name/>
</partitioning>
- <filename>${PROJECT_HOME}/0042-user-defined-java-class-template.hpl</filename>
- <run_configuration>local</run_configuration>
- <source_transform/>
- <source_output_fields> </source_output_fields>
- <target_file>${java.io.tmpdir}/user-defined-java-class.hpl</target_file>
- <create_parent_folder>Y</create_parent_folder>
- <no_execution>Y</no_execution>
<allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
- <stream_source_transform/>
- <stream_target_transform/>
+ <create_parent_folder>Y</create_parent_folder>
+ <filename>${PROJECT_HOME}/0045-meta-inject-template.hpl</filename>
<mappings>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>CLASS_NAME</target_attribute_key>
+ <source_field>source_output_field_name</source_field>
+ <source_transform>files/meta-inject.xml field</source_transform>
+ <target_attribute_key>SOURCE_OUTPUT_NAME</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_name</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>CLASS_SOURCE</target_attribute_key>
+ <source_field>source_output_field_type</source_field>
+ <source_transform>files/meta-inject.xml field</source_transform>
+ <target_attribute_key>SOURCE_OUTPUT_TYPE</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_source</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_LENGTH</target_attribute_key>
+ <source_field>source_output_field_length</source_field>
+ <source_transform>files/meta-inject.xml field</source_transform>
+ <target_attribute_key>SOURCE_OUTPUT_LENGTH</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_length</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>VALUE</target_attribute_key>
+ <source_field>source_output_field_precision</source_field>
+ <source_transform>files/meta-inject.xml field</source_transform>
+ <target_attribute_key>SOURCE_OUTPUT_PRECISION</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_value</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_PRECISION</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_precision</source_field>
+ <source_field>filename</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>FILE_NAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>CLEAR_RESULT_FIELDS</target_attribute_key>
+ <source_field>run_configuration</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>RUN_CONFIGURATION</target_attribute_key>
<target_detail>N</target_detail>
- <source_transform>files/user-defined-java-class.xml</source_transform>
- <source_field>clear_result_fields</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>TARGET_DESCRIPTION</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_description</source_field>
+ <source_field>source_transform</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>SOURCE_TRANSFORM_NAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>CLASS_TYPE</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>definitions</source_transform>
- <source_field>class_type</source_field>
+ <source_field>target_file</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>TARGET_FILE</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>TARGET_TRANSFORM_NAME</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_name</source_field>
+ <source_field>create_parent_folder</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>CREATE_PARENT_FOLDER</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>INFO_TAG</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_tag</source_field>
+ <source_field>no_execution</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>NO_EXECUTION</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_NAME</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_name</source_field>
+ <source_field>allow_empty_stream_on_execution</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>ALLOW_EMPTY_STREAM_ON_EXECUTION</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>TAG</target_attribute_key>
- <target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_tag</source_field>
+ <source_field>stream_source_transform</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>STREAMING_SOURCE_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
+ </mapping>
+ <mapping>
+ <source_field>stream_target_transform</source_field>
+ <source_transform>files/meta-inject.xml</source_transform>
+ <target_attribute_key>STREAMING_TARGET_TRANSFORM</target_attribute_key>
+ <target_detail>N</target_detail>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>INFO_TRANSFORM_NAME</target_attribute_key>
+ <source_field>source_field</source_field>
+ <source_transform>files/meta-inject.xml mappings/mapping</source_transform>
+ <target_attribute_key>MAPPING_SOURCE_FIELD</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_name</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>INFO_DESCRIPTION</target_attribute_key>
+ <source_field>source_transform</source_field>
+ <source_transform>files/meta-inject.xml mappings/mapping</source_transform>
+ <target_attribute_key>MAPPING_SOURCE_TRANSFORM</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>infos</source_transform>
- <source_field>transform_description</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>DESCRIPTION</target_attribute_key>
+ <source_field>target_detail</source_field>
+ <source_transform>files/meta-inject.xml mappings/mapping</source_transform>
+ <target_attribute_key>MAPPING_TARGET_DETAIL</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>parameters</source_transform>
- <source_field>parameter_description</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>TARGET_TAG</target_attribute_key>
+ <source_field>target_transform_name</source_field>
+ <source_transform>files/meta-inject.xml mappings/mapping</source_transform>
+ <target_attribute_key>MAPPING_TARGET_TRANSFORM</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>targets</source_transform>
- <source_field>transform_tag</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
<mapping>
- <target_transform_name>User defined Java class</target_transform_name>
- <target_attribute_key>FIELD_TYPE</target_attribute_key>
+ <source_field>target_attribute_key</source_field>
+ <source_transform>files/meta-inject.xml mappings/mapping</source_transform>
+ <target_attribute_key>MAPPING_TARGET_FIELD</target_attribute_key>
<target_detail>Y</target_detail>
- <source_transform>fields</source_transform>
- <source_field>field_type</source_field>
+ <target_transform_name>ETL metadata injection</target_transform_name>
</mapping>
</mappings>
+ <no_execution>Y</no_execution>
+ <run_configuration>local</run_configuration>
+ <source_output_fields>
+</source_output_fields>
+ <target_file>${java.io.tmpdir}/meta-inject.hpl</target_file>
<attributes/>
<GUI>
- <xloc>432</xloc>
- <yloc>176</yloc>
+ <xloc>352</xloc>
+ <yloc>48</yloc>
</GUI>
</transform>
<transform>
- <name>definitions</name>
+ <name>files/meta-inject.xml</name>
<type>getXMLData</type>
<description/>
<distribute>Y</distribute>
@@ -247,13 +232,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>class_type</name>
+ <name>filename</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>class_type</xPath>
+ <xPath>filename</xPath>
</field>
<field>
<currency/>
@@ -262,13 +247,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>class_name</name>
+ <name>run_configuration</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>class_name</xPath>
+ <xPath>run_configuration</xPath>
</field>
<field>
<currency/>
@@ -277,55 +262,14 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>class_source</name>
+ <name>source_transform</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>class_source</xPath>
+ <xPath>source_transform</xPath>
</field>
- </fields>
- <file>
- <file_required>N</file_required>
- <include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
- </file>
- <ignorecomments>N</ignorecomments>
- <include>N</include>
- <limit>0</limit>
- <loopxpath>/transform/definitions/definition</loopxpath>
- <namespaceaware>N</namespaceaware>
- <readurl>N</readurl>
- <rownum>N</rownum>
- <usetoken>N</usetoken>
- <validating>N</validating>
- <attributes/>
- <GUI>
- <xloc>96</xloc>
- <yloc>112</yloc>
- </GUI>
- </transform>
- <transform>
- <name>fields with Strings</name>
- <type>getXMLData</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <IsAFile>N</IsAFile>
- <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
- <IsInFields>N</IsInFields>
- <XmlField/>
- <addresultfile>N</addresultfile>
- <doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <encoding>UTF-8</encoding>
- <extensionFieldName/>
- <fields>
<field>
<currency/>
<decimal/>
@@ -333,13 +277,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>field_name</name>
+ <name>target_file</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>field_name</xPath>
+ <xPath>target_file</xPath>
</field>
<field>
<currency/>
@@ -348,13 +292,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>field_type</name>
+ <name>create_parent_folder</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>field_type</xPath>
+ <xPath>create_parent_folder</xPath>
</field>
<field>
<currency/>
@@ -363,13 +307,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>field_length</name>
+ <name>no_execution</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
- <type>Integer</type>
- <xPath>field_length</xPath>
+ <type>String</type>
+ <xPath>no_execution</xPath>
</field>
<field>
<currency/>
@@ -378,65 +322,14 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>field_precision</name>
+ <name>allow_empty_stream_on_execution</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
- <type>Integer</type>
- <xPath>field_precision</xPath>
+ <type>String</type>
+ <xPath>allow_empty_stream_on_execution</xPath>
</field>
- </fields>
- <file>
- <exclude_filemask/>
- <file_required>N</file_required>
- <filemask/>
- <include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
- </file>
- <hiddenFieldName/>
- <ignorecomments>N</ignorecomments>
- <include>N</include>
- <include_field/>
- <lastModificationTimeFieldName/>
- <limit>0</limit>
- <loopxpath>/transform/fields/field</loopxpath>
- <namespaceaware>N</namespaceaware>
- <pathFieldName/>
- <prunePath/>
- <readurl>N</readurl>
- <rootUriNameFieldName/>
- <rownum>N</rownum>
- <rownum_field/>
- <shortFileFieldName/>
- <sizeFieldName/>
- <uriNameFieldName/>
- <usetoken>N</usetoken>
- <validating>N</validating>
- <attributes/>
- <GUI>
- <xloc>96</xloc>
- <yloc>176</yloc>
- </GUI>
- </transform>
- <transform>
- <name>files/user-defined-java-class.xml</name>
- <type>getXMLData</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <IsAFile>N</IsAFile>
- <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
- <IsInFields>N</IsInFields>
- <addresultfile>N</addresultfile>
- <doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <encoding>UTF-8</encoding>
- <fields>
<field>
<currency/>
<decimal/>
@@ -444,23 +337,42 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>clear_result_fields</name>
+ <name>stream_source_transform</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>clear_result_fields</xPath>
+ <xPath>stream_source_transform</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>stream_target_transform</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>stream_target_transform</xPath>
</field>
</fields>
<file>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
+ <name>${PROJECT_HOME}/files/meta-inject.xml</name>
+ </file>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
</file>
<ignorecomments>N</ignorecomments>
<include>N</include>
- <limit>0</limit>
+ <limit>5</limit>
<loopxpath>/transform</loopxpath>
<namespaceaware>N</namespaceaware>
<readurl>N</readurl>
@@ -474,7 +386,7 @@ limitations under the License.
</GUI>
</transform>
<transform>
- <name>infos</name>
+ <name>files/meta-inject.xml field</name>
<type>getXMLData</type>
<description/>
<distribute>Y</distribute>
@@ -498,13 +410,28 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_tag</name>
+ <name>source_output_field_name</name>
+ <precision>-1</precision>
+ <repeat>N</repeat>
+ <result_type>valueof</result_type>
+ <trim_type>none</trim_type>
+ <type>String</type>
+ <xPath>source_output_field_name</xPath>
+ </field>
+ <field>
+ <currency/>
+ <decimal/>
+ <element_type>Node</element_type>
+ <format/>
+ <group/>
+ <length>-1</length>
+ <name>source_output_field_type</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_tag</xPath>
+ <xPath>source_output_field_type</xPath>
</field>
<field>
<currency/>
@@ -513,13 +440,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_name</name>
+ <name>source_output_field_length</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_name</xPath>
+ <xPath>source_output_field_length</xPath>
</field>
<field>
<currency/>
@@ -528,24 +455,28 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_description</name>
+ <name>source_output_field_precision</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_description</xPath>
+ <xPath>source_output_field_precision</xPath>
</field>
</fields>
<file>
<file_required>N</file_required>
<include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
+ <name>${PROJECT_HOME}/files/meta-inject.xml</name>
+ </file>
+ <file>
+ <file_required>N</file_required>
+ <include_subfolders>N</include_subfolders>
</file>
<ignorecomments>N</ignorecomments>
<include>N</include>
- <limit>0</limit>
- <loopxpath>/transform/info_transforms/info_transform</loopxpath>
+ <limit>5</limit>
+ <loopxpath>/transform/source_output_fields/source_output_field</loopxpath>
<namespaceaware>N</namespaceaware>
<readurl>N</readurl>
<rownum>N</rownum>
@@ -553,12 +484,12 @@ limitations under the License.
<validating>N</validating>
<attributes/>
<GUI>
- <xloc>96</xloc>
- <yloc>240</yloc>
+ <xloc>128</xloc>
+ <yloc>128</yloc>
</GUI>
</transform>
<transform>
- <name>parameters</name>
+ <name>files/meta-inject.xml mappings/mapping</name>
<type>getXMLData</type>
<description/>
<distribute>Y</distribute>
@@ -571,9 +502,11 @@ limitations under the License.
<IsAFile>N</IsAFile>
<IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
<IsInFields>N</IsInFields>
+ <XmlField/>
<addresultfile>N</addresultfile>
<doNotFailIfNoFile>Y</doNotFailIfNoFile>
<encoding>UTF-8</encoding>
+ <extensionFieldName/>
<fields>
<field>
<currency/>
@@ -582,28 +515,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>parameter_tag</name>
- <precision>-1</precision>
- <repeat>N</repeat>
- <result_type>valueof</result_type>
- <trim_type>none</trim_type>
- <type>String</type>
- <xPath>parameter_tag</xPath>
- </field>
- <field>
- <currency/>
- <decimal/>
- <element_type>Node</element_type>
- <format/>
- <group/>
- <length>-1</length>
- <name>parameter_value</name>
+ <name>target_transform_name</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>parameter_value</xPath>
+ <xPath>target_transform_name</xPath>
</field>
<field>
<currency/>
@@ -612,53 +530,14 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>parameter_description</name>
+ <name>target_attribute_key</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>parameter_description</xPath>
+ <xPath>target_attribute_key</xPath>
</field>
- </fields>
- <file>
- <file_required>N</file_required>
- <include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
- </file>
- <ignorecomments>N</ignorecomments>
- <include>N</include>
- <limit>0</limit>
- <loopxpath>/transform/usage_parameters/usage_parameter</loopxpath>
- <namespaceaware>N</namespaceaware>
- <readurl>N</readurl>
- <rownum>N</rownum>
- <usetoken>N</usetoken>
- <validating>N</validating>
- <attributes/>
- <GUI>
- <xloc>96</xloc>
- <yloc>368</yloc>
- </GUI>
- </transform>
- <transform>
- <name>targets</name>
- <type>getXMLData</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <IsAFile>N</IsAFile>
- <IsIgnoreEmptyFile>N</IsIgnoreEmptyFile>
- <IsInFields>N</IsInFields>
- <addresultfile>N</addresultfile>
- <doNotFailIfNoFile>Y</doNotFailIfNoFile>
- <encoding>UTF-8</encoding>
- <fields>
<field>
<currency/>
<decimal/>
@@ -666,13 +545,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_tag</name>
+ <name>target_detail</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_tag</xPath>
+ <xPath>target_detail</xPath>
</field>
<field>
<currency/>
@@ -681,13 +560,13 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_name</name>
+ <name>source_transform</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_name</xPath>
+ <xPath>source_transform</xPath>
</field>
<field>
<currency/>
@@ -696,76 +575,52 @@ limitations under the License.
<format/>
<group/>
<length>-1</length>
- <name>transform_description</name>
+ <name>source_field</name>
<precision>-1</precision>
<repeat>N</repeat>
<result_type>valueof</result_type>
<trim_type>none</trim_type>
<type>String</type>
- <xPath>transform_description</xPath>
+ <xPath>source_field</xPath>
</field>
</fields>
<file>
+ <exclude_filemask/>
+ <file_required>N</file_required>
+ <filemask/>
+ <include_subfolders>N</include_subfolders>
+ <name>${PROJECT_HOME}/files/meta-inject.xml</name>
+ </file>
+ <file>
+ <exclude_filemask/>
<file_required>N</file_required>
+ <filemask/>
<include_subfolders>N</include_subfolders>
- <name>${PROJECT_HOME}/files/user-defined-java-class.xml</name>
+ <name/>
</file>
+ <hiddenFieldName/>
<ignorecomments>N</ignorecomments>
<include>N</include>
+ <include_field/>
+ <lastModificationTimeFieldName/>
<limit>0</limit>
- <loopxpath>/transform/target_transforms/target_transform</loopxpath>
+ <loopxpath>/transform/mappings/mapping</loopxpath>
<namespaceaware>N</namespaceaware>
+ <pathFieldName/>
+ <prunePath/>
<readurl>N</readurl>
+ <rootUriNameFieldName/>
<rownum>N</rownum>
+ <rownum_field/>
+ <shortFileFieldName/>
+ <sizeFieldName/>
+ <uriNameFieldName/>
<usetoken>N</usetoken>
<validating>N</validating>
<attributes/>
- <GUI>
- <xloc>96</xloc>
- <yloc>304</yloc>
- </GUI>
- </transform>
- <transform>
- <name>fields</name>
- <type>ValueMapper</type>
- <description/>
- <distribute>Y</distribute>
- <custom_distribution/>
- <copies>1</copies>
- <partitioning>
- <method>none</method>
- <schema_name/>
- </partitioning>
- <field_to_use>field_type</field_to_use>
- <fields>
- <field>
- <source_value>Number</source_value>
- <target_value>1</target_value>
- </field>
- <field>
- <source_value>String</source_value>
- <target_value>2</target_value>
- </field>
- <field>
- <source_value>Date</source_value>
- <target_value>3</target_value>
- </field>
- <field>
- <source_value>Boolean</source_value>
- <target_value>4</target_value>
- </field>
- <field>
- <source_value>Integer</source_value>
- <target_value>5</target_value>
- </field>
- </fields>
- <non_match_default/>
- <target_field/>
- <target_type/>
- <attributes/>
<GUI>
<xloc>208</xloc>
- <yloc>176</yloc>
+ <yloc>192</yloc>
</GUI>
</transform>
<transform_error_handling>
diff --git a/integration-tests/mdi/datasets/golden-meta-inject-fields.csv b/integration-tests/mdi/datasets/golden-meta-inject-fields.csv
new file mode 100644
index 0000000000..78e10f8810
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-meta-inject-fields.csv
@@ -0,0 +1,3 @@
+source_output_field_name,source_output_field_type,source_output_field_length,source_output_field_precision
+f1,String,100,-1
+f2,Number,7,2
diff --git a/integration-tests/mdi/datasets/golden-meta-inject-mappings.csv b/integration-tests/mdi/datasets/golden-meta-inject-mappings.csv
new file mode 100644
index 0000000000..ce07e6ec28
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-meta-inject-mappings.csv
@@ -0,0 +1,39 @@
+target_transform_name,target_attribute_key,target_detail,source_transform,source_field
+JSON input,IGNORE_EMPTY_FILE,N,files/json-input.xml,IsIgnoreEmptyFile
+JSON input,FILE_PATH_FIELDNAME,N,files/json-input.xml,pathFieldName
+JSON input,FIELD_CURRENCY,Y,files/json-input.xml fields,currency
+JSON input,FILE_SHORT_FILE_FIELDNAME,N,files/json-input.xml,shortFileFieldName
+JSON input,FIELD_TYPE,Y,files/json-input.xml fields,type
+JSON input,FILEMASK,Y,files/json-input.xml files,filemask
+JSON input,DO_NOT_FAIL_IF_NO_FILE,N,files/json-input.xml,doNotFailIfNoFile
+JSON input,EXCLUDE_FILEMASK,Y,files/json-input.xml files,exclude_filemask
+JSON input,FIELD_PATH,Y,files/json-input.xml fields,path
+JSON input,FIELD_PRECISION,Y,files/json-input.xml fields,precision
+JSON input,FILE_HIDDEN_FIELDNAME,N,files/json-input.xml,hiddenFieldName
+JSON input,READ_SOURCE_AS_URL,N,files/json-input.xml,readurl
+JSON input,SOURCE_FIELD_NAME,N,files/json-input.xml,valueField
+JSON input,FIELD_FORMAT,Y,files/json-input.xml fields,format
+JSON input,FIELD_REPEAT,Y,files/json-input.xml fields,repeat
+JSON input,FIELD_GROUP,Y,files/json-input.xml fields,group
+JSON input,FILE_LAST_MODIFICATION_FIELDNAME,N,files/json-input.xml,lastModificationTimeFieldName
+JSON input,ROW_NUMBER_OUTPUT,N,files/json-input.xml,rownum
+JSON input,FILE_URI_FIELDNAME,N,files/json-input.xml,uriNameFieldName
+JSON input,FILE_ROOT_URI_FIELDNAME,N,files/json-input.xml,rootUriNameFieldName
+JSON input,FIELD_DECIMAL,Y,files/json-input.xml fields,decimal
+JSON input,FIELD_LENGTH,Y,files/json-input.xml fields,length
+JSON input,ROW_NUMBER_FIELDNAME,N,files/json-input.xml,rownum_field
+JSON input,SOURCE_FIELD_IS_FILENAME,N,files/json-input.xml,IsAFile
+JSON input,FIELD_TRIM_TYPE,Y,files/json-input.xml fields,trim_type
+JSON input,FILENAME,Y,files/json-input.xml files,name
+JSON input,FIELD_NAME,Y,files/json-input.xml fields,name
+JSON input,FILE_EXTENSION_FIELDNAME,N,files/json-input.xml,extensionFieldName
+JSON input,SOURCE_IN_FIELD,N,files/json-input.xml,IsInFields
+JSON input,INCLUDE_SUBFOLDERS,Y,files/json-input.xml files,include_subfolders
+JSON input,ADD_RESULT_FILE,N,files/json-input.xml,addresultfile
+JSON input,REMOVE_SOURCE_FIELDS,N,files/json-input.xml,removeSourceField
+JSON input,FILE_NAME_OUTPUT,N,files/json-input.xml,include
+JSON input,ROW_LIMIT,N,files/json-input.xml,limit
+JSON input,IGNORE_MISSING_PATH,N,files/json-input.xml,ignoreMissingPath
+JSON input,FILE_NAME_FIELDNAME,N,files/json-input.xml,include_field
+JSON input,FILE_SIZE_FIELDNAME,N,files/json-input.xml,sizeFieldName
+JSON input,FILE_REQUIRED,Y,files/json-input.xml files,file_required
diff --git a/integration-tests/mdi/datasets/golden-meta-inject.csv b/integration-tests/mdi/datasets/golden-meta-inject.csv
new file mode 100644
index 0000000000..a6eccc00dd
--- /dev/null
+++ b/integration-tests/mdi/datasets/golden-meta-inject.csv
@@ -0,0 +1,2 @@
+filename,run_configuration,source_transform,target_file,create_parent_folder,no_execution,allow_empty_stream_on_execution,stream_source_transform,stream_target_transform
+${PROJECT_HOME}/0038-json-input-template.hpl,local,sourceTransform,${java.io.tmpdir}/json-input.hpl,Y,Y,N,sourceTransform,targetTransform
diff --git a/integration-tests/mdi/files/excelwriter-mdi-test.xlsx b/integration-tests/mdi/files/excelwriter-mdi-test.xlsx
index b2e50bac77..ef8f12feb6 100644
Binary files a/integration-tests/mdi/files/excelwriter-mdi-test.xlsx and b/integration-tests/mdi/files/excelwriter-mdi-test.xlsx differ
diff --git a/integration-tests/mdi/files/janino.xml b/integration-tests/mdi/files/janino.xml
index 6abc2f9d84..08509604ae 100644
--- a/integration-tests/mdi/files/janino.xml
+++ b/integration-tests/mdi/files/janino.xml
@@ -33,7 +33,7 @@
<replace_field>replace2</replace_field>
</formula>
<formula>
- <field_name>f3<Get rows from result XML cleanup/field_name>
+ <field_name>f3</field_name>
<formula_string>expression3</formula_string>
<value_type>Number</value_type>
<value_length>9</value_length>
diff --git a/integration-tests/mdi/files/meta-inject.xml b/integration-tests/mdi/files/meta-inject.xml
new file mode 100644
index 0000000000..5e3bd0b99d
--- /dev/null
+++ b/integration-tests/mdi/files/meta-inject.xml
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<transform>
+ <filename>${PROJECT_HOME}/0038-json-input-template.hpl</filename>
+ <run_configuration>local</run_configuration>
+ <source_transform>sourceTransform</source_transform>
+ <source_output_fields>
+ <source_output_field>
+ <source_output_field_name>f1</source_output_field_name>
+ <source_output_field_type>String</source_output_field_type>
+ <source_output_field_length>100</source_output_field_length>
+ <source_output_field_precision>-1</source_output_field_precision>
+ </source_output_field>
+ <source_output_field>
+ <source_output_field_name>f2</source_output_field_name>
+ <source_output_field_type>Number</source_output_field_type>
+ <source_output_field_length>7</source_output_field_length>
+ <source_output_field_precision>2</source_output_field_precision>
+ </source_output_field>
+ </source_output_fields>
+ <target_file>${java.io.tmpdir}/json-input.hpl</target_file>
+ <create_parent_folder>Y</create_parent_folder>
+ <no_execution>Y</no_execution>
+ <allow_empty_stream_on_execution>N</allow_empty_stream_on_execution>
+ <stream_source_transform>sourceTransform</stream_source_transform>
+ <stream_target_transform>targetTransform</stream_target_transform>
+ <mappings>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>IGNORE_EMPTY_FILE</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>IsIgnoreEmptyFile</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_PATH_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>pathFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_CURRENCY</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>currency</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_SHORT_FILE_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>shortFileFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_TYPE</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>type</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILEMASK</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml files</source_transform>
+ <source_field>filemask</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>DO_NOT_FAIL_IF_NO_FILE</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>doNotFailIfNoFile</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>EXCLUDE_FILEMASK</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml files</source_transform>
+ <source_field>exclude_filemask</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_PATH</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>path</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_PRECISION</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>precision</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_HIDDEN_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>hiddenFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>READ_SOURCE_AS_URL</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>readurl</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>SOURCE_FIELD_NAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>valueField</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_FORMAT</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>format</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_REPEAT</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>repeat</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_GROUP</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>group</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_LAST_MODIFICATION_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>lastModificationTimeFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>ROW_NUMBER_OUTPUT</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>rownum</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_URI_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>uriNameFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_ROOT_URI_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>rootUriNameFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_DECIMAL</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>decimal</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_LENGTH</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>length</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>ROW_NUMBER_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>rownum_field</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>SOURCE_FIELD_IS_FILENAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>IsAFile</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_TRIM_TYPE</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>trim_type</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILENAME</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml files</source_transform>
+ <source_field>name</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FIELD_NAME</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml fields</source_transform>
+ <source_field>name</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_EXTENSION_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>extensionFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>SOURCE_IN_FIELD</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>IsInFields</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>INCLUDE_SUBFOLDERS</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml files</source_transform>
+ <source_field>include_subfolders</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>ADD_RESULT_FILE</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>addresultfile</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>REMOVE_SOURCE_FIELDS</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>removeSourceField</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_NAME_OUTPUT</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>include</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>ROW_LIMIT</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>limit</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>IGNORE_MISSING_PATH</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>ignoreMissingPath</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_NAME_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>include_field</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_SIZE_FIELDNAME</target_attribute_key>
+ <target_detail>N</target_detail>
+ <source_transform>files/json-input.xml</source_transform>
+ <source_field>sizeFieldName</source_field>
+ </mapping>
+ <mapping>
+ <target_transform_name>JSON input</target_transform_name>
+ <target_attribute_key>FILE_REQUIRED</target_attribute_key>
+ <target_detail>Y</target_detail>
+ <source_transform>files/json-input.xml files</source_transform>
+ <source_field>file_required</source_field>
+ </mapping>
+ </mappings>
+</transform>
diff --git a/integration-tests/mdi/main-0042-user-defined-java-expression.hwf b/integration-tests/mdi/main-0042-user-defined-java-expression.hwf
index bbd6b91986..89f22acd57 100644
--- a/integration-tests/mdi/main-0042-user-defined-java-expression.hwf
+++ b/integration-tests/mdi/main-0042-user-defined-java-expression.hwf
@@ -49,8 +49,65 @@ limitations under the License.
<yloc>64</yloc>
<attributes_hac/>
</action>
+ <action>
+ <name>0042-user-defined-java-expression.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0042-user-defined-java-expression.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0042-user-defined-java-expression-validate UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>608</xloc>
+ <yloc>64</yloc>
+ <attributes_hac/>
+ </action>
</actions>
<hops>
+ <hop>
+ <from>Start</from>
+ <to>0042-user-defined-java-expression.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0042-user-defined-java-expression.hpl</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
</hops>
<notepads>
</notepads>
diff --git a/integration-tests/mdi/main-0045-meta-inject.hwf b/integration-tests/mdi/main-0045-meta-inject.hwf
new file mode 100644
index 0000000000..89478178cd
--- /dev/null
+++ b/integration-tests/mdi/main-0045-meta-inject.hwf
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0045-meta-inject</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/03/19 18:04:34.977</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/03/19 18:04:34.977</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>192</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0045-meta-inject</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0045-meta-inject.hpl</filename>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>368</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Run Pipeline Unit Tests</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0045-meta-inject-valiate UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>0045-meta-inject</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0045-meta-inject</from>
+ <to>Run Pipeline Unit Tests</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git a/integration-tests/mdi/main-0087-javascript.hwf b/integration-tests/mdi/main-0087-javascript.hwf
deleted file mode 100644
index 207028173a..0000000000
--- a/integration-tests/mdi/main-0087-javascript.hwf
+++ /dev/null
@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--->
-<workflow>
- <name>main-0087-javascript</name>
- <name_sync_with_filename>Y</name_sync_with_filename>
- <description/>
- <extended_description/>
- <workflow_version/>
- <created_user>-</created_user>
- <created_date>2026/03/18 19:32:06.360</created_date>
- <modified_user>-</modified_user>
- <modified_date>2026/03/18 19:32:06.360</modified_date>
- <parameters>
- </parameters>
- <actions>
- <action>
- <name>Start</name>
- <description/>
- <type>SPECIAL</type>
- <attributes/>
- <DayOfMonth>1</DayOfMonth>
- <doNotWaitOnFirstExecution>N</doNotWaitOnFirstExecution>
- <hour>12</hour>
- <intervalMinutes>60</intervalMinutes>
- <intervalSeconds>0</intervalSeconds>
- <minutes>0</minutes>
- <repeat>N</repeat>
- <schedulerType>0</schedulerType>
- <weekDay>1</weekDay>
- <parallel>N</parallel>
- <xloc>50</xloc>
- <yloc>50</yloc>
- <attributes_hac/>
- </action>
- </actions>
- <hops>
- </hops>
- <notepads>
- </notepads>
- <attributes/>
-</workflow>
diff --git a/integration-tests/mdi/metadata/dataset/golden-meta-inject--fields.json b/integration-tests/mdi/metadata/dataset/golden-meta-inject--fields.json
new file mode 100644
index 0000000000..b25805ebd9
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-meta-inject--fields.json
@@ -0,0 +1,40 @@
+{
+ "base_filename": "golden-meta-inject-fields.csv",
+ "name": "golden-meta-inject--fields",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_output_field_name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_output_field_type",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_output_field_length",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_output_field_precision",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/mdi/metadata/dataset/golden-meta-inject-mappings.json b/integration-tests/mdi/metadata/dataset/golden-meta-inject-mappings.json
new file mode 100644
index 0000000000..ccce706c01
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-meta-inject-mappings.json
@@ -0,0 +1,48 @@
+{
+ "base_filename": "golden-meta-inject-mappings.csv",
+ "name": "golden-meta-inject-mappings",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "target_transform_name",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "target_attribute_key",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "target_detail",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_transform",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_field",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/mdi/metadata/dataset/golden-meta-inject.json b/integration-tests/mdi/metadata/dataset/golden-meta-inject.json
new file mode 100644
index 0000000000..eda875c272
--- /dev/null
+++ b/integration-tests/mdi/metadata/dataset/golden-meta-inject.json
@@ -0,0 +1,80 @@
+{
+ "base_filename": "golden-meta-inject.csv",
+ "name": "golden-meta-inject",
+ "description": "",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "filename",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "run_configuration",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "source_transform",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "target_file",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "create_parent_folder",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "no_execution",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "allow_empty_stream_on_execution",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "stream_source_transform",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "stream_target_transform",
+ "field_format": ""
+ }
+ ],
+ "folder_name": ""
+}
\ No newline at end of file
diff --git a/integration-tests/mdi/metadata/unit-test/0045-meta-inject-valiate UNIT.json b/integration-tests/mdi/metadata/unit-test/0045-meta-inject-valiate UNIT.json
new file mode 100644
index 0000000000..067eef21bd
--- /dev/null
+++ b/integration-tests/mdi/metadata/unit-test/0045-meta-inject-valiate UNIT.json
@@ -0,0 +1,129 @@
+{
+ "database_replacements": [],
+ "autoOpening": true,
+ "description": "",
+ "persist_filename": "",
+ "test_type": "UNIT_TEST",
+ "variableValues": [],
+ "basePath": "${HOP_UNIT_TESTS_FOLDER}",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "source_field",
+ "data_set_field": "source_field"
+ },
+ {
+ "transform_field": "source_transform",
+ "data_set_field": "source_transform"
+ },
+ {
+ "transform_field": "target_attribute_key",
+ "data_set_field": "target_attribute_key"
+ },
+ {
+ "transform_field": "target_detail",
+ "data_set_field": "target_detail"
+ },
+ {
+ "transform_field": "target_transform_name",
+ "data_set_field": "target_transform_name"
+ }
+ ],
+ "field_order": [
+ "target_transform_name",
+ "target_attribute_key",
+ "target_detail",
+ "source_transform",
+ "source_field"
+ ],
+ "data_set_name": "golden-meta-inject-mappings",
+ "transform_name": "validate mappings"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "source_output_field_length",
+ "data_set_field": "source_output_field_length"
+ },
+ {
+ "transform_field": "source_output_field_name",
+ "data_set_field": "source_output_field_name"
+ },
+ {
+ "transform_field": "source_output_field_precision",
+ "data_set_field": "source_output_field_precision"
+ },
+ {
+ "transform_field": "source_output_field_type",
+ "data_set_field": "source_output_field_type"
+ }
+ ],
+ "field_order": [
+ "source_output_field_name",
+ "source_output_field_type",
+ "source_output_field_length",
+ "source_output_field_precision"
+ ],
+ "data_set_name": "golden-meta-inject--fields",
+ "transform_name": "validate fields"
+ },
+ {
+ "field_mappings": [
+ {
+ "transform_field": "allow_empty_stream_on_execution",
+ "data_set_field": "allow_empty_stream_on_execution"
+ },
+ {
+ "transform_field": "create_parent_folder",
+ "data_set_field": "create_parent_folder"
+ },
+ {
+ "transform_field": "filename",
+ "data_set_field": "filename"
+ },
+ {
+ "transform_field": "no_execution",
+ "data_set_field": "no_execution"
+ },
+ {
+ "transform_field": "run_configuration",
+ "data_set_field": "run_configuration"
+ },
+ {
+ "transform_field": "source_transform",
+ "data_set_field": "source_transform"
+ },
+ {
+ "transform_field": "stream_source_transform",
+ "data_set_field": "stream_source_transform"
+ },
+ {
+ "transform_field": "stream_target_transform",
+ "data_set_field": "stream_target_transform"
+ },
+ {
+ "transform_field": "target_file",
+ "data_set_field": "target_file"
+ }
+ ],
+ "field_order": [
+ "filename",
+ "run_configuration",
+ "source_transform",
+ "target_file",
+ "create_parent_folder",
+ "no_execution",
+ "allow_empty_stream_on_execution",
+ "stream_source_transform",
+ "stream_target_transform"
+ ],
+ "data_set_name": "golden-meta-inject",
+ "transform_name": "validate"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0045-meta-inject-valiate UNIT",
+ "trans_test_tweaks": [],
+ "pipeline_filename": "./0045-meta-inject-valiate.hpl"
+}
\ No newline at end of file
diff --git a/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java b/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java
index 0b5f6c8fb6..31805e41dd 100644
--- a/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java
+++ b/plugins/transforms/abort/src/main/java/org/apache/hop/pipeline/transforms/abort/AbortMeta.java
@@ -35,7 +35,7 @@ import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.w3c.dom.Node;
-/** Meta data for the abort transform. */
+/** Metadata for the abort transform. */
@Setter
@Getter
@Transform(
diff --git a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java
index f54c1a1d87..74ed8a7b30 100644
--- a/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java
+++ b/plugins/transforms/calculator/src/main/java/org/apache/hop/pipeline/transforms/calculator/CalculatorMetaFunction.java
@@ -28,67 +28,80 @@ public class CalculatorMetaFunction implements Cloneable {
@HopMetadataProperty(
key = "field_name",
+ injectionKey = "field_name",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldName")
private String fieldName;
@HopMetadataProperty(
key = "calc_type",
+ injectionKey = "calc_type",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.CalculationType",
storeWithCode = true)
private CalculationType calcType;
@HopMetadataProperty(
key = "field_a",
+ injectionKey = "field_a",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldA")
private String fieldA;
@HopMetadataProperty(
key = "field_b",
+ injectionKey = "field_b",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldB")
private String fieldB;
@HopMetadataProperty(
key = "field_c",
+ injectionKey = "field_c",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.FieldC")
private String fieldC;
@HopMetadataProperty(
key = "value_type",
+ injectionKey = "value_type",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueType")
private String valueType;
@HopMetadataProperty(
key = "value_length",
+ injectionKey = "value_length",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueLength")
private int valueLength;
@HopMetadataProperty(
key = "value_precision",
+ injectionKey = "value_precision",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValuePrecision")
private int valuePrecision;
@HopMetadataProperty(
key = "conversion_mask",
+ injectionKey = "conversion_mask",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueFormat")
private String conversionMask;
@HopMetadataProperty(
key = "decimal_symbol",
+ injectionKey = "decimal_symbol",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueDecimal")
private String decimalSymbol;
@HopMetadataProperty(
key = "grouping_symbol",
+ injectionKey = "grouping_symbol",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueGroup")
private String groupingSymbol;
@HopMetadataProperty(
key = "currency_symbol",
+ injectionKey = "currency_symbol",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.ValueCurrency")
private String currencySymbol;
@HopMetadataProperty(
key = "remove",
+ injectionKey = "remove",
injectionKeyDescription = "CalculatorMeta.Injection.Calculation.Remove")
private boolean removedFromResult;
diff --git a/plugins/transforms/datagrid/src/main/java/org/apache/hop/pipeline/transforms/datagrid/DataGridDataMeta.java b/plugins/transforms/datagrid/src/main/java/org/apache/hop/pipeline/transforms/datagrid/DataGridDataMeta.java
index 4f976703f9..615abc23f2 100644
--- a/plugins/transforms/datagrid/src/main/java/org/apache/hop/pipeline/transforms/datagrid/DataGridDataMeta.java
+++ b/plugins/transforms/datagrid/src/main/java/org/apache/hop/pipeline/transforms/datagrid/DataGridDataMeta.java
@@ -29,13 +29,17 @@ import org.apache.hop.metadata.api.HopMetadataProperty;
@Setter
public class DataGridDataMeta implements Cloneable {
- @HopMetadataProperty(key = "item")
+ @HopMetadataProperty(key = "item", isExcludedFromInjection = true)
private List<String> datalines;
- @HopMetadataProperty(injectionKeyDescription = "DataGrid.Injection.DataLine")
+ @HopMetadataProperty(
+ injectionKeyDescription = "DataGrid.Injection.DataLine",
+ isExcludedFromSerialization = true)
private String injectionDatalines;
- @HopMetadataProperty(injectionKeyDescription = "DataGrid.Injection.Splitter")
+ @HopMetadataProperty(
+ injectionKeyDescription = "DataGrid.Injection.Splitter",
+ isExcludedFromSerialization = true)
private String splitter;
public DataGridDataMeta() {
diff --git a/plugins/transforms/denormaliser/src/main/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMeta.java b/plugins/transforms/denormaliser/src/main/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMeta.java
index 0c76e2ada5..26df4062d8 100644
--- a/plugins/transforms/denormaliser/src/main/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMeta.java
+++ b/plugins/transforms/denormaliser/src/main/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMeta.java
@@ -19,6 +19,8 @@ package org.apache.hop.pipeline.transforms.denormaliser;
import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
@@ -43,6 +45,8 @@ import org.apache.hop.pipeline.transform.TransformMeta;
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Transform",
keywords = "i18n::DenormaliserMeta.keyword",
documentationUrl = "/pipeline/transforms/rowdenormaliser.html")
+@Getter
+@Setter
public class DenormaliserMeta extends BaseTransformMeta<Denormaliser, DenormaliserData> {
private static final Class<?> PKG = DenormaliserMeta.class;
diff --git a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigrationTest.java b/plugins/transforms/denormaliser/src/test/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMetaTest.java
similarity index 50%
rename from plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigrationTest.java
rename to plugins/transforms/denormaliser/src/test/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMetaTest.java
index 900e15afd7..95b27391a7 100644
--- a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigrationTest.java
+++ b/plugins/transforms/denormaliser/src/test/java/org/apache/hop/pipeline/transforms/denormaliser/DenormaliserMetaTest.java
@@ -13,33 +13,27 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
-package org.apache.hop.pipeline.transforms.metainject;
+package org.apache.hop.pipeline.transforms.denormaliser;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
import org.junit.jupiter.api.Test;
-class MetaInjectMigrationTest {
+class DenormaliserMetaTest {
@Test
- void test70() {
- Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping = new HashMap<>();
- TargetTransformAttribute target =
- new TargetTransformAttribute("transform", "SCHENAMENAMEFIELD", true);
- SourceTransformField source = new SourceTransformField("transform", "field");
- targetSourceMapping.put(target, source);
-
- MetaInjectMigration.migrateFrom70(targetSourceMapping);
-
- assertEquals(1, targetSourceMapping.size());
- TargetTransformAttribute target2 = targetSourceMapping.keySet().iterator().next();
- assertEquals("SCHEMANAMEFIELD", target2.getAttributeKey());
- assertEquals(target.getTransformName(), target2.getTransformName());
- assertEquals(target.isDetail(), target2.isDetail());
-
- assertEquals(source, targetSourceMapping.get(target2));
+ void testMetadataGroupMapping() throws Exception {
+ Map<String, Set<String>> map =
+ HopMetadataInjector.findInjectionGroupKeys(DenormaliserMeta.class);
+ assertEquals(2, map.size());
+ Set<String> groupKeys = map.get("group");
+ assertEquals(1, groupKeys.size());
+ Set<String> fieldKeys = map.get("fields");
+ assertEquals(12, fieldKeys.size());
}
}
diff --git a/plugins/transforms/filterrows/src/main/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsMeta.java b/plugins/transforms/filterrows/src/main/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsMeta.java
index 495b6d7aad..5297f9aaac 100644
--- a/plugins/transforms/filterrows/src/main/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsMeta.java
+++ b/plugins/transforms/filterrows/src/main/java/org/apache/hop/pipeline/transforms/filterrows/FilterRowsMeta.java
@@ -21,6 +21,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.CheckResult;
import org.apache.hop.core.Condition;
import org.apache.hop.core.Const;
@@ -55,6 +57,8 @@ import org.apache.hop.pipeline.transform.stream.StreamIcon;
categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Flow",
keywords = "i18n::FilterRowsMeta.keyword",
documentationUrl = "/pipeline/transforms/filterrows.html")
+@Getter
+@Setter
public class FilterRowsMeta extends BaseTransformMeta<FilterRows, FilterRowsData> {
private static final Class<?> PKG = FilterRowsMeta.class;
diff --git a/plugins/transforms/getvariable/pom.xml b/plugins/transforms/getvariable/pom.xml
index 005eb6bd8d..b0b382d106 100644
--- a/plugins/transforms/getvariable/pom.xml
+++ b/plugins/transforms/getvariable/pom.xml
@@ -28,5 +28,13 @@
<artifactId>hop-transform-getvariable</artifactId>
<packaging>jar</packaging>
<name>Hop Plugins Transforms Get Variable</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-core</artifactId>
+ <version>2.18.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git a/plugins/transforms/getvariable/src/main/java/org/apache/hop/pipeline/transforms/getvariable/GetVariableMeta.java b/plugins/transforms/getvariable/src/main/java/org/apache/hop/pipeline/transforms/getvariable/GetVariableMeta.java
index 59b28f233a..f222d694d2 100644
--- a/plugins/transforms/getvariable/src/main/java/org/apache/hop/pipeline/transforms/getvariable/GetVariableMeta.java
+++ b/plugins/transforms/getvariable/src/main/java/org/apache/hop/pipeline/transforms/getvariable/GetVariableMeta.java
@@ -32,7 +32,6 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
-import org.apache.hop.metadata.api.IIntCodeConverter;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
@@ -126,22 +125,6 @@ public class GetVariableMeta extends BaseTransformMeta<GetVariable, GetVariableD
}
}
- public static final class IntTypeConverter implements IIntCodeConverter {
- public IntTypeConverter() {
- // Do nothing
- }
-
- @Override
- public String getCode(int type) {
- return ValueMetaFactory.getValueMetaName(type);
- }
-
- @Override
- public int getType(String code) {
- return ValueMetaFactory.getIdForValueMeta(code);
- }
- }
-
public static final class FieldDefinition {
@HopMetadataProperty(
key = "name",
@@ -158,8 +141,7 @@ public class GetVariableMeta extends BaseTransformMeta<GetVariable, GetVariableD
@HopMetadataProperty(
key = "type",
injectionKey = "FIELDTYPE",
- injectionKeyDescription = "GetVariable.Injection.FIELDTYPE",
- intCodeConverter = IntTypeConverter.class)
+ injectionKeyDescription = "GetVariable.Injection.FIELDTYPE")
private String fieldType;
@HopMetadataProperty(
diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/JaninoMeta.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/JaninoMeta.java
index 4e3060959f..1cd6d46443 100644
--- a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/JaninoMeta.java
+++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/janino/JaninoMeta.java
@@ -24,7 +24,6 @@ import org.apache.hop.core.CheckResult;
import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaFactory;
@@ -37,11 +36,6 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
-@InjectionSupported(
- localizationPrefix = "Janino.Injection.",
- groups = {
- "FORMULA",
- })
@Transform(
id = "Janino",
image = "janino.svg",
diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java
index 2ea516c923..7324381566 100644
--- a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java
+++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassDialog.java
@@ -531,7 +531,7 @@ public class UserDefinedJavaClassDialog extends BaseTransformDialog {
fdClearResultFields.top = new FormAttachment(0, 0);
wClearResultFields.setLayoutData(fdClearResultFields);
- final int fieldsRows = input.getFieldInfo().size();
+ final int fieldsRows = input.getFields().size();
ColumnInfo[] colinf =
new ColumnInfo[] {
@@ -980,7 +980,7 @@ public class UserDefinedJavaClassDialog extends BaseTransformDialog {
/** Copy information from the meta-data input to the dialog fields. */
public void getData() {
int i = 0;
- for (FieldInfo fi : input.getFieldInfo()) {
+ for (FieldInfo fi : input.getFields()) {
TableItem item = wFields.table.getItem(i);
i++;
item.setText(1, fi.getName());
diff --git a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassMeta.java b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassMeta.java
index 65fbe82df2..c6142725b7 100644
--- a/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassMeta.java
+++ b/plugins/transforms/janino/src/main/java/org/apache/hop/pipeline/transforms/userdefinedjavaclass/UserDefinedJavaClassMeta.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Method;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import lombok.Getter;
@@ -148,7 +147,7 @@ public class UserDefinedJavaClassMeta
@HopMetadataProperty(
key = "definition",
groupKey = "definitions",
- injectionKeyDescription = "JAVA_CLASSES",
+ injectionGroupKey = "JAVA_CLASSES",
injectionGroupDescription = "UserDefinedJavaClass.Injection.JAVA_CLASSES")
private List<UserDefinedJavaClassDef> definitions = new ArrayList<>();
@@ -305,10 +304,6 @@ public class UserDefinedJavaClassMeta
}
}
- public List<FieldInfo> getFieldInfo() {
- return Collections.unmodifiableList(fields);
- }
-
public void setFieldInfo(List<FieldInfo> fields) {
replaceFields(fields);
}
@@ -318,10 +313,6 @@ public class UserDefinedJavaClassMeta
changed = true;
}
- public List<UserDefinedJavaClassDef> getDefinitions() {
- return Collections.unmodifiableList(definitions);
- }
-
/**
* This method orders the classes by sorting all the normal classes by alphabetic order and then
* sorting all the transaction classes by alphabetic order. This makes the resolution of classes
@@ -368,7 +359,7 @@ public class UserDefinedJavaClassMeta
try {
cookClasses();
} catch (HopException e) {
- throw new RuntimeException(e);
+ throw new RuntimeException("Error cooking class", e);
}
if (cookedTransformClass == null) {
@@ -454,7 +445,7 @@ public class UserDefinedJavaClassMeta
info,
nextTransform,
variables,
- getFieldInfo());
+ getFields());
} catch (Exception e) {
throw new HopTransformException("Error executing UserDefinedJavaClass.getFields(): ", e);
}
diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
index 58e4446a41..577d47d5bb 100644
--- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
+++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputMeta.java
@@ -181,8 +181,6 @@ public class JsonInputMeta extends BaseFileInputMeta<JsonInput, JsonInputData, B
@HopMetadataProperty(
key = "file",
inline = true,
- injectionKey = "FILE",
- injectionKeyDescription = "TextFileInput.Injection.FILE",
childKeysToIgnore = {
"accept_filenames",
"accept_transform_name",
@@ -195,8 +193,6 @@ public class JsonInputMeta extends BaseFileInputMeta<JsonInput, JsonInputData, B
@HopMetadataProperty(
key = "field",
groupKey = "fields",
- injectionKey = "FIELD",
- injectionKeyDescription = "JsonInput.Injection.FIELD",
injectionGroupKey = "FIELDS",
injectionGroupDescription = "JsonInput.Injection.FIELDS")
private List<JsonInputField> inputFields;
diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputField.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputField.java
index bc5c324bc3..8435c35b0c 100644
--- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputField.java
+++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputField.java
@@ -28,13 +28,11 @@ import org.apache.hop.metadata.api.HopMetadataProperty;
public class JsonOutputField implements Cloneable {
@HopMetadataProperty(
key = "name",
- injectionGroupKey = "FIELDS",
injectionKeyDescription = "JsonOutput.Injection.JSON_FIELDNAME")
private String fieldName;
@HopMetadataProperty(
key = "element",
- injectionGroupKey = "FIELDS",
injectionKeyDescription = "JsonOutput.Injection.JSON_ELEMENTNAME")
private String elementName;
diff --git a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputMetaTest.java b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputMetaTest.java
index 26107a6a96..0f4a691831 100644
--- a/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputMetaTest.java
+++ b/plugins/transforms/json/src/test/java/org/apache/hop/pipeline/transforms/jsonoutput/JsonOutputMetaTest.java
@@ -23,6 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.Test;
@@ -68,4 +71,13 @@ public class JsonOutputMetaTest {
assertEquals("f3", f3.getFieldName());
assertEquals("element3", f3.getElementName());
}
+
+ @Test
+ void testGroupMappings() throws Exception {
+ Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(JsonOutputMeta.class);
+ assertEquals(1, map.size());
+ Set<String> fields = map.get("FIELDS");
+ assertTrue(fields.contains("name"));
+ assertTrue(fields.contains("element"));
+ }
}
diff --git a/plugins/transforms/mergejoin/src/test/java/org/apache/hop/pipeline/transforms/mergejoin/MergeJoinMetaTest.java b/plugins/transforms/mergejoin/src/test/java/org/apache/hop/pipeline/transforms/mergejoin/MergeJoinMetaTest.java
index f4bc694628..6e8aac0e82 100644
--- a/plugins/transforms/mergejoin/src/test/java/org/apache/hop/pipeline/transforms/mergejoin/MergeJoinMetaTest.java
+++ b/plugins/transforms/mergejoin/src/test/java/org/apache/hop/pipeline/transforms/mergejoin/MergeJoinMetaTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
@@ -37,6 +38,7 @@ import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transform.stream.IStream;
import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
@@ -238,4 +240,14 @@ class MergeJoinMetaTest {
assertEquals(4, rowMeta.size());
}
+
+ @Test
+ void testHopMetadataMapping() throws Exception {
+ Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(MergeJoinMeta.class);
+ assertEquals(2, map.size());
+ Set<String> keys1 = map.get("KEY_FIELDS1");
+ assertEquals(1, keys1.size());
+ Set<String> keys2 = map.get("KEY_FIELDS2");
+ assertEquals(1, keys2.size());
+ }
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
index 7682e7188c..33aa71bd33 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
@@ -18,20 +18,18 @@
package org.apache.hop.pipeline.transforms.metainject;
import java.io.OutputStream;
+import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.IRowSet;
@@ -39,15 +37,24 @@ import org.apache.hop.core.Result;
import org.apache.hop.core.RowMetaAndData;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopTransformException;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.injection.bean.BeanInjectionInfo;
import org.apache.hop.core.injection.bean.BeanInjector;
import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.core.row.RowDataUtil;
+import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.util.ExecutorUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
+import org.apache.hop.metadata.util.ReflectionUtil;
import org.apache.hop.pipeline.Pipeline;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.RowProducer;
@@ -63,8 +70,6 @@ import org.apache.hop.pipeline.transform.TransformMeta;
public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
private static final Class<?> PKG = MetaInject.class;
- private static final Lock repoSaveLock = new ReentrantLock();
-
public MetaInject(
TransformMeta transformMeta,
MetaInjectMeta meta,
@@ -77,20 +82,180 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
@Override
public boolean processRow() throws HopException {
-
// Read the data from all input transforms and keep it in memory...
// Skip the transform from which we stream data. Keep that available for runtime action.
//
data.rowMap = new HashMap<>();
- boolean receivedRows = false;
- boolean hasEmptyList = false;
+ data.receivedRows = false;
+ data.hasEmptyList = false;
// If there are no previous transforms we will set receivedRows to true. This allows execution
// using constants only
if (getPipelineMeta().getPrevTransformNames(getTransformMeta()).length == 0) {
- receivedRows = true;
+ data.receivedRows = true;
+ }
+
+ readDataFromSourceTransforms();
+
+ injectMetadataIntoTemplateTransforms();
+
+ // Check if all previous transforms are returning data, unless isAllowEmptyStreamOnExecution is
+ // true then execute if at least one branch has data
+ if (!data.receivedRows || (data.hasEmptyList && !meta.isAllowEmptyStreamOnExecution())) {
+ setOutputDone();
+ return false;
+ }
+
+ postInjectionHandling();
+
+ if (!meta.isNoExecution()) {
+ executeTemplatePipeline();
+ }
+
+ // We let the pipeline complete its execution to allow for any customizations to MDI to
+ // happen in the init methods of transforms.
+ //
+ String targetFile = resolve(meta.getTargetFile());
+ if (!Utils.isEmpty(targetFile)) {
+ writeInjectedHpl(targetFile);
+ }
+
+ // All done!
+ setOutputDone();
+
+ return false;
+ }
+
+ private void executeTemplatePipeline() throws HopException {
+ // Now we can execute this modified transformation metadata.
+ //
+ final Pipeline injectPipeline = createInjectPipeline();
+ injectPipeline.setParentPipeline(getPipeline());
+ injectPipeline.setMetadataProvider(getMetadataProvider());
+ if (getPipeline().getParentWorkflow() != null) {
+ injectPipeline.setParentWorkflow(getPipeline().getParentWorkflow());
+ }
+
+ // Copy all variables over...
+ //
+ injectPipeline.copyFrom(this);
+
+ // Copy parameter definitions with empty values.
+ // Then set those parameters to the values if have any.
+
+ injectPipeline.copyParametersFromDefinitions(data.pipelineMeta);
+ for (String variableName : injectPipeline.getVariableNames()) {
+ String variableValue = getVariable(variableName);
+ if (StringUtils.isNotEmpty(variableValue)) {
+ injectPipeline.setParameterValue(variableName, variableValue);
+ }
+ }
+
+ getPipeline().addExecutionStoppedListener(e -> injectPipeline.stopAll());
+
+ injectPipeline.setLogLevel(getLogLevel());
+
+ // Parameters get activated below so we need to make sure they have values
+ //
+ injectPipeline.prepareExecution();
+
+ // See if we need to stream some data over...
+ //
+ RowProducer rowProducer =
+ data.streaming ? injectPipeline.addRowProducer(data.streamingTargetTransformName, 0) : null;
+
+ // Finally, add the mapping transformation to the active sub-transformations
+ // map in the parent transformation
+ //
+ getPipeline().addActiveSubPipeline(getTransformName(), injectPipeline);
+
+ if (!Utils.isEmpty(meta.getSourceTransformName())) {
+ handleReadingFromSourceTransform(injectPipeline);
+ }
+
+ injectPipeline.startThreads();
+
+ Future<HopTransformException> steamingFuture = handleStreamingToTargetTransform(rowProducer);
+
+ // Wait until the child transformation finished processing...
+ //
+ while (!injectPipeline.isFinished() && !injectPipeline.isStopped() && !isStopped()) {
+ copyResult(injectPipeline);
+
+ // Wait a little bit.
+ try {
+ Thread.sleep(50);
+ } catch (Exception e) {
+ // Ignore errors
+ }
+ }
+ copyResult(injectPipeline);
+ waitUntilFinished(injectPipeline);
+ try {
+ if (steamingFuture != null && steamingFuture.get() != null) {
+ throw steamingFuture.get();
+ }
+ } catch (ExecutionException ignore) {
+ // Ignore
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private Future<HopTransformException> handleStreamingToTargetTransform(RowProducer rowProducer)
+ throws HopException {
+ Future<HopTransformException> steamingFuture = null;
+ if (data.streaming) {
+ // Deplete all the rows from the parent transformation into the modified transformation
+ //
+ IRowSet rowSet = findInputRowSet(data.streamingSourceTransformName);
+ if (rowSet == null) {
+ throw new HopException(
+ "Unable to find transform '"
+ + data.streamingSourceTransformName
+ + "' to stream data from");
+ }
+ steamingFuture = ExecutorUtil.getExecutor().submit(() -> putRows(rowSet, rowProducer));
}
+ return steamingFuture;
+ }
+ private void handleReadingFromSourceTransform(Pipeline injectPipeline) throws HopException {
+ ITransform transformInterface = injectPipeline.getTransform(meta.getSourceTransformName(), 0);
+ if (transformInterface == null) {
+ throw new HopException(
+ "Unable to find transform '" + meta.getSourceTransformName() + "' to read from.");
+ }
+ transformInterface.addRowListener(
+ new RowAdapter() {
+ @Override
+ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
+ // Just pass along the data as output of this transform...
+ //
+ MetaInject.this.putRow(rowMeta, row);
+ }
+ });
+ }
+
+ private void postInjectionHandling() {
+ List<TransformMeta> targetTransforms = data.pipelineMeta.getTransforms();
+ for (Entry<String, ITransformMeta> en : data.transformInjectionMetasMap.entrySet()) {
+ en.getValue().resetTransformIoMeta();
+ en.getValue().searchInfoAndTargetTransforms(targetTransforms);
+ }
+
+ for (String targetTransformName : data.transformInjectionMetasMap.keySet()) {
+ if (!data.transformInjectionMetasMap.containsKey(targetTransformName)) {
+ TransformMeta targetTransform =
+ TransformMeta.findTransform(targetTransforms, targetTransformName);
+ if (targetTransform != null) {
+ targetTransform.getTransform().searchInfoAndTargetTransforms(targetTransforms);
+ }
+ }
+ }
+ }
+
+ private void readDataFromSourceTransforms() throws HopTransformException {
for (String prevTransformName : getPipelineMeta().getPrevTransformNames(getTransformMeta())) {
// Don't read from the streaming source transform
//
@@ -108,167 +273,264 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
row = getRowFrom(rowSet);
}
if (!list.isEmpty()) {
- receivedRows = true;
+ data.receivedRows = true;
data.rowMap.put(prevTransformName, list);
} else {
- hasEmptyList = true;
+ data.hasEmptyList = true;
}
}
}
+ }
- // Check if all previous transforms are returning data, unless isAllowEmptyStreamOnExecution is
- // true then execute if at least one branch has data
- if (!receivedRows || (hasEmptyList && !meta.isAllowEmptyStreamOnExecution())) {
- setOutputDone();
- return false;
- }
+ private void injectMetadataIntoTemplateTransforms() throws HopException {
+ for (Entry<String, ITransformMeta> entry : data.transformInjectionMetasMap.entrySet()) {
+ String targetTransformName = entry.getKey();
+ ITransformMeta targetTransformMeta = entry.getValue();
- List<TransformMeta> transforms = data.pipelineMeta.getTransforms();
- for (Entry<String, ITransformMeta> en : data.transformInjectionMetasMap.entrySet()) {
- newInjection(en.getKey(), en.getValue());
- }
- /*
- * constants injection should be executed after transforms, because if constant should be inserted into target with array
- * in path, constants should be inserted into all arrays items
- */
- for (Entry<String, ITransformMeta> en : data.transformInjectionMetasMap.entrySet()) {
- newInjectionConstants(this, en.getKey(), en.getValue());
+ checkInjectionSupported(targetTransformMeta.getClass());
+
+ InjectionSupported annotation =
+ targetTransformMeta.getClass().getAnnotation(InjectionSupported.class);
+ if (annotation == null) {
+ // Injects using @HopMetadataProperty annotations.
+ injectMetadataIntoTemplateTransform(targetTransformMeta, targetTransformName);
+ } else {
+ // Old style metadata injection (Legacy)
+ newInjection(targetTransformName, targetTransformMeta);
+ newInjectionConstants(this, targetTransformName, targetTransformMeta);
+ }
}
- for (Entry<String, ITransformMeta> en : data.transformInjectionMetasMap.entrySet()) {
- en.getValue().resetTransformIoMeta();
- en.getValue().searchInfoAndTargetTransforms(transforms);
+ }
+
+ private void checkInjectionSupported(Class<?> clazz) throws HopException {
+ InjectionSupported annotation = clazz.getAnnotation(InjectionSupported.class);
+ if (annotation != null) {
+ return;
}
- for (String targetTransformName : data.transformInjectionMetasMap.keySet()) {
- if (!data.transformInjectionMetasMap.containsKey(targetTransformName)) {
- TransformMeta targetTransform =
- TransformMeta.findTransform(transforms, targetTransformName);
- if (targetTransform != null) {
- targetTransform.getTransform().searchInfoAndTargetTransforms(transforms);
- }
+ // See if there are any Hop metadata properties we can use...
+ //
+ for (Field field : ReflectionUtil.findAllFields(clazz)) {
+ HopMetadataProperty property = field.getAnnotation(HopMetadataProperty.class);
+ if (property != null) {
+ return;
}
}
+ throw new HopException("Injection is not supported for class " + clazz.getSimpleName());
+ }
- if (!meta.isNoExecution()) {
- // Now we can execute this modified transformation metadata.
- //
- final Pipeline injectPipeline = createInjectPipeline();
- injectPipeline.setParentPipeline(getPipeline());
- injectPipeline.setMetadataProvider(getMetadataProvider());
- if (getPipeline().getParentWorkflow() != null) {
- injectPipeline.setParentWorkflow(getPipeline().getParentWorkflow());
- }
+ private void injectMetadataIntoTemplateTransform(
+ ITransformMeta targetTransformMeta, String targetTransformName) throws HopException {
+ // Find the injection key-to-group mapping
+ //
+ Map<String, Set<String>> injectionKeyGroupMap =
+ HopMetadataInjector.findInjectionGroupKeys(targetTransformMeta.getClass());
- // Copy all variables over...
- //
- injectPipeline.copyFrom(this);
+ // Which keys need injection into the class (not group rows of data)?
+ //
+ Map<String, Object> injectionKeyData = new HashMap<>();
- // Copy parameter definitions with empty values.
- // Then set those parameters to the values if have any.
+ // These are the values to inject into keys belonging to the groups.
+ //
+ Map<String, RowBuffer> injectionGroupData = new HashMap<>();
- injectPipeline.copyParametersFromDefinitions(data.pipelineMeta);
- for (String variableName : injectPipeline.getVariableNames()) {
- String variableValue = getVariable(variableName);
- if (StringUtils.isNotEmpty(variableValue)) {
- injectPipeline.setParameterValue(variableName, variableValue);
- }
- }
+ // What is the row layout for a particular group data?
+ //
+ Map<String, IRowMeta> groupRowMetaMap = new HashMap<>();
- getPipeline().addExecutionStoppedListener(e -> injectPipeline.stopAll());
+ // We keep track of where an injection group data comes from.
+ //
+ Map<String, String> groupSourceMap = new HashMap<>();
- injectPipeline.setLogLevel(getLogLevel());
+ // We already have the rows. We just need to rename the fields to the injection key
+ //
+ for (MetaInjectMapping mapping : meta.getMappings()) {
+ collectDataForOneMapping(
+ targetTransformName,
+ mapping,
+ injectionKeyGroupMap,
+ groupRowMetaMap,
+ groupSourceMap,
+ injectionGroupData,
+ injectionKeyData);
+ }
- // Parameters get activated below so we need to make sure they have values
+ // Now all source rows are mapped and we can construct the group data map
+ //
+ for (Entry<String, String> e : groupSourceMap.entrySet()) {
+ String groupKey = e.getKey();
+ String sourceTransformName = e.getValue();
+ IRowMeta rowMeta = groupRowMetaMap.get(groupKey);
+ List<RowMetaAndData> rowMetaAndData = data.rowMap.get(sourceTransformName);
+ // Make a copy of the row metadata and data to avoid any risk of
+ // another transform modifying the rows during injection.
//
- injectPipeline.prepareExecution();
+ RowBuffer rowBuffer = new RowBuffer(rowMeta.clone());
+ for (RowMetaAndData rd : rowMetaAndData) {
+ rowBuffer.getBuffer().add(rowMeta.cloneRow(rd.getData()));
+ }
+ injectionGroupData.put(groupKey, rowBuffer);
+ }
- // See if we need to stream some data over...
- //
- RowProducer rowProducer =
- data.streaming
- ? injectPipeline.addRowProducer(data.streamingTargetTransformName, 0)
- : null;
+ // Now everything is ready to be injected into the target transform
+ //
+ if (!injectionKeyData.isEmpty() || !injectionGroupData.isEmpty()) {
+ HopMetadataInjector.inject(
+ metadataProvider, targetTransformMeta, injectionKeyData, injectionGroupData);
+ }
+ }
- // Finally, add the mapping transformation to the active sub-transformations
- // map in the parent transformation
- //
- getPipeline().addActiveSubPipeline(getTransformName(), injectPipeline);
+ private void collectDataForOneMapping(
+ String targetTransformName,
+ MetaInjectMapping mapping,
+ Map<String, Set<String>> injectionKeyGroupMap,
+ Map<String, IRowMeta> groupRowMetaMap,
+ Map<String, String> groupSourceMap,
+ Map<String, RowBuffer> injectionGroupData,
+ Map<String, Object> injectionKeyData)
+ throws HopTransformException, HopValueException {
+ if (!targetTransformName.equalsIgnoreCase(mapping.getTargetTransformName())) {
+ // Covered elsewhere.
+ return;
+ }
+ List<RowMetaAndData> sourceRows = data.rowMap.get(mapping.getSourceTransformName());
- if (!Utils.isEmpty(meta.getSourceTransformName())) {
- ITransform transformInterface =
- injectPipeline.getTransform(meta.getSourceTransformName(), 0);
- if (transformInterface == null) {
- throw new HopException(
- "Unable to find transform '" + meta.getSourceTransformName() + "' to read from.");
- }
- transformInterface.addRowListener(
- new RowAdapter() {
- @Override
- public void rowWrittenEvent(IRowMeta rowMeta, Object[] row)
- throws HopTransformException {
- // Just pass along the data as output of this transform...
- //
- MetaInject.this.putRow(rowMeta, row);
- }
- });
+ // To which group does this belong?
+ //
+ String groupKey = lookupGroupKey(injectionKeyGroupMap, mapping.getTargetAttributeKey());
+ if (mapping.isTargetDetail() || groupKey != null) {
+ // If it's a detail and not part of a group, we have a configuration error.
+ //
+ if (groupKey == null) {
+ throw new HopTransformException(
+ "The injection group key for target key '"
+ + mapping.getTargetAttributeKey()
+ + "' was not found.");
}
- injectPipeline.startThreads();
-
- Future<HopTransformException> steamingFuture = null;
- if (data.streaming) {
- // Deplete all the rows from the parent transformation into the modified transformation
+ if (sourceRows != null && !sourceRows.isEmpty()) {
+ // We need to collect the data for the mapping
//
- IRowSet rowSet = findInputRowSet(data.streamingSourceTransformName);
- if (rowSet == null) {
- throw new HopException(
- "Unable to find transform '"
- + data.streamingSourceTransformName
- + "' to stream data from");
+ collectDataForOneMappingGroup(
+ mapping, groupRowMetaMap, groupSourceMap, groupKey, sourceRows);
+ } else {
+ // See if this is a constant without a source
+ //
+ if (StringUtils.isEmpty(mapping.getSourceTransformName())) {
+ addConstantToGroupData(mapping, injectionGroupData, groupKey);
}
- steamingFuture = ExecutorUtil.getExecutor().submit(() -> putRows(rowSet, rowProducer));
}
-
- // Wait until the child transformation finished processing...
+ } else {
+ // A simple property from the first row.
+ // This also captures the "Constant" mappings where we don't have source rows to feed the
+ // injection.
//
- while (!injectPipeline.isFinished() && !injectPipeline.isStopped() && !isStopped()) {
- copyResult(injectPipeline);
-
- // Wait a little bit.
- try {
- Thread.sleep(50);
- } catch (Exception e) {
- // Ignore errors
- }
- }
- copyResult(injectPipeline);
- waitUntilFinished(injectPipeline);
- try {
- if (steamingFuture != null && steamingFuture.get() != null) {
- throw steamingFuture.get();
- }
- } catch (ExecutionException ignore) {
- // Ignore
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
+ collectInjectionKeyValue(mapping, sourceRows, injectionKeyData);
}
+ }
- // let the transformation complete it's execution to allow for any customizations to MDI to
- // happen in the init methods of transforms
- if (isDetailed()) {
- logDetailed("XML of transformation after injection: " + data.pipelineMeta.getXml(this));
+ private static void collectDataForOneMappingGroup(
+ MetaInjectMapping mapping,
+ Map<String, IRowMeta> groupRowMetaMap,
+ Map<String, String> groupSourceMap,
+ String groupKey,
+ List<RowMetaAndData> sourceRows)
+ throws HopTransformException {
+ IRowMeta rowMeta;
+ rowMeta =
+ groupRowMetaMap.computeIfAbsent(groupKey, f -> sourceRows.getFirst().getRowMeta().clone());
+ int index = rowMeta.indexOfValue(mapping.getSourceField());
+ if (index < 0) {
+ // Is the field already renamed to the target?
+ //
+ index = rowMeta.indexOfValue(mapping.getTargetAttributeKey());
}
- String targetFile = resolve(meta.getTargetFile());
- if (!Utils.isEmpty(targetFile)) {
- writeInjectedHpl(targetFile);
+ if (index < 0) {
+ throw new HopTransformException(
+ "The field '"
+ + mapping.getSourceField()
+ + "' from transform '"
+ + mapping.getSourceTransformName()
+ + "' to inject could not be found");
}
+ // We give the value the name of the target injection key
+ //
+ IValueMeta valueMeta = rowMeta.getValueMeta(index);
+ valueMeta.setName(mapping.getTargetAttributeKey());
- // All done!
+ // Let's keep track of where the rows are coming from for the current group
+ //
+ groupSourceMap.put(groupKey, mapping.getSourceTransformName());
+ }
- setOutputDone();
+ private static void addConstantToGroupData(
+ MetaInjectMapping mapping, Map<String, RowBuffer> injectionGroupData, String groupKey) {
+ // We need to add or extend a single row in a row buffer for the given group.
+ //
+ RowBuffer rowBuffer = injectionGroupData.computeIfAbsent(groupKey, f -> new RowBuffer());
- return false;
+ // We need to extend this row buffer with one value in the metadata and keep it to one
+ // row.
+ //
+ IRowMeta rowMeta = rowBuffer.getRowMeta();
+ rowMeta.addValueMeta(new ValueMetaString(mapping.getTargetAttributeKey()));
+ List<Object[]> rows = rowBuffer.getBuffer();
+ Object[] row;
+ if (rows.isEmpty()) {
+ row = RowDataUtil.allocateRowData(1);
+ rows.add(row);
+ } else {
+ row = rows.getFirst();
+ }
+ row = RowDataUtil.createResizedCopy(row, rowMeta.size());
+ row[rowMeta.size() - 1] = mapping.getSourceField();
+ // Let's not forget to update the row after re-sizing it.
+ rows.set(0, row);
+ }
+
+ private static String lookupGroupKey(
+ Map<String, Set<String>> groupKeysMap, String targetAttributeKey) {
+ // Find the first group that matches.
+ // If for some reason there is more than one group that matches, we're out of luck.
+ // This transform failed to capture the group to which the target attribute key belongs to.
+ //
+ for (Entry<String, Set<String>> entry : groupKeysMap.entrySet()) {
+ if (entry.getValue().contains(targetAttributeKey)) {
+ return entry.getKey();
+ }
+ }
+ return null;
+ }
+
+ private static void collectInjectionKeyValue(
+ MetaInjectMapping mapping,
+ List<RowMetaAndData> sourceRows,
+ Map<String, Object> injectionKeyData)
+ throws HopTransformException, HopValueException {
+ if (sourceRows == null || sourceRows.isEmpty()) {
+ if (StringUtils.isEmpty(mapping.getSourceTransformName())) {
+ // This is a constant String value to set.
+ // The value is set in the source field.
+ //
+ injectionKeyData.put(mapping.getTargetAttributeKey(), mapping.getSourceField());
+ }
+ return;
+ }
+ RowMetaAndData sourceRow = sourceRows.getFirst();
+ int index = sourceRow.getRowMeta().indexOfValue(mapping.getSourceField());
+ if (index < 0) {
+ throw new HopTransformException(
+ "The field '"
+ + mapping.getSourceField()
+ + "' from transform '"
+ + mapping.getSourceTransformName()
+ + "' to inject could not be found");
+ }
+ // We have the field, we can rename it to the target key
+ //
+ IValueMeta valueMeta = sourceRow.getRowMeta().getValueMeta(index);
+ Object valueData = valueMeta.getNativeDataType(sourceRow.getData()[index]);
+ injectionKeyData.put(mapping.getTargetAttributeKey(), valueData);
}
HopTransformException putRows(IRowSet rowSet, RowProducer rowProducer) {
@@ -310,80 +572,59 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
* Writes the generated meta injection transformation to the file system.
*
* @param targetFilePath the filesystem path to which to save the generated injection hpl
- * @throws HopException
+ * @throws HopException In case we couldn't write the pipeline XML to the given target path.
*/
private void writeInjectedHplToFs(String targetFilePath) throws HopException {
-
- OutputStream os = null;
- try {
-
- if (meta.isCreateParentFolder()) {
- createParentFolder(targetFilePath);
- }
-
- os = HopVfs.getOutputStream(targetFilePath, false, variables);
+ if (meta.isCreateParentFolder()) {
+ createParentFolder(targetFilePath);
+ }
+ try (OutputStream os = HopVfs.getOutputStream(targetFilePath, false, variables)) {
os.write(XmlHandler.getXmlHeader().getBytes(StandardCharsets.UTF_8));
os.write(data.pipelineMeta.getXml(this).getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
throw new HopException(
"Unable to write target file (hpl after injection) to file '" + targetFilePath + "'", e);
- } finally {
- if (os != null) {
- try {
- os.close();
- } catch (Exception e) {
- throw new HopException(e);
- }
- }
}
}
- private void createParentFolder(String filename) throws Exception {
+ private void createParentFolder(String filename) throws HopException {
// Check for parent folder
- FileObject parentfolder = null;
-
- try {
- // Get parent folder
- parentfolder = HopVfs.getFileObject(filename, variables).getParent();
-
- if (parentfolder.exists()) {
+ //
+ try (FileObject parentFolder = HopVfs.getFileObject(filename, variables).getParent()) {
+ // Check the parent folder
+ if (parentFolder.exists()) {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
- PKG, "MetaInject.Log.ParentFolderExist", HopVfs.getFriendlyURI(parentfolder)));
+ PKG, "MetaInject.Log.ParentFolderExist", HopVfs.getFriendlyURI(parentFolder)));
}
} else {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
- PKG, "MetaInject.Log.ParentFolderNotExist", HopVfs.getFriendlyURI(parentfolder)));
+ PKG, "MetaInject.Log.ParentFolderNotExist", HopVfs.getFriendlyURI(parentFolder)));
}
if (meta.isCreateParentFolder()) {
- parentfolder.createFolder();
+ parentFolder.createFolder();
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"MetaInject.Log.ParentFolderCreated",
- HopVfs.getFriendlyURI(parentfolder)));
+ HopVfs.getFriendlyURI(parentFolder)));
}
} else {
throw new HopException(
BaseMessages.getString(
PKG,
"MetaInject.Log.ParentFolderNotExistCreateIt",
- HopVfs.getFriendlyURI(parentfolder),
+ HopVfs.getFriendlyURI(parentFolder),
HopVfs.getFriendlyURI(filename, variables)));
}
}
- } finally {
- if (parentfolder != null) {
- try {
- parentfolder.close();
- } catch (Exception ex) {
- // Ignore
- }
- }
+ } catch (Exception e) {
+ throw new HopException(
+ "Unable to create parent folder for injected pipeline file: " + filename, e);
}
}
@@ -398,32 +639,29 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
// Collect all the metadata for this target transform...
//
- Map<TargetTransformAttribute, SourceTransformField> targetMap = meta.getTargetSourceMapping();
boolean wasInjection = false;
- for (TargetTransformAttribute target : targetMap.keySet()) {
- SourceTransformField source = targetMap.get(target);
-
- if (target.getTransformName().equalsIgnoreCase(targetTransform)
- && source.getTransformName() != null) {
+ for (MetaInjectMapping mapping : meta.getMappings()) {
+ if (mapping.getTargetTransformName().equalsIgnoreCase(targetTransform)
+ && StringUtils.isNotEmpty(mapping.getSourceTransformName())) {
// This is the transform to collect data for...
// We also know which transform to read the data from. (source)
//
// from specified transform
- List<RowMetaAndData> rows = data.rowMap.get(source.getTransformName());
+ List<RowMetaAndData> rows = data.rowMap.get(mapping.getSourceTransformName());
if (!Utils.isEmpty(rows)) {
// Which metadata key is this referencing? Find the attribute key in the metadata
// entries...
//
- if (injector.hasProperty(targetTransformMeta, target.getAttributeKey())) {
+ if (injector.hasProperty(targetTransformMeta, mapping.getTargetAttributeKey())) {
// target transform has specified key
boolean skip = false;
for (RowMetaAndData r : rows) {
- if (r.getRowMeta().indexOfValue(source.getField()) < 0) {
+ if (r.getRowMeta().indexOfValue(mapping.getSourceField()) < 0) {
logError(
BaseMessages.getString(
PKG,
"MetaInject.SourceFieldIsNotDefined.Message",
- source.getField(),
+ mapping.getSourceField(),
getPipelineMeta().getName()));
// source transform doesn't contain specified field
skip = true;
@@ -432,7 +670,10 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
if (!skip) {
// specified field exist - need to inject
injector.setProperty(
- targetTransformMeta, target.getAttributeKey(), rows, source.getField());
+ targetTransformMeta,
+ mapping.getTargetAttributeKey(),
+ rows,
+ mapping.getSourceField());
wasInjection = true;
}
} else {
@@ -441,7 +682,7 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
BaseMessages.getString(
PKG,
"MetaInject.TargetKeyIsNotDefined.Message",
- target.getAttributeKey(),
+ mapping.getTargetAttributeKey(),
getPipelineMeta().getName()));
}
}
@@ -470,27 +711,25 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
// Collect all the metadata for this target transform...
//
- Map<TargetTransformAttribute, SourceTransformField> targetMap = meta.getTargetSourceMapping();
- for (TargetTransformAttribute target : targetMap.keySet()) {
- SourceTransformField source = targetMap.get(target);
+ for (MetaInjectMapping mapping : meta.getMappings()) {
- if (target.getTransformName().equalsIgnoreCase(targetTransform)
- && source.getTransformName() == null) {
+ if (mapping.getTargetTransformName().equalsIgnoreCase(targetTransform)
+ && StringUtils.isNotEmpty(mapping.getSourceTransformName())) {
// This is the transform to collect data for...
// We also know which transform to read the data from. (source)
//
// inject constant
- if (injector.hasProperty(targetTransformMeta, target.getAttributeKey())) {
+ if (injector.hasProperty(targetTransformMeta, mapping.getTargetAttributeKey())) {
// target transform has specified key
- String value = variables.resolve(source.getField());
- injector.setProperty(targetTransformMeta, target.getAttributeKey(), null, value);
+ String value = variables.resolve(mapping.getSourceField());
+ injector.setProperty(targetTransformMeta, mapping.getTargetAttributeKey(), null, value);
} else {
// target transform doesn't have specified key - just report but don't fail like in 6.0
logError(
BaseMessages.getString(
PKG,
"MetaInject.TargetKeyIsNotDefined.Message",
- target.getAttributeKey(),
+ mapping.getTargetAttributeKey(),
getPipelineMeta().getName()));
}
}
@@ -513,7 +752,6 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
if (super.init()) {
try {
- meta.actualizeMetaInjectMapping();
data.pipelineMeta = loadPipelineMeta();
checkSourceTransformsAvailability();
checkTargetTransformsAvailability();
@@ -530,13 +768,11 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
// See if we need to stream data from a specific transform into the template
//
- if (meta.getStreamSourceTransform() != null
- && !Utils.isEmpty(meta.getStreamTargetTransformName())) {
+ if (StringUtils.isNotEmpty(meta.getStreamSourceTransformName())) {
data.streaming = true;
- data.streamingSourceTransformName = meta.getStreamSourceTransform().getName();
+ data.streamingSourceTransformName = meta.getStreamSourceTransformName();
data.streamingTargetTransformName = meta.getStreamTargetTransformName();
}
-
return true;
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "MetaInject.BadEncoding.Message"), e);
@@ -548,78 +784,50 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
}
private void checkTargetTransformsAvailability() {
- Set<String> existedTransformNames =
- convertToUpperCaseSet(data.pipelineMeta.getTransformNames());
- Map<TargetTransformAttribute, SourceTransformField> targetMap = meta.getTargetSourceMapping();
- Set<TargetTransformAttribute> unavailableTargetTransforms =
- getUnavailableTargetTransforms(targetMap, data.pipelineMeta);
- Set<String> alreadyMarkedTransforms = new HashSet<>();
- for (TargetTransformAttribute currentTarget : unavailableTargetTransforms) {
- if (alreadyMarkedTransforms.contains(currentTarget.getTransformName())) {
- continue;
- }
- alreadyMarkedTransforms.add(currentTarget.getTransformName());
- if (existedTransformNames.contains(currentTarget.getTransformName().toUpperCase())) {
- logError(
- BaseMessages.getString(
- PKG,
- "MetaInject.TargetTransformIsNotUsed.Message",
- currentTarget.getTransformName(),
- data.pipelineMeta.getName()));
- } else {
- logError(
- BaseMessages.getString(
- PKG,
- "MetaInject.TargetTransformIsNotDefined.Message",
- currentTarget.getTransformName(),
- data.pipelineMeta.getName()));
- }
+ Set<String> unavailableTargetTransforms =
+ getUnavailableTargetTransforms(meta.getMappings(), data.pipelineMeta);
+ for (String unavailableTargetTransform : unavailableTargetTransforms) {
+ logError(
+ BaseMessages.getString(
+ PKG,
+ "MetaInject.TargetTransformIsNotDefined.Message",
+ unavailableTargetTransform,
+ data.pipelineMeta.getName()));
}
- // alreadyMarked contains wrong transforms. Hop-Gui can report error if it will not fail
- // transformation [BACKLOG-6753]
}
public static void removeUnavailableTransformsFromMapping(
- Map<TargetTransformAttribute, SourceTransformField> targetMap,
- Set<SourceTransformField> unavailableSourceTransforms,
- Set<TargetTransformAttribute> unavailableTargetTransforms) {
- Iterator<Entry<TargetTransformAttribute, SourceTransformField>> targetMapIterator =
- targetMap.entrySet().iterator();
- while (targetMapIterator.hasNext()) {
- Entry<TargetTransformAttribute, SourceTransformField> entry = targetMapIterator.next();
- SourceTransformField currentSourceTransformField = entry.getValue();
- TargetTransformAttribute currentTargetTransformAttribute = entry.getKey();
- if (unavailableSourceTransforms.contains(currentSourceTransformField)
- || unavailableTargetTransforms.contains(currentTargetTransformAttribute)) {
- targetMapIterator.remove();
- }
- }
+ List<MetaInjectMapping> mappings,
+ Set<String> unavailableSourceTransforms,
+ Set<String> unavailableTargetTransforms) {
+ mappings.removeIf(
+ mapping ->
+ unavailableSourceTransforms.contains(mapping.getSourceTransformName())
+ || unavailableTargetTransforms.contains(mapping.getTargetTransformName()));
}
- public static Set<TargetTransformAttribute> getUnavailableTargetTransforms(
- Map<TargetTransformAttribute, SourceTransformField> targetMap,
- PipelineMeta injectedPipelineMeta) {
- Set<String> usedTransformNames = getUsedTransformsForReferencedPipeline(injectedPipelineMeta);
- Set<TargetTransformAttribute> unavailableTargetTransforms = new HashSet<>();
- for (TargetTransformAttribute currentTarget : targetMap.keySet()) {
- if (!usedTransformNames.contains(currentTarget.getTransformName().toUpperCase())) {
- unavailableTargetTransforms.add(currentTarget);
- }
- }
- return Collections.unmodifiableSet(unavailableTargetTransforms);
+ public static Set<String> getUnavailableTargetTransforms(
+ List<MetaInjectMapping> mappings, PipelineMeta injectedPipelineMeta) {
+ // If we remove the available transforms from the used set,
+ // the ones that are left are the unavailable target transforms.
+ //
+ Set<String> availableTargetTransforms = getTransformsInPipeline(injectedPipelineMeta);
+ Set<String> usedTargetTransforms = getUsedTargetTransforms(mappings);
+ usedTargetTransforms.removeAll(availableTargetTransforms);
+ return Collections.unmodifiableSet(usedTargetTransforms);
}
- public static Set<TargetTransformAttribute> getUnavailableTargetKeys(
- Map<TargetTransformAttribute, SourceTransformField> targetMap,
+ public static Set<MetaInjectMapping> getUnavailableTargetKeys(
+ List<MetaInjectMapping> mappings,
PipelineMeta injectedPipelineMeta,
- Set<TargetTransformAttribute> unavailableTargetTransforms) {
- Set<TargetTransformAttribute> missingKeys = new HashSet<>();
+ Set<String> unavailableTargetTransforms) {
+ Set<MetaInjectMapping> missingKeys = new HashSet<>();
Map<String, BeanInjectionInfo> beanInfos = getUsedTransformBeanInfos(injectedPipelineMeta);
- for (TargetTransformAttribute key : targetMap.keySet()) {
- if (!unavailableTargetTransforms.contains(key)) {
- BeanInjectionInfo info = beanInfos.get(key.getTransformName().toUpperCase());
- if (info != null && !info.getProperties().containsKey(key.getAttributeKey())) {
- missingKeys.add(key);
+ for (MetaInjectMapping mapping : mappings) {
+ if (!unavailableTargetTransforms.contains(mapping.getTargetTransformName())) {
+ BeanInjectionInfo info = beanInfos.get(mapping.getTargetTransformName().toUpperCase());
+ if (info != null && !info.getProperties().containsKey(mapping.getTargetAttributeKey())) {
+ missingKeys.add(mapping);
}
}
}
@@ -638,7 +846,7 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
return res;
}
- private static Set<String> getUsedTransformsForReferencedPipeline(PipelineMeta pipelineMeta) {
+ private static Set<String> getTransformsInPipeline(PipelineMeta pipelineMeta) {
Set<String> usedTransformNames = new HashSet<>();
for (TransformMeta currentTransform : pipelineMeta.getUsedTransforms()) {
usedTransformNames.add(currentTransform.getName().toUpperCase());
@@ -646,37 +854,46 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> {
return usedTransformNames;
}
- public static Set<SourceTransformField> getUnavailableSourceTransforms(
- Map<TargetTransformAttribute, SourceTransformField> targetMap,
- PipelineMeta sourcePipelineMeta,
- TransformMeta transformMeta) {
- String[] transformNamesArray = sourcePipelineMeta.getPrevTransformNames(transformMeta);
- Set<String> existedTransformNames = convertToUpperCaseSet(transformNamesArray);
- Set<SourceTransformField> unavailableSourceTransforms = new HashSet<>();
- for (SourceTransformField currentSource : targetMap.values()) {
- if (currentSource.getTransformName() != null
- && !existedTransformNames.contains(currentSource.getTransformName().toUpperCase())) {
- unavailableSourceTransforms.add(currentSource);
+ private static Set<String> getUsedSourceTransforms(List<MetaInjectMapping> mappings) {
+ Set<String> transforms = new HashSet<>();
+ for (MetaInjectMapping mapping : mappings) {
+ // Watch out for constant values where the source transform is not specified.
+ if (StringUtils.isNotEmpty(mapping.getSourceTransformName())) {
+ transforms.add(mapping.getSourceTransformName().toUpperCase());
}
}
- return Collections.unmodifiableSet(unavailableSourceTransforms);
+ return transforms;
+ }
+
+ private static Set<String> getUsedTargetTransforms(List<MetaInjectMapping> mappings) {
+ Set<String> transforms = new HashSet<>();
+ for (MetaInjectMapping mapping : mappings) {
+ transforms.add(mapping.getTargetTransformName().toUpperCase());
+ }
+ return transforms;
+ }
+
+ public static Set<String> getUnavailableSourceTransforms(
+ List<MetaInjectMapping> mappings, PipelineMeta pipelineMeta, TransformMeta transformMeta) {
+ // The unavailable source transforms are the used ones minus the available
+ //
+ Set<String> availableSourceTransforms =
+ new HashSet<>(convertToUpperCaseSet(pipelineMeta.getPrevTransformNames(transformMeta)));
+ Set<String> usedSourceTransforms = getUsedSourceTransforms(mappings);
+ usedSourceTransforms.removeAll(availableSourceTransforms);
+ return Collections.unmodifiableSet(usedSourceTransforms);
}
private void checkSourceTransformsAvailability() {
- Map<TargetTransformAttribute, SourceTransformField> targetMap = meta.getTargetSourceMapping();
- Set<SourceTransformField> unavailableSourceTransforms =
- getUnavailableSourceTransforms(targetMap, getPipelineMeta(), getTransformMeta());
- Set<String> alreadyMarkedTransforms = new HashSet<>();
- for (SourceTransformField currentSource : unavailableSourceTransforms) {
- if (alreadyMarkedTransforms.contains(currentSource.getTransformName())) {
- continue;
- }
- alreadyMarkedTransforms.add(currentSource.getTransformName());
+ Set<String> unavailableSourceTransforms =
+ getUnavailableSourceTransforms(meta.getMappings(), getPipelineMeta(), getTransformMeta());
+
+ for (String unavailableTransform : unavailableSourceTransforms) {
logError(
BaseMessages.getString(
PKG,
"MetaInject.SourceTransformIsNotAvailable.Message",
- currentSource.getTransformName(),
+ unavailableTransform,
getPipelineMeta().getName()));
}
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectData.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectData.java
index 3ac6c41147..211b68fbc8 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectData.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectData.java
@@ -33,6 +33,8 @@ public class MetaInjectData extends BaseTransformData implements ITransformData
public boolean streaming;
public String streamingSourceTransformName;
public String streamingTargetTransformName;
+ public boolean receivedRows;
+ public boolean hasEmptyList;
public MetaInjectData() {
super();
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
index 35b3117c05..53d650309a 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
@@ -68,7 +68,6 @@ import org.eclipse.swt.custom.CTabItem;
import org.eclipse.swt.custom.ScrolledComposite;
import org.eclipse.swt.events.FocusAdapter;
import org.eclipse.swt.events.FocusEvent;
-import org.eclipse.swt.events.ModifyListener;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.graphics.Image;
@@ -101,6 +100,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
"MetaInjectDialog.ErrorLoadingSpecifiedPipeline.Message";
private final MetaInjectMeta metaInjectMeta;
+ private final ArrayList<MetaInjectMapping> targetMappings;
private TextVar wPath;
@@ -110,8 +110,6 @@ public class MetaInjectDialog extends BaseTransformDialog {
protected boolean transModified;
- private ModifyListener lsMod;
-
protected Label wlRunConfiguration;
protected ComboVar wRunConfiguration;
@@ -149,9 +147,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
//
private Tree wTree;
- private Map<TreeItem, TargetTransformAttribute> treeItemTargetMap;
-
- private final Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping;
+ private Map<TreeItem, MetaInjectMapping> treeItemTargetMap;
private Text wSearchText = null;
private String filterString = null;
@@ -162,8 +158,11 @@ public class MetaInjectDialog extends BaseTransformDialog {
metaInjectMeta = transformMeta;
transModified = false;
- targetSourceMapping = new HashMap<>();
- targetSourceMapping.putAll(metaInjectMeta.getTargetSourceMapping());
+ // Create a copy of the mappings to work with
+ targetMappings = new ArrayList<>();
+ for (MetaInjectMapping mapping : metaInjectMeta.getMappings()) {
+ targetMappings.add(new MetaInjectMapping(mapping));
+ }
}
@Override
@@ -275,13 +274,13 @@ public class MetaInjectDialog extends BaseTransformDialog {
return;
}
}
- Set<SourceTransformField> unavailableSourceTransforms =
- MetaInject.getUnavailableSourceTransforms(targetSourceMapping, pipelineMeta, transformMeta);
- Set<TargetTransformAttribute> unavailableTargetTransforms =
- MetaInject.getUnavailableTargetTransforms(targetSourceMapping, injectPipelineMeta);
- Set<TargetTransformAttribute> missingTargetKeys =
+ Set<String> unavailableSourceTransforms =
+ MetaInject.getUnavailableSourceTransforms(targetMappings, pipelineMeta, transformMeta);
+ Set<String> unavailableTargetTransforms =
+ MetaInject.getUnavailableTargetTransforms(targetMappings, injectPipelineMeta);
+ Set<MetaInjectMapping> missingTargetKeys =
MetaInject.getUnavailableTargetKeys(
- targetSourceMapping, injectPipelineMeta, unavailableTargetTransforms);
+ targetMappings, injectPipelineMeta, unavailableTargetTransforms);
if (unavailableSourceTransforms.isEmpty()
&& unavailableTargetTransforms.isEmpty()
&& missingTargetKeys.isEmpty()) {
@@ -292,18 +291,18 @@ public class MetaInjectDialog extends BaseTransformDialog {
}
private void showInvalidMappingDialog(
- Set<SourceTransformField> unavailableSourceTransforms,
- Set<TargetTransformAttribute> unavailableTargetTransforms,
- Set<TargetTransformAttribute> missingTargetKeys) {
+ Set<String> unavailableSourceTransforms,
+ Set<String> unavailableTargetTransforms,
+ Set<MetaInjectMapping> missingTargetKeys) {
MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
mb.setMessage(BaseMessages.getString(PKG, "MetaInjectDialog.InvalidMapping.Question"));
mb.setText(BaseMessages.getString(PKG, "MetaInjectDialog.InvalidMapping.Title"));
int id = mb.open();
if (id == SWT.YES) {
MetaInject.removeUnavailableTransformsFromMapping(
- targetSourceMapping, unavailableSourceTransforms, unavailableTargetTransforms);
- for (TargetTransformAttribute target : missingTargetKeys) {
- targetSourceMapping.remove(target);
+ metaInjectMeta.getMappings(), unavailableSourceTransforms, unavailableTargetTransforms);
+ for (MetaInjectMapping target : missingTargetKeys) {
+ targetMappings.remove(target);
}
}
}
@@ -661,37 +660,33 @@ public class MetaInjectDialog extends BaseTransformDialog {
wInjectSComp.setMinHeight(bounds.height);
wInjectTab.setControl(wInjectSComp);
-
- // ///////////////////////////////////////////////////////////
- // / END OF INJECT TAB
- // ///////////////////////////////////////////////////////////
}
+ private record TransformField(String transformName, String fieldName) {}
+
private void treeClicked(Event event) {
try {
Point point = new Point(event.x, event.y);
TreeItem item = wTree.getItem(point);
if (item != null) {
- TargetTransformAttribute target = treeItemTargetMap.get(item);
- if (target != null) {
- SourceTransformField source = targetSourceMapping.get(target);
-
+ MetaInjectMapping mapping = treeItemTargetMap.get(item);
+ if (mapping != null) {
String[] prevTransformNames = pipelineMeta.getPrevTransformNames(transformMeta);
Arrays.sort(prevTransformNames);
- Map<String, SourceTransformField> fieldMap = new HashMap<>();
+ Map<String, TransformField> fieldMap = new HashMap<>();
for (String prevTransformName : prevTransformNames) {
IRowMeta fields = pipelineMeta.getTransformFields(variables, prevTransformName);
for (IValueMeta field : fields.getValueMetaList()) {
String key = buildTransformFieldKey(prevTransformName, field.getName());
- fieldMap.put(key, new SourceTransformField(prevTransformName, field.getName()));
+ fieldMap.put(key, new TransformField(prevTransformName, field.getName()));
}
}
- String[] sourceFields = fieldMap.keySet().toArray(new String[fieldMap.size()]);
+ String[] sourceFields = fieldMap.keySet().toArray(new String[0]);
Arrays.sort(sourceFields);
String constant =
- source != null && source.getTransformName() == null ? source.getField() : "";
+ mapping.getSourceTransformName() == null ? mapping.getSourceField() : "";
EnterSelectionDialog selectSourceFieldDialog =
new EnterSelectionDialog(
shell,
@@ -701,38 +696,39 @@ public class MetaInjectDialog extends BaseTransformDialog {
constant,
variables);
selectSourceFieldDialog.setAddNoneOption(true);
- if (source != null) {
- if (source.getTransformName() != null && !Utils.isEmpty(source.getTransformName())) {
- String key = buildTransformFieldKey(source.getTransformName(), source.getField());
- selectSourceFieldDialog.setCurrentValue(key);
- int index = Const.indexOfString(key, sourceFields);
- if (index >= 0) {
- selectSourceFieldDialog.setSelectedNrs(
- new int[] {
- index,
- });
- }
- } else {
- selectSourceFieldDialog.setCurrentValue(source.getField());
+ if (mapping.getSourceTransformName() != null
+ && !Utils.isEmpty(mapping.getSourceTransformName())) {
+ String key =
+ buildTransformFieldKey(mapping.getSourceTransformName(), mapping.getSourceField());
+ selectSourceFieldDialog.setCurrentValue(key);
+ int index = Const.indexOfString(key, sourceFields);
+ if (index >= 0) {
+ selectSourceFieldDialog.setSelectedNrs(
+ new int[] {
+ index,
+ });
}
+ } else {
+ selectSourceFieldDialog.setCurrentValue(mapping.getSourceField());
}
String selectedTransformField = selectSourceFieldDialog.open();
if (selectedTransformField != null) {
- SourceTransformField newSource = fieldMap.get(selectedTransformField);
+ TransformField newSource = fieldMap.get(selectedTransformField);
if (newSource == null) {
- newSource = new SourceTransformField(null, selectedTransformField);
+ newSource = new TransformField(null, selectedTransformField);
item.setText(1, CONST_VALUE);
item.setText(2, selectedTransformField);
} else {
- item.setText(1, newSource.getTransformName());
- item.setText(2, newSource.getField());
+ item.setText(1, newSource.transformName);
+ item.setText(2, newSource.fieldName);
}
- targetSourceMapping.put(target, newSource);
+ mapping.setSourceTransformName(newSource.transformName);
+ mapping.setSourceField(newSource.fieldName);
} else {
if (selectSourceFieldDialog.isNoneClicked()) {
item.setText(1, "");
item.setText(2, "");
- targetSourceMapping.remove(target);
+ targetMappings.remove(mapping);
}
}
}
@@ -794,10 +790,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
wPath.setText(filename);
}
loadPipelineFile(filename);
- if (injectPipelineMeta == null) {
- return false;
- }
- return true;
+ return injectPipelineMeta != null;
}
public void updateWidgets() {
@@ -811,21 +804,13 @@ public class MetaInjectDialog extends BaseTransformDialog {
/** Copy information from the meta-data input to the dialog fields. */
public void getData() {
- wPath.setText(Const.NVL(metaInjectMeta.getFileName(), ""));
+ wPath.setText(Const.NVL(metaInjectMeta.getTemplateFileName(), ""));
try {
List<String> runConfigurations =
metadataProvider.getSerializer(PipelineRunConfiguration.class).listObjectNames();
- try {
- ExtensionPointHandler.callExtensionPoint(
- HopGui.getInstance().getLog(),
- variables,
- HopExtensionPoint.HopGuiRunConfiguration.id,
- new Object[] {runConfigurations, PipelineMeta.XML_TAG});
- } catch (HopException e) {
- // Ignore errors
- }
+ runExtensionPointHopGuiRunConfiguration(runConfigurations);
wRunConfiguration.setItems(runConfigurations.toArray(new String[0]));
wRunConfiguration.setText(Const.NVL(metaInjectMeta.getRunConfigurationName(), ""));
@@ -857,12 +842,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
wNoExecution.setSelection(!metaInjectMeta.isNoExecution());
wAllowEmptyStreamOnExecution.setSelection(metaInjectMeta.isAllowEmptyStreamOnExecution());
- wStreamingSourceTransform.setText(
- Const.NVL(
- metaInjectMeta.getStreamSourceTransform() == null
- ? null
- : metaInjectMeta.getStreamSourceTransform().getName(),
- ""));
+ wStreamingSourceTransform.setText(Const.NVL(metaInjectMeta.getStreamSourceTransformName(), ""));
wStreamingTargetTransform.setText(Const.NVL(metaInjectMeta.getStreamTargetTransformName(), ""));
updateWidgets();
@@ -871,6 +851,18 @@ public class MetaInjectDialog extends BaseTransformDialog {
wTabFolder.setSelection(0);
}
+ private void runExtensionPointHopGuiRunConfiguration(List<String> runConfigurations) {
+ try {
+ ExtensionPointHandler.callExtensionPoint(
+ HopGui.getInstance().getLog(),
+ variables,
+ HopExtensionPoint.HopGuiRunConfiguration.id,
+ new Object[] {runConfigurations, PipelineMeta.XML_TAG});
+ } catch (HopException e) {
+ // Ignore errors
+ }
+ }
+
protected String buildTransformFieldKey(String transformName, String field) {
return transformName + " : " + field;
}
@@ -916,13 +908,12 @@ public class MetaInjectDialog extends BaseTransformDialog {
//
ITransformMeta metaInterface = transformMeta.getTransform();
if (BeanInjectionInfo.isInjectionSupported(metaInterface.getClass())) {
- expanded = expanded || processMDIDescription(transformMeta, transformItem, metaInterface);
+ expanded = processMDIDescription(transformMeta, transformItem, metaInterface);
}
transformItem.setExpanded(expanded);
}
-
- } catch (Throwable t) {
+ } catch (Exception e) {
// Ignore errors
}
@@ -945,7 +936,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
private boolean processMDIDescription(
TransformMeta transformMeta, TreeItem transformItem, ITransformMeta metaInterface) {
boolean hasUsedKeys = false;
- BeanInjectionInfo transformInjectionInfo = new BeanInjectionInfo(metaInterface.getClass());
+ BeanInjectionInfo transformInjectionInfo = new BeanInjectionInfo<>(metaInterface.getClass());
List<BeanInjectionInfo.Group> groupsList = transformInjectionInfo.getGroups();
@@ -966,7 +957,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
List<BeanInjectionInfo.Property> propertyList = gr.getProperties();
- for (BeanInjectionInfo.Property property : propertyList) {
+ for (BeanInjectionInfo<?>.Property property : propertyList) {
if (!property.hasMatch(filterString)) {
continue;
}
@@ -975,19 +966,28 @@ public class MetaInjectDialog extends BaseTransformDialog {
TreeItem treeItem = new TreeItem(rootGroup ? transformItem : groupItem, SWT.NONE);
treeItem.setText(Const.NVL(property.getTranslatedDescription(), property.getKey()));
- TargetTransformAttribute target =
- new TargetTransformAttribute(transformMeta.getName(), property.getKey(), !rootGroup);
- treeItemTargetMap.put(treeItem, target);
+ MetaInjectMapping targetMapping = new MetaInjectMapping();
+ targetMapping.setTargetTransformName(transformMeta.getName());
+ targetMapping.setTargetAttributeKey(property.getKey());
+ targetMapping.setTargetDetail(!rootGroup);
+
+ treeItemTargetMap.put(treeItem, targetMapping);
- SourceTransformField source = targetSourceMapping.get(target);
- if (source != null) {
+ // Equals only works on the target fields
+ int index = targetMappings.indexOf(targetMapping);
+ if (index >= 0) {
+ MetaInjectMapping mapping = targetMappings.get(index);
+ // Update with source mapping information
+ treeItemTargetMap.put(treeItem, mapping);
hasUsedKeys = true;
treeItem.setText(
1,
Const.NVL(
- source.getTransformName() == null ? CONST_VALUE : source.getTransformName(),
+ mapping.getSourceTransformName() == null
+ ? CONST_VALUE
+ : mapping.getSourceTransformName(),
""));
- treeItem.setText(2, Const.NVL(source.getField(), ""));
+ treeItem.setText(2, Const.NVL(mapping.getSourceField(), ""));
}
}
}
@@ -1053,7 +1053,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
}
private void getInfo(MetaInjectMeta meta) {
- meta.setFileName(wPath.getText());
+ meta.setTemplateFileName(wPath.getText());
meta.setSourceTransformName(wSourceTransform.getText());
meta.setRunConfigurationName(wRunConfiguration.getText());
@@ -1073,27 +1073,15 @@ public class MetaInjectDialog extends BaseTransformDialog {
meta.setNoExecution(!wNoExecution.getSelection());
meta.setAllowEmptyStreamOnExecution(wAllowEmptyStreamOnExecution.getSelection());
- final TransformMeta streamSourceTransform =
- pipelineMeta.findTransform(wStreamingSourceTransform.getText());
- meta.setStreamSourceTransform(streamSourceTransform);
- // Save streamSourceTransformName to find streamSourceTransform when loading
- meta.setStreamSourceTransformName(
- streamSourceTransform != null ? streamSourceTransform.getName() : "");
+ meta.setStreamSourceTransformName(wStreamingSourceTransform.getText());
meta.setStreamTargetTransformName(wStreamingTargetTransform.getText());
- meta.setTargetSourceMapping(targetSourceMapping);
+ meta.getMappings().clear();
+ meta.getMappings().addAll(targetMappings);
meta.setChanged(true);
}
- private class MappingSource {
- public TransformMeta transformMeta;
- public IValueMeta valueMeta;
-
- public MappingSource(TransformMeta transformMeta, IValueMeta valueMeta) {
- this.transformMeta = transformMeta;
- this.valueMeta = valueMeta;
- }
-
+ private record MappingSource(TransformMeta transformMeta, IValueMeta valueMeta) {
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -1108,17 +1096,7 @@ public class MetaInjectDialog extends BaseTransformDialog {
}
}
- private class MappingTarget {
- public TransformMeta transformMeta;
- public String attributeKey;
- public boolean detail;
-
- public MappingTarget(TransformMeta transformMeta, String attributeKey, boolean detail) {
- this.transformMeta = transformMeta;
- this.attributeKey = attributeKey;
- this.detail = detail;
- }
-
+ private record MappingTarget(TransformMeta transformMeta, String attributeKey, boolean detail) {
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -1131,11 +1109,15 @@ public class MetaInjectDialog extends BaseTransformDialog {
return Objects.equals(transformMeta, that.transformMeta)
&& Objects.equals(attributeKey, that.attributeKey);
}
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(transformMeta, attributeKey);
+ }
}
/** Enter the mapping between (unmapped) source fields and target fields. */
private void enterMapping() {
-
try {
loadPipeline();
} catch (HopException e) {
@@ -1211,21 +1193,14 @@ public class MetaInjectDialog extends BaseTransformDialog {
// Calculate the existing mappings...
//
List<SourceToTargetMapping> mappings = new ArrayList<>();
- Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping =
- meta.getTargetSourceMapping();
- for (TargetTransformAttribute targetTransformAttribute : targetSourceMapping.keySet()) {
- SourceTransformField sourceTransformField = targetSourceMapping.get(targetTransformAttribute);
- if (sourceTransformField == null) {
- continue;
- }
+ for (MetaInjectMapping mapping : meta.getMappings()) {
int sourceIndex = -1;
TransformMeta sourceTransformMeta =
- pipelineMeta.findTransform(sourceTransformField.getTransformName());
+ pipelineMeta.findTransform(mapping.getSourceTransformName());
if (sourceTransformMeta != null) {
IRowMeta sourceRowMeta = sourceRowMetas.get(sourceTransformMeta.getName());
if (sourceRowMeta != null) {
- IValueMeta sourceValueMeta =
- sourceRowMeta.searchValueMeta(sourceTransformField.getField());
+ IValueMeta sourceValueMeta = sourceRowMeta.searchValueMeta(mapping.getSourceField());
if (sourceValueMeta != null) {
MappingSource mappingSource = new MappingSource(sourceTransformMeta, sourceValueMeta);
sourceIndex = mappingSources.indexOf(mappingSource);
@@ -1234,13 +1209,11 @@ public class MetaInjectDialog extends BaseTransformDialog {
}
int targetIndex = -1;
TransformMeta targetTransformMeta =
- injectPipelineMeta.findTransform(targetTransformAttribute.getTransformName());
+ injectPipelineMeta.findTransform(mapping.getTargetTransformName());
if (targetTransformMeta != null) {
MappingTarget mapingTarget =
new MappingTarget(
- targetTransformMeta,
- targetTransformAttribute.getAttributeKey(),
- targetTransformAttribute.isDetail());
+ targetTransformMeta, mapping.getTargetAttributeKey(), mapping.isTargetDetail());
targetIndex = mappingTargets.indexOf(mapingTarget);
}
if (sourceIndex >= 0 && targetIndex >= 0) {
@@ -1259,27 +1232,25 @@ public class MetaInjectDialog extends BaseTransformDialog {
// Add the mappings...
//
- targetSourceMapping.clear();
+ targetMappings.clear();
for (SourceToTargetMapping newMapping : newMappings) {
+ MetaInjectMapping mapping = new MetaInjectMapping();
MappingSource mappingSource = mappingSources.get(newMapping.getSourcePosition());
- SourceTransformField sourceTransformField =
- new SourceTransformField(
- mappingSource.transformMeta.getName(), mappingSource.valueMeta.getName());
-
MappingTarget mappingTarget = mappingTargets.get(newMapping.getTargetPosition());
- TargetTransformAttribute targetTransformAttribute =
- new TargetTransformAttribute(
- mappingTarget.transformMeta.getName(),
- mappingTarget.attributeKey,
- mappingTarget.detail);
- targetSourceMapping.put(targetTransformAttribute, sourceTransformField);
+ mapping.setSourceTransformName(mappingSource.transformMeta.getName());
+ mapping.setSourceField(mappingSource.valueMeta.getName());
+ mapping.setTargetTransformName(mappingTarget.transformMeta.getName());
+ mapping.setTargetAttributeKey(mappingTarget.attributeKey);
+ mapping.setTargetDetail(mappingTarget.detail);
- // Refresh the tree...
- //
- refreshTree();
+ targetMappings.add(mapping);
}
+
+ // Refresh the tree...
+ //
+ refreshTree();
}
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMapping.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMapping.java
index 0e9a4bdc73..100ca153c8 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMapping.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMapping.java
@@ -17,55 +17,70 @@
package org.apache.hop.pipeline.transforms.metainject;
-import org.apache.hop.core.injection.Injection;
-
+import java.util.Objects;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.Const;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+
+@Getter
+@Setter
public class MetaInjectMapping {
-
- @Injection(name = "MAPPING_SOURCE_TRANSFORM", group = "MAPPING_FIELDS")
- private String sourceTransform;
-
- @Injection(name = "MAPPING_SOURCE_FIELD", group = "MAPPING_FIELDS")
+ @HopMetadataProperty(
+ key = "source_transform",
+ injectionKey = "MAPPING_SOURCE_TRANSFORM",
+ injectionKeyDescription = "MetaInject.Injection.MAPPING_SOURCE_TRANSFORM")
+ private String sourceTransformName;
+
+ @HopMetadataProperty(
+ key = "source_field",
+ injectionKey = "MAPPING_SOURCE_FIELD",
+ injectionKeyDescription = "MetaInject.Injection.MAPPING_SOURCE_FIELD")
private String sourceField;
- @Injection(name = "MAPPING_TARGET_TRANSFORM", group = "MAPPING_FIELDS")
- private String targetTransform;
-
- @Injection(name = "MAPPING_TARGET_FIELD", group = "MAPPING_FIELDS")
- private String targetField;
-
- public MetaInjectMapping() {
- // Do nothing
- }
-
- public String getSourceTransform() {
- return sourceTransform;
- }
-
- public void setSourceTransform(String sourceTransform) {
- this.sourceTransform = sourceTransform;
- }
-
- public String getSourceField() {
- return sourceField;
- }
-
- public void setSourceField(String sourceField) {
- this.sourceField = sourceField;
- }
-
- public String getTargetTransform() {
- return targetTransform;
- }
-
- public void setTargetTransform(String targetTransform) {
- this.targetTransform = targetTransform;
+ @HopMetadataProperty(
+ key = "target_transform_name",
+ injectionKey = "MAPPING_TARGET_TRANSFORM",
+ injectionKeyDescription = "MetaInject.Injection.MAPPING_TARGET_TRANSFORM")
+ private String targetTransformName;
+
+ @HopMetadataProperty(
+ key = "target_attribute_key",
+ injectionKey = "MAPPING_TARGET_FIELD",
+ injectionKeyDescription = "MetaInject.Injection.MAPPING_TARGET_FIELD")
+ private String targetAttributeKey;
+
+ @HopMetadataProperty(
+ key = "target_detail",
+ injectionKey = "MAPPING_TARGET_DETAIL",
+ injectionKeyDescription = "MetaInject.Injection.MAPPING_TARGET_DETAIL")
+ private boolean targetDetail;
+
+ public MetaInjectMapping() {}
+
+ public MetaInjectMapping(MetaInjectMapping m) {
+ this();
+ this.sourceField = m.sourceField;
+ this.sourceTransformName = m.sourceTransformName;
+ this.targetAttributeKey = m.targetAttributeKey;
+ this.targetDetail = m.targetDetail;
+ this.targetTransformName = m.targetTransformName;
}
- public String getTargetField() {
- return targetField;
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof MetaInjectMapping mapping)) {
+ return false;
+ }
+ return targetDetail == mapping.targetDetail
+ && Const.NVL(targetTransformName, "")
+ .equalsIgnoreCase(Const.NVL(mapping.targetTransformName, ""))
+ && Const.NVL(targetAttributeKey, "")
+ .equalsIgnoreCase(Const.NVL(mapping.targetAttributeKey, ""));
}
- public void setTargetField(String targetField) {
- this.targetField = targetField;
+ @Override
+ public int hashCode() {
+ return Objects.hash(targetTransformName, targetAttributeKey, targetDetail);
}
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
index e853d26471..253aef0db3 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
@@ -18,29 +18,23 @@
package org.apache.hop.pipeline.transforms.metainject;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.hop.core.Const;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.annotations.ActionTransformType;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.exception.HopTransformException;
-import org.apache.hop.core.exception.HopXmlException;
import org.apache.hop.core.file.IHasFilename;
-import org.apache.hop.core.injection.Injection;
-import org.apache.hop.core.injection.InjectionDeep;
-import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.row.IRowMeta;
-import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.util.CurrentDirectoryResolver;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.ISubPipelineAwareMeta;
import org.apache.hop.pipeline.PipelineHopMeta;
@@ -52,7 +46,6 @@ import org.apache.hop.resource.IResourceNaming;
import org.apache.hop.resource.ResourceDefinition;
import org.apache.hop.resource.ResourceEntry;
import org.apache.hop.resource.ResourceReference;
-import org.w3c.dom.Node;
@Transform(
id = "MetaInject",
@@ -63,216 +56,106 @@ import org.w3c.dom.Node;
keywords = "i18n::MetaInjectMeta.keyword",
documentationUrl = "/pipeline/transforms/metainject.html",
actionTransformTypes = {ActionTransformType.HOP_FILE, ActionTransformType.HOP_PIPELINE})
-@InjectionSupported(
- localizationPrefix = "MetaInject.Injection.",
- groups = {"SOURCE_OUTPUT_FIELDS", "MAPPING_FIELDS"})
+@Getter
+@Setter
public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData>
implements ITransformMetaChangeListener, ISubPipelineAwareMeta {
private static final Class<?> PKG = MetaInjectMeta.class;
- private static final String MAPPINGS = "mappings";
- private static final String MAPPING = "mapping";
-
- private static final String FILENAME = "filename";
- private static final String RUN_CONFIG = "run_configuration";
- private static final String TARGET_FILE = "target_file";
- private static final String CREATE_PARENT_FOLDER = "create_parent_folder";
- private static final String NO_EXECUTION = "no_execution";
- private static final String ALLOW_EMPTY_STREAM_ON_EXECUTION = "allow_empty_stream_on_execution";
- private static final String SOURCE_TRANSFORM = "source_transform";
-
- private static final String STREAM_SOURCE_TRANSFORM = "stream_source_transform";
- private static final String STREAM_TARGET_TRANSFORM = "stream_target_transform";
- private static final String TARGET_TRANSFORM_NAME = "target_transform_name";
- private static final String TARGET_ATTRIBUTE_KEY = "target_attribute_key";
- private static final String TARGET_DETAIL = "target_detail";
- private static final String SOURCE_FIELD = "source_field";
- private static final String SOURCE_OUTPUT_FIELDS = "source_output_fields";
- private static final String SOURCE_OUTPUT_FIELD = "source_output_field";
- private static final String SOURCE_OUTPUT_FIELD_NAME = "source_output_field_name";
- private static final String SOURCE_OUTPUT_FIELD_TYPE = "source_output_field_type";
- private static final String SOURCE_OUTPUT_FIELD_LENGTH = "source_output_field_length";
- private static final String SOURCE_OUTPUT_FIELD_PRECISION = "source_output_field_precision";
-
- private static final String GROUP_AND_NAME_DELIMITER = ".";
- private static final String CONST_SPACE = " ";
- private static final String CONST_SPACE_LONG = " ";
-
- // description of the transformation to execute...
- //
- @Injection(name = "FILE_NAME")
- private String fileName;
-
- @Injection(name = "SOURCE_TRANSFORM_NAME")
- private String sourceTransformName;
-
- @InjectionDeep private List<MetaInjectOutputField> sourceOutputFields;
+ @HopMetadataProperty(
+ key = "filename",
+ injectionKey = "FILE_NAME",
+ injectionKeyDescription = "MetaInject.Injection.FILE_NAME")
+ private String templateFileName;
- private Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping;
-
- @InjectionDeep private List<MetaInjectMapping> metaInjectMapping;
+ @HopMetadataProperty(
+ key = "source_transform",
+ injectionKey = "SOURCE_TRANSFORM_NAME",
+ injectionKeyDescription = "MetaInject.Injection.SOURCE_TRANSFORM_NAME")
+ private String sourceTransformName;
- @Injection(name = "TARGET_FILE")
+ @HopMetadataProperty(
+ key = "source_output_field",
+ groupKey = "source_output_fields",
+ injectionGroupKey = "SOURCE_OUTPUT_FIELDS",
+ injectionGroupDescription = "MetaInject.Injection.SOURCE_OUTPUT_FIELDS")
+ private List<MetaInjectOutputField> sourceOutputFields;
+
+ @HopMetadataProperty(
+ key = "mapping",
+ groupKey = "mappings",
+ injectionGroupKey = "MAPPING_FIELDS",
+ injectionGroupDescription = "MetaInject.Injection.MAPPING_FIELDS")
+ private List<MetaInjectMapping> mappings;
+
+ @HopMetadataProperty(
+ key = "target_file",
+ injectionKey = "TARGET_FILE",
+ injectionKeyDescription = "MetaInject.Injection.TARGET_FILE")
private String targetFile;
- @Injection(name = "NO_EXECUTION")
+ @HopMetadataProperty(
+ key = "no_execution",
+ injectionKey = "NO_EXECUTION",
+ injectionKeyDescription = "MetaInject.Injection.NO_EXECUTION")
private boolean noExecution;
- @Injection(name = "ALLOW_EMPTY_STREAM_ON_EXECUTION")
+ @HopMetadataProperty(
+ key = "allow_empty_stream_on_execution",
+ injectionKey = "ALLOW_EMPTY_STREAM_ON_EXECUTION",
+ injectionKeyDescription = "MetaInject.Injection.ALLOW_EMPTY_STREAM_ON_EXECUTION")
private boolean allowEmptyStreamOnExecution;
- @Injection(name = "STREAMING_SOURCE_TRANSFORM")
+ @HopMetadataProperty(
+ key = "stream_source_transform",
+ injectionKey = "STREAMING_SOURCE_TRANSFORM",
+ injectionKeyDescription = "MetaInject.Injection.STREAMING_SOURCE_TRANSFORM")
private String streamSourceTransformName;
- private TransformMeta streamSourceTransform;
-
- @Injection(name = "STREAMING_TARGET_TRANSFORM")
+ @HopMetadataProperty(
+ key = "stream_target_transform",
+ injectionKey = "STREAMING_TARGET_TRANSFORM",
+ injectionKeyDescription = "MetaInject.Injection.STREAMING_TARGET_TRANSFORM")
private String streamTargetTransformName;
+ @HopMetadataProperty(
+ key = "run_configuration",
+ injectionKey = "RUN_CONFIGURATION",
+ injectionKeyDescription = "MetaInject.Injection.RUN_CONFIGURATION")
private String runConfigurationName;
+ @HopMetadataProperty(
+ key = "create_parent_folder",
+ injectionKey = "CREATE_PARENT_FOLDER",
+ injectionKeyDescription = "MetaInject.Injection.CREATE_PARENT_FOLDER")
private boolean createParentFolder;
public MetaInjectMeta() {
- super(); // allocate BaseTransformMeta
- targetSourceMapping = new HashMap<>();
+ super();
+ mappings = new ArrayList<>();
sourceOutputFields = new ArrayList<>();
+ createParentFolder = true;
}
- @Override
- public Object clone() {
- Object retval = super.clone();
- return retval;
- }
-
- @Override
- public String getXml() {
- actualizeMetaInjectMapping();
- StringBuilder retval = new StringBuilder(500);
-
- retval.append(" ").append(XmlHandler.addTagValue(FILENAME, fileName));
- retval.append(" ").append(XmlHandler.addTagValue(RUN_CONFIG, runConfigurationName));
-
- retval.append(" ").append(XmlHandler.addTagValue(SOURCE_TRANSFORM, sourceTransformName));
- retval.append(" ").append(XmlHandler.openTag(SOURCE_OUTPUT_FIELDS));
- for (MetaInjectOutputField field : sourceOutputFields) {
- retval.append(CONST_SPACE).append(XmlHandler.openTag(SOURCE_OUTPUT_FIELD));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_OUTPUT_FIELD_NAME, field.getName()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_OUTPUT_FIELD_TYPE, field.getTypeDescription()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_OUTPUT_FIELD_LENGTH, field.getLength()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_OUTPUT_FIELD_PRECISION, field.getPrecision()));
- retval.append(CONST_SPACE).append(XmlHandler.closeTag(SOURCE_OUTPUT_FIELD));
- }
- retval.append(" ").append(XmlHandler.closeTag(SOURCE_OUTPUT_FIELDS));
-
- retval.append(" ").append(XmlHandler.addTagValue(TARGET_FILE, targetFile));
- retval.append(" ").append(XmlHandler.addTagValue(CREATE_PARENT_FOLDER, createParentFolder));
- retval.append(" ").append(XmlHandler.addTagValue(NO_EXECUTION, noExecution));
- retval
- .append(" ")
- .append(
- XmlHandler.addTagValue(ALLOW_EMPTY_STREAM_ON_EXECUTION, allowEmptyStreamOnExecution));
-
- if ((streamSourceTransformName == null) && (streamSourceTransform != null)) {
- streamSourceTransformName = streamSourceTransform.getName();
- }
- retval
- .append(" ")
- .append(XmlHandler.addTagValue(STREAM_SOURCE_TRANSFORM, streamSourceTransformName));
- retval
- .append(" ")
- .append(XmlHandler.addTagValue(STREAM_TARGET_TRANSFORM, streamTargetTransformName));
-
- retval.append(" ").append(XmlHandler.openTag(MAPPINGS));
- for (TargetTransformAttribute target : targetSourceMapping.keySet()) {
- retval.append(CONST_SPACE).append(XmlHandler.openTag(MAPPING));
- SourceTransformField source = targetSourceMapping.get(target);
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(TARGET_TRANSFORM_NAME, target.getTransformName()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(TARGET_ATTRIBUTE_KEY, target.getAttributeKey()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(TARGET_DETAIL, target.isDetail()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_TRANSFORM, source.getTransformName()));
- retval
- .append(CONST_SPACE_LONG)
- .append(XmlHandler.addTagValue(SOURCE_FIELD, source.getField()));
- retval.append(CONST_SPACE).append(XmlHandler.closeTag(MAPPING));
- }
- retval.append(" ").append(XmlHandler.closeTag(MAPPINGS));
-
- return retval.toString();
+ public MetaInjectMeta(MetaInjectMeta m) {
+ this();
+ this.allowEmptyStreamOnExecution = m.allowEmptyStreamOnExecution;
+ this.createParentFolder = m.createParentFolder;
+ this.noExecution = m.noExecution;
+ this.runConfigurationName = m.runConfigurationName;
+ this.sourceTransformName = m.sourceTransformName;
+ this.streamSourceTransformName = m.streamSourceTransformName;
+ this.streamTargetTransformName = m.streamTargetTransformName;
+ this.targetFile = m.targetFile;
+ this.templateFileName = m.templateFileName;
+ m.sourceOutputFields.forEach(f -> this.sourceOutputFields.add(new MetaInjectOutputField(f)));
+ m.mappings.forEach(map -> this.mappings.add(new MetaInjectMapping(map)));
}
@Override
- public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider)
- throws HopXmlException {
- try {
- fileName = XmlHandler.getTagValue(transformNode, FILENAME);
- runConfigurationName = XmlHandler.getTagValue(transformNode, RUN_CONFIG);
-
- sourceTransformName = XmlHandler.getTagValue(transformNode, SOURCE_TRANSFORM);
- Node outputFieldsNode = XmlHandler.getSubNode(transformNode, SOURCE_OUTPUT_FIELDS);
- List<Node> outputFieldNodes = XmlHandler.getNodes(outputFieldsNode, SOURCE_OUTPUT_FIELD);
- sourceOutputFields = new ArrayList<>();
- for (Node outputFieldNode : outputFieldNodes) {
- String name = XmlHandler.getTagValue(outputFieldNode, SOURCE_OUTPUT_FIELD_NAME);
- String typeName = XmlHandler.getTagValue(outputFieldNode, SOURCE_OUTPUT_FIELD_TYPE);
- int length =
- Const.toInt(XmlHandler.getTagValue(outputFieldNode, SOURCE_OUTPUT_FIELD_LENGTH), -1);
- int precision =
- Const.toInt(XmlHandler.getTagValue(outputFieldNode, SOURCE_OUTPUT_FIELD_PRECISION), -1);
- int type = ValueMetaFactory.getIdForValueMeta(typeName);
- sourceOutputFields.add(new MetaInjectOutputField(name, type, length, precision));
- }
-
- targetFile = XmlHandler.getTagValue(transformNode, TARGET_FILE);
- createParentFolder =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, CREATE_PARENT_FOLDER));
- noExecution = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, NO_EXECUTION));
- allowEmptyStreamOnExecution =
- "Y"
- .equalsIgnoreCase(
- XmlHandler.getTagValue(transformNode, ALLOW_EMPTY_STREAM_ON_EXECUTION));
-
- streamSourceTransformName = XmlHandler.getTagValue(transformNode, STREAM_SOURCE_TRANSFORM);
- streamTargetTransformName = XmlHandler.getTagValue(transformNode, STREAM_TARGET_TRANSFORM);
-
- Node mappingsNode = XmlHandler.getSubNode(transformNode, MAPPINGS);
- int nrMappings = XmlHandler.countNodes(mappingsNode, MAPPING);
- for (int i = 0; i < nrMappings; i++) {
- Node mappingNode = XmlHandler.getSubNodeByNr(mappingsNode, MAPPING, i);
- String targetTransformName = XmlHandler.getTagValue(mappingNode, TARGET_TRANSFORM_NAME);
- String targetAttributeKey = XmlHandler.getTagValue(mappingNode, TARGET_ATTRIBUTE_KEY);
- boolean targetDetail =
- "Y".equalsIgnoreCase(XmlHandler.getTagValue(mappingNode, TARGET_DETAIL));
- String sourceTransformName = XmlHandler.getTagValue(mappingNode, SOURCE_TRANSFORM);
- String sourceField = XmlHandler.getTagValue(mappingNode, SOURCE_FIELD);
-
- TargetTransformAttribute target =
- new TargetTransformAttribute(targetTransformName, targetAttributeKey, targetDetail);
- SourceTransformField source = new SourceTransformField(sourceTransformName, sourceField);
- targetSourceMapping.put(target, source);
- }
-
- MetaInjectMigration.migrateFrom70(targetSourceMapping);
- } catch (Exception e) {
- throw new HopXmlException("Unable to load transform info from XML", e);
- }
+ public Object clone() {
+ return new MetaInjectMeta(this);
}
@Override
@@ -298,30 +181,7 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
}
}
- public Map<TargetTransformAttribute, SourceTransformField> getTargetSourceMapping() {
- return targetSourceMapping;
- }
-
- public void setTargetSourceMapping(
- Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping) {
- this.targetSourceMapping = targetSourceMapping;
- }
-
- /**
- * @return the fileName
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * @param fileName the fileName to set
- */
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
- public static final synchronized PipelineMeta loadPipelineMeta(
+ public static synchronized PipelineMeta loadPipelineMeta(
MetaInjectMeta injectMeta, IHopMetadataProvider metadataProvider, IVariables variables)
throws HopException {
PipelineMeta mappingPipelineMeta = null;
@@ -329,20 +189,16 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
CurrentDirectoryResolver resolver = new CurrentDirectoryResolver();
IVariables tmpSpace =
resolver.resolveCurrentDirectory(
- variables, injectMeta.getParentTransformMeta(), injectMeta.getFileName());
+ variables, injectMeta.getParentTransformMeta(), injectMeta.getTemplateFileName());
- String realFilename = tmpSpace.resolve(injectMeta.getFileName());
+ String realFilename = tmpSpace.resolve(injectMeta.getTemplateFileName());
try {
- // OK, load the meta-data from file...
+ // Load the meta-data from file.
//
- // Don't set internal variables: they belong to the parent thread!
- //
- if (mappingPipelineMeta == null) {
- mappingPipelineMeta = new PipelineMeta(realFilename, metadataProvider, tmpSpace);
- LogChannel.GENERAL.logDetailed(
- "Loading Mapping from repository",
- "Mapping transformation was loaded from XML file [" + realFilename + "]");
- }
+ mappingPipelineMeta = new PipelineMeta(realFilename, metadataProvider, tmpSpace);
+ LogChannel.GENERAL.logDetailed(
+ "Loading Mapping from repository",
+ "Mapping transformation was loaded from XML file [" + realFilename + "]");
} catch (Exception e) {
throw new HopException(
BaseMessages.getString(
@@ -368,7 +224,7 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
IVariables variables, TransformMeta transformMeta) {
List<ResourceReference> references = new ArrayList<>(5);
- String realFilename = variables.resolve(fileName);
+ String realFilename = variables.resolve(templateFileName);
ResourceReference reference = new ResourceReference(transformMeta);
references.add(reference);
@@ -408,23 +264,19 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
executorPipelineMeta.exportResources(
variables, definitions, resourceNamingInterface, metadataProvider);
- // To get a relative path to it, we inject
- // ${Internal.Entry.Current.Directory}
- //
- String newFilename = proposedNewFilename;
-
// Set the correct filename inside the XML.
//
- executorPipelineMeta.setFilename(newFilename);
+ executorPipelineMeta.setFilename(proposedNewFilename);
// change it in the entry
//
- fileName = newFilename;
+ templateFileName = proposedNewFilename;
return proposedNewFilename;
} catch (Exception e) {
throw new HopException(
- BaseMessages.getString(PKG, "MetaInjectMeta.Exception.UnableToLoadTrans", fileName));
+ BaseMessages.getString(
+ PKG, "MetaInjectMeta.Exception.UnableToLoadTrans", templateFileName));
}
}
@@ -438,78 +290,6 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
return true;
}
- /**
- * @return the sourceTransformName
- */
- public String getSourceTransformName() {
- return sourceTransformName;
- }
-
- /**
- * @param sourceTransformName the sourceTransformName to set
- */
- public void setSourceTransformName(String sourceTransformName) {
- this.sourceTransformName = sourceTransformName;
- }
-
- /**
- * @return the targetFile
- */
- public String getTargetFile() {
- return targetFile;
- }
-
- /**
- * @param targetFile the targetFile to set
- */
- public void setTargetFile(String targetFile) {
- this.targetFile = targetFile;
- }
-
- /**
- * @return the noExecution
- */
- public boolean isNoExecution() {
- return noExecution;
- }
-
- /**
- * @param noExecution the noExecution to set
- */
- public void setNoExecution(boolean noExecution) {
- this.noExecution = noExecution;
- }
-
- /**
- * @return the allowEmptyStreamOnExecution
- */
- public boolean isAllowEmptyStreamOnExecution() {
- return allowEmptyStreamOnExecution;
- }
-
- /**
- * @param allowEmptyStreamOnExecution the allowEmptyStreamOnExecution to set
- */
- public void setAllowEmptyStreamOnExecution(boolean allowEmptyStreamOnExecution) {
- this.allowEmptyStreamOnExecution = allowEmptyStreamOnExecution;
- }
-
- public String getRunConfigurationName() {
- return runConfigurationName;
- }
-
- public void setRunConfigurationName(String runConfigurationName) {
- this.runConfigurationName = runConfigurationName;
- }
-
- public boolean isCreateParentFolder() {
- return createParentFolder;
- }
-
- public void setCreateParentFolder(boolean createParentFolder) {
- this.createParentFolder = createParentFolder;
- }
-
/**
* @return The objects referenced in the transform, like a mapping, a pipeline, a workflow, ...
*/
@@ -521,7 +301,7 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
}
private boolean isTransformationDefined() {
- return !Utils.isEmpty(fileName);
+ return !Utils.isEmpty(templateFileName);
}
@Override
@@ -543,7 +323,7 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
* @param metadataProvider metadataProvider
* @param variables the variable variables to use
* @return the referenced object once loaded
- * @throws HopException
+ * @throws HopException in case there was an error loading the template pipeline.
*/
@Override
public IHasFilename loadReferencedObject(
@@ -551,109 +331,6 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
return loadPipelineMeta(this, metadataProvider, variables);
}
- public String getStreamSourceTransformName() {
- return streamSourceTransformName;
- }
-
- public void setStreamSourceTransformName(String streamSourceTransformName) {
- this.streamSourceTransformName = streamSourceTransformName;
- }
-
- public TransformMeta getStreamSourceTransform() {
- return streamSourceTransform;
- }
-
- public void setStreamSourceTransform(TransformMeta streamSourceTransform) {
- this.streamSourceTransform = streamSourceTransform;
- }
-
- public String getStreamTargetTransformName() {
- return streamTargetTransformName;
- }
-
- public void setStreamTargetTransformName(String streamTargetTransformName) {
- this.streamTargetTransformName = streamTargetTransformName;
- }
-
- @Override
- public void searchInfoAndTargetTransforms(List<TransformMeta> transforms) {
- streamSourceTransform = TransformMeta.findTransform(transforms, streamSourceTransformName);
- }
-
- public List<MetaInjectOutputField> getSourceOutputFields() {
- return sourceOutputFields;
- }
-
- public void setSourceOutputFields(List<MetaInjectOutputField> sourceOutputFields) {
- this.sourceOutputFields = sourceOutputFields;
- }
-
- public List<MetaInjectMapping> getMetaInjectMapping() {
- return metaInjectMapping;
- }
-
- public void setMetaInjectMapping(List<MetaInjectMapping> metaInjectMapping) {
- this.metaInjectMapping = metaInjectMapping;
- }
-
- public void actualizeMetaInjectMapping() {
- if (Utils.isEmpty(metaInjectMapping)) {
- return;
- }
- Map<TargetTransformAttribute, SourceTransformField> targetToSourceMap =
- convertToMap(metaInjectMapping);
- setTargetSourceMapping(targetToSourceMap);
- }
-
- /** package-local visibility for testing purposes */
- static Map<TargetTransformAttribute, SourceTransformField> convertToMap(
- List<MetaInjectMapping> metaInjectMapping) {
- Map<TargetTransformAttribute, SourceTransformField> targetToSourceMap = new HashMap<>();
- for (MetaInjectMapping mappingEntry : metaInjectMapping) {
- if (!isMappingEntryFilled(mappingEntry)) {
- continue;
- }
- TargetTransformAttribute targetTransformAttribute =
- createTargetTransformAttribute(mappingEntry);
- SourceTransformField sourceTransformField = createSourceTransformField(mappingEntry);
- targetToSourceMap.put(targetTransformAttribute, sourceTransformField);
- }
- return targetToSourceMap;
- }
-
- private static TargetTransformAttribute createTargetTransformAttribute(
- MetaInjectMapping mappingEntry) {
- String targetFieldName = mappingEntry.getTargetField();
- if (targetFieldName.contains(GROUP_AND_NAME_DELIMITER)) {
- String[] targetFieldGroupAndName = targetFieldName.split("\\" + GROUP_AND_NAME_DELIMITER);
- return new TargetTransformAttribute(
- mappingEntry.getTargetTransform(), targetFieldGroupAndName[1], true);
- }
- return new TargetTransformAttribute(
- mappingEntry.getTargetTransform(), mappingEntry.getTargetField(), false);
- }
-
- private static boolean isMappingEntryFilled(MetaInjectMapping mappingEntry) {
- if (mappingEntry.getSourceTransform() == null
- || mappingEntry.getSourceField() == null
- || mappingEntry.getTargetTransform() == null
- || mappingEntry.getTargetField() == null) {
- return false;
- }
- return true;
- }
-
- private static SourceTransformField createSourceTransformField(MetaInjectMapping mappingEntry) {
- return new SourceTransformField(
- mappingEntry.getSourceTransform(), mappingEntry.getSourceField());
- }
-
- @Override
- public void setDefault() {
- super.setDefault();
- createParentFolder = true;
- }
-
@Override
public void onTransformChange(
PipelineMeta pipelineMeta, TransformMeta oldMeta, TransformMeta newMeta) {
@@ -663,14 +340,11 @@ public class MetaInjectMeta extends BaseTransformMeta<MetaInject, MetaInjectData
TransformMeta toTransformMeta = hopMeta.getToTransform();
if ((toTransformMeta.getTransform() instanceof MetaInjectMeta toMeta)
&& (toTransformMeta.equals(this.getParentTransformMeta()))) {
- Map<TargetTransformAttribute, SourceTransformField> sourceMapping =
- toMeta.getTargetSourceMapping();
- for (Entry<TargetTransformAttribute, SourceTransformField> entry :
- sourceMapping.entrySet()) {
- SourceTransformField value = entry.getValue();
- if (value.getTransformName() != null
- && value.getTransformName().equals(oldMeta.getName())) {
- value.setTransformName(newMeta.getName());
+ List<MetaInjectMapping> sourceMappings = toMeta.getMappings();
+ for (MetaInjectMapping sourceMapping : sourceMappings) {
+ if (sourceMapping.getSourceTransformName() != null
+ && sourceMapping.getSourceTransformName().equals(oldMeta.getName())) {
+ sourceMapping.setSourceTransformName(newMeta.getName());
}
}
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigration.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigration.java
deleted file mode 100644
index 0b92e1dd96..0000000000
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMigration.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.metainject;
-
-import java.util.ArrayList;
-import java.util.Map;
-
-/** This class converts old mapping info into new one. */
-public class MetaInjectMigration {
- /** Migrate mapping from 7.0 version. */
- public static void migrateFrom70(
- Map<TargetTransformAttribute, SourceTransformField> targetSourceMapping) {
- /*
- * Need to convert GetTableNamesMeta.SCHENAMENAMEFIELD to the GetTableNamesMeta.SCHEMANAMEFIELD
- */
- for (TargetTransformAttribute target : new ArrayList<>(targetSourceMapping.keySet())) {
- if ("SCHENAMENAMEFIELD".equals(target.getAttributeKey())) {
- SourceTransformField so = targetSourceMapping.remove(target);
- TargetTransformAttribute target2 =
- new TargetTransformAttribute(
- target.getTransformName(), "SCHEMANAMEFIELD", target.isDetail());
- targetSourceMapping.put(target2, so);
- }
- }
- }
-}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectOutputField.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectOutputField.java
index d69d15640a..d0ac9e84ae 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectOutputField.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectOutputField.java
@@ -17,33 +17,52 @@
package org.apache.hop.pipeline.transforms.metainject;
+import lombok.Getter;
+import lombok.Setter;
import org.apache.hop.core.exception.HopPluginException;
-import org.apache.hop.core.exception.HopValueException;
-import org.apache.hop.core.injection.Injection;
-import org.apache.hop.core.injection.InjectionTypeConverter;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+@Getter
+@Setter
public class MetaInjectOutputField {
-
- @Injection(name = "SOURCE_OUTPUT_NAME", group = "SOURCE_OUTPUT_FIELDS")
+ @HopMetadataProperty(
+ key = "source_output_field_name",
+ injectionKey = "SOURCE_OUTPUT_NAME",
+ injectionKeyDescription = "MetaInject.Injection.SOURCE_OUTPUT_NAME")
private String name;
- @Injection(
- name = "SOURCE_OUTPUT_TYPE",
- group = "SOURCE_OUTPUT_FIELDS",
- converter = DataTypeConverter.class)
+ @HopMetadataProperty(
+ key = "source_output_field_type",
+ intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class,
+ injectionKey = "SOURCE_OUTPUT_TYPE",
+ injectionKeyDescription = "MetaInject.Injection.SOURCE_OUTPUT_TYPE")
private int type;
- @Injection(name = "SOURCE_OUTPUT_LENGTH", group = "SOURCE_OUTPUT_FIELDS")
+ @HopMetadataProperty(
+ key = "source_output_field_length",
+ injectionKey = "SOURCE_OUTPUT_LENGTH",
+ injectionKeyDescription = "MetaInject.Injection.SOURCE_OUTPUT_LENGTH")
private int length;
- @Injection(name = "SOURCE_OUTPUT_PRECISION", group = "SOURCE_OUTPUT_FIELDS")
+ @HopMetadataProperty(
+ key = "source_output_field_precision",
+ injectionKey = "SOURCE_OUTPUT_PRECISION",
+ injectionKeyDescription = "MetaInject.Injection.SOURCE_OUTPUT_PRECISION")
private int precision;
public MetaInjectOutputField() {}
+ public MetaInjectOutputField(MetaInjectOutputField f) {
+ super();
+ this.name = f.name;
+ this.type = f.type;
+ this.length = f.length;
+ this.precision = f.precision;
+ }
+
public MetaInjectOutputField(String name, int type, int length, int precision) {
super();
this.name = name;
@@ -56,46 +75,7 @@ public class MetaInjectOutputField {
return ValueMetaFactory.getValueMetaName(type);
}
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public int getType() {
- return type;
- }
-
- public void setType(int type) {
- this.type = type;
- }
-
- public int getLength() {
- return length;
- }
-
- public void setLength(int length) {
- this.length = length;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public void setPrecision(int precision) {
- this.precision = precision;
- }
-
public IValueMeta createValueMeta() throws HopPluginException {
return ValueMetaFactory.createValueMeta(name, type, length, precision);
}
-
- public static class DataTypeConverter extends InjectionTypeConverter {
- @Override
- public int string2intPrimitive(String v) throws HopValueException {
- return ValueMetaBase.getType(v);
- }
- }
}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/SourceTransformField.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/SourceTransformField.java
deleted file mode 100644
index 37fb35bc32..0000000000
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/SourceTransformField.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.metainject;
-
-import java.util.Objects;
-
-public class SourceTransformField {
-
- private String transformName;
- private String field;
-
- /**
- * @param transformName
- * @param field
- */
- public SourceTransformField(String transformName, String field) {
- this.transformName = transformName;
- this.field = field;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof SourceTransformField)) {
- return false;
- }
- if (obj == this) {
- return true;
- }
-
- SourceTransformField source = (SourceTransformField) obj;
- return transformName.equalsIgnoreCase(source.getTransformName())
- && field.equals(source.getField());
- }
-
- @Override
- public int hashCode() {
- // transformName can be null if using has constant value
- return Objects.hash(transformName, field);
- }
-
- /**
- * @return the transform name
- */
- public String getTransformName() {
- return transformName;
- }
-
- /**
- * @param transformName the transform name to set
- */
- public void setTransformName(String transformName) {
- this.transformName = transformName;
- }
-
- /**
- * @return the field
- */
- public String getField() {
- return field;
- }
-
- /**
- * @param field the field to set
- */
- public void setField(String field) {
- this.field = field;
- }
-}
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/TargetTransformAttribute.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/TargetTransformAttribute.java
deleted file mode 100644
index 1461ae27b6..0000000000
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/TargetTransformAttribute.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.metainject;
-
-public class TargetTransformAttribute {
-
- private String transformName;
- private String attributeKey;
- private boolean detail;
-
- /**
- * @param transformName
- * @param attributeKey
- */
- public TargetTransformAttribute(String transformName, String attributeKey, boolean detail) {
- this.transformName = transformName;
- this.attributeKey = attributeKey;
- this.detail = detail;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof TargetTransformAttribute)) {
- return false;
- }
- if (obj == this) {
- return true;
- }
-
- TargetTransformAttribute target = (TargetTransformAttribute) obj;
- return transformName.equalsIgnoreCase(target.getTransformName())
- && attributeKey.equals(target.getAttributeKey());
- }
-
- @Override
- public int hashCode() {
- return transformName.hashCode() ^ attributeKey.hashCode();
- }
-
- /**
- * @return the transform name
- */
- public String getTransformName() {
- return transformName;
- }
-
- /**
- * @param transformName the transform name to set
- */
- public void setTransformName(String transformName) {
- this.transformName = transformName;
- }
-
- /**
- * @return the attributeKey
- */
- public String getAttributeKey() {
- return attributeKey;
- }
-
- /**
- * @param attributeKey the attributeKey to set
- */
- public void setAttributeKey(String attributeKey) {
- this.attributeKey = attributeKey;
- }
-
- public void setDetail(boolean detail) {
- this.detail = detail;
- }
-
- public boolean isDetail() {
- return detail;
- }
-}
diff --git a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
index 016580c57c..ba10deeca9 100644
--- a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
+++ b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
@@ -15,8 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-
MetaInject.BadEncoding.Message=Bad encoding\!
MetaInject.Description=This transform allows you to inject metadata into an existing pipeline prior to execution. This allows for the creation of dynamic and highly flexible data integration solutions.
MetaInject.Injection.DIRECTORY_PATH=The path to the pipeline to inject values into, if located in a repository.
@@ -24,10 +22,11 @@ MetaInject.Injection.FILE_NAME=The path to the pipeline to inject values into, i
MetaInject.Injection.MAPPING_FIELDS=The list of mapping records.
MetaInject.Injection.MAPPING_SOURCE_FIELD=The name of the source field.
MetaInject.Injection.MAPPING_SOURCE_TRANSFORM=The name of the source transform.
-MetaInject.Injection.MAPPING_TARGET_FIELD=The name of the target field from the TARGET_TRANSFORM_NAME.
+MetaInject.Injection.MAPPING_TARGET_FIELD=The target (attribute) field.
+MetaInject.Injection.MAPPING_TARGET_DETAIL=The detail flag.
MetaInject.Injection.MAPPING_TARGET_TRANSFORM=The name of the target transform.
MetaInject.Injection.NO_EXECUTION=This option will skip the execution phase of the pipeline template.
-MetaInject.Injection.SOURCE_OUTPUT_FIELDS=The list of field attributes from transforms defined by the SOURCE_TRANSFORM_NAME.
+MetaInject.Injection.SOURCE_OUTPUT_FIELDS=The list of field attributes from transforms defined by the source transform name.
MetaInject.Injection.SOURCE_OUTPUT_LENGTH=Specify to force output strings to be padding to the specified length.
MetaInject.Injection.SOURCE_OUTPUT_NAME=The name of the output field from the transform to read from.
MetaInject.Injection.SOURCE_OUTPUT_PRECISION=The precision used for decimals on the source output transform.
@@ -36,9 +35,9 @@ MetaInject.Injection.SOURCE_TRANSFORM_NAME=The name of the transform which provi
MetaInject.Injection.STREAMING_SOURCE_TRANSFORM=The name of the transform to use for the source data stream.
MetaInject.Injection.STREAMING_TARGET_TRANSFORM=The name of the transform to use for the target stream.
MetaInject.Injection.TARGET_FILE=The name of file where Hop will save the pipeline template after injecting values.
-MetaInject.Injection.TRANS_NAME=The name of the pipeline to inject values into if located in the repository.
-MetaInject.Injection.TRANS_OBJECT_ID=The ID of the pipeline in the repository.
-MetaInject.Injection.TRANS_SEPECIFICATION_METHOD=The different ways to link to a pipeline: From File, Repository or Reference.
+MetaInject.Injection.ALLOW_EMPTY_STREAM_ON_EXECUTION=Allow empty stream on execution?
+MetaInject.Injection.CREATE_PARENT_FOLDER=Create parent folder?
+MetaInject.Injection.RUN_CONFIGURATION=Run configuration
MetaInject.Log.ParentFolderCreated=Folder [{0}] was created.
MetaInject.Log.ParentFolderExist=Parent folder exists [{0}]!
MetaInject.Log.ParentFolderNotExist=We can not find parent folder [{0}]!
diff --git a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaInjectionTest.java b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaInjectionTest.java
deleted file mode 100644
index 1cfefb2f74..0000000000
--- a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaInjectionTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hop.pipeline.transforms.metainject;
-
-import org.apache.hop.core.injection.BaseMetadataInjectionTestJunit5;
-import org.apache.hop.core.row.value.ValueMetaBase;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-class MetaInjectMetaInjectionTest extends BaseMetadataInjectionTestJunit5<MetaInjectMeta> {
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();
-
- private static final String TEST_ID = "TEST_ID";
-
- @BeforeEach
- void setup() throws Exception {
- setup(new MetaInjectMeta());
- }
-
- @Test
- void test() throws Exception {
- check("FILE_NAME", () -> meta.getFileName());
- check("SOURCE_TRANSFORM_NAME", () -> meta.getSourceTransformName());
- check("TARGET_FILE", () -> meta.getTargetFile());
- check("NO_EXECUTION", () -> meta.isNoExecution());
- check("ALLOW_EMPTY_STREAM_ON_EXECUTION", () -> meta.isAllowEmptyStreamOnExecution());
- check("STREAMING_SOURCE_TRANSFORM", () -> meta.getStreamSourceTransformName());
- check("STREAMING_TARGET_TRANSFORM", () -> meta.getStreamTargetTransformName());
- check("SOURCE_OUTPUT_NAME", () -> meta.getSourceOutputFields().get(0).getName());
- String[] typeNames = ValueMetaBase.getAllTypes();
-
- checkStringToInt(
- "SOURCE_OUTPUT_TYPE",
- () -> meta.getSourceOutputFields().get(0).getType(),
- typeNames,
- getTypeCodes(typeNames));
- check("SOURCE_OUTPUT_LENGTH", () -> meta.getSourceOutputFields().get(0).getLength());
- check("SOURCE_OUTPUT_PRECISION", () -> meta.getSourceOutputFields().get(0).getPrecision());
- check(
- "MAPPING_SOURCE_TRANSFORM", () -> meta.getMetaInjectMapping().get(0).getSourceTransform());
- check("MAPPING_SOURCE_FIELD", () -> meta.getMetaInjectMapping().get(0).getSourceField());
- check(
- "MAPPING_TARGET_TRANSFORM", () -> meta.getMetaInjectMapping().get(0).getTargetTransform());
- check("MAPPING_TARGET_FIELD", () -> meta.getMetaInjectMapping().get(0).getTargetField());
- }
-}
diff --git a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaLoadSaveTest.java b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaLoadSaveTest.java
deleted file mode 100644
index 1bf0ff0c01..0000000000
--- a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaLoadSaveTest.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hop.pipeline.transforms.metainject;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.hop.core.exception.HopException;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester;
-import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator;
-import org.apache.hop.pipeline.transforms.loadsave.validator.ListLoadSaveValidator;
-import org.apache.hop.pipeline.transforms.loadsave.validator.MapLoadSaveValidator;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-class MetaInjectMetaLoadSaveTest {
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();
-
- LoadSaveTester loadSaveTester;
- Class<MetaInjectMeta> testMetaClass = MetaInjectMeta.class;
-
- @BeforeEach
- void setUpLoadSave() throws Exception {
- List<String> attributes =
- Arrays.asList(
- "fileName",
- "sourceTransformName",
- "targetFile",
- "noExecution",
- "allowEmptyStreamOnExecution",
- "streamSourceTransformName",
- "streamTargetTransformName",
- "sourceOutputFields");
-
- Map<String, String> getterMap = new HashMap<>();
- Map<String, String> setterMap = new HashMap<>();
-
- Map<String, IFieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
- attrValidatorMap.put(
- "sourceOutputFields",
- new ListLoadSaveValidator<>(new MetaInjectOutputFieldLoadSaveValidator(), 5));
- //
- // Note - these seem to be runtime-built and not persisted.
- attrValidatorMap.put(
- "metaInjectMapping",
- new ListLoadSaveValidator<>(new MetaInjectMappingLoadSaveValidator(), 5));
- attrValidatorMap.put(
- "targetSourceMapping",
- new MapLoadSaveValidator<>(
- new TargetTransformAttributeLoadSaveValidator(),
- new SourceTransformFieldLoadSaveValidator(),
- 5));
-
- Map<String, IFieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();
-
- loadSaveTester =
- new LoadSaveTester(
- testMetaClass, attributes, getterMap, setterMap, attrValidatorMap, typeValidatorMap);
- }
-
- @Test
- void testSerialization() throws HopException {
- loadSaveTester.testSerialization();
- }
-
- public class MetaInjectOutputFieldLoadSaveValidator
- implements IFieldLoadSaveValidator<MetaInjectOutputField> {
- final Random rand = new Random();
-
- @Override
- public MetaInjectOutputField getTestObject() {
- MetaInjectOutputField rtn = new MetaInjectOutputField();
- rtn.setName(UUID.randomUUID().toString());
- rtn.setLength(rand.nextInt(100));
- rtn.setPrecision(rand.nextInt(9));
- rtn.setType(rand.nextInt(7));
- return rtn;
- }
-
- @Override
- public boolean validateTestObject(MetaInjectOutputField testObject, Object actual) {
- if (!(actual instanceof MetaInjectOutputField)) {
- return false;
- }
- MetaInjectOutputField another = (MetaInjectOutputField) actual;
- return new EqualsBuilder()
- .append(testObject.getLength(), another.getLength())
- .append(testObject.getPrecision(), another.getPrecision())
- .append(testObject.getName(), another.getName())
- .append(testObject.getType(), another.getType())
- .isEquals();
- }
- }
-
- // MetaInjectMappingLoadSaveValidator
- public class MetaInjectMappingLoadSaveValidator
- implements IFieldLoadSaveValidator<MetaInjectMapping> {
- final Random rand = new Random();
-
- @Override
- public MetaInjectMapping getTestObject() {
- MetaInjectMapping rtn = new MetaInjectMapping();
- rtn.setSourceField(UUID.randomUUID().toString());
- rtn.setSourceTransform(UUID.randomUUID().toString());
- rtn.setTargetField(UUID.randomUUID().toString());
- rtn.setTargetTransform(UUID.randomUUID().toString());
- return rtn;
- }
-
- @Override
- public boolean validateTestObject(MetaInjectMapping testObject, Object actual) {
- if (!(actual instanceof MetaInjectMapping)) {
- return false;
- }
- MetaInjectMapping another = (MetaInjectMapping) actual;
- return new EqualsBuilder()
- .append(testObject.getSourceField(), another.getSourceField())
- .append(testObject.getSourceTransform(), another.getSourceTransform())
- .append(testObject.getTargetField(), another.getTargetField())
- .append(testObject.getTargetTransform(), another.getTargetTransform())
- .isEquals();
- }
- }
-
- // TargetTransformAttributeLoadSaveValidator
- public class TargetTransformAttributeLoadSaveValidator
- implements IFieldLoadSaveValidator<TargetTransformAttribute> {
- final Random rand = new Random();
-
- @Override
- public TargetTransformAttribute getTestObject() {
- return new TargetTransformAttribute(
- UUID.randomUUID().toString(), UUID.randomUUID().toString(), rand.nextBoolean());
- }
-
- @Override
- public boolean validateTestObject(TargetTransformAttribute testObject, Object actual) {
- if (!(actual instanceof TargetTransformAttribute)) {
- return false;
- }
- TargetTransformAttribute another = (TargetTransformAttribute) actual;
- return new EqualsBuilder()
- .append(testObject.getTransformName(), another.getTransformName())
- .append(testObject.getAttributeKey(), another.getAttributeKey())
- .append(testObject.isDetail(), another.isDetail())
- .isEquals();
- }
- }
-
- // SourceTransformFieldLoadSaveValidator
- public class SourceTransformFieldLoadSaveValidator
- implements IFieldLoadSaveValidator<SourceTransformField> {
- final Random rand = new Random();
-
- @Override
- public SourceTransformField getTestObject() {
- return new SourceTransformField(UUID.randomUUID().toString(), UUID.randomUUID().toString());
- }
-
- @Override
- public boolean validateTestObject(SourceTransformField testObject, Object actual) {
- if (!(actual instanceof SourceTransformField)) {
- return false;
- }
- SourceTransformField another = (SourceTransformField) actual;
- return new EqualsBuilder()
- .append(testObject.getTransformName(), another.getTransformName())
- .append(testObject.getField(), another.getField())
- .isEquals();
- }
- }
-}
diff --git a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaTest.java b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaTest.java
index 5a74e50fd7..d12b912427 100644
--- a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaTest.java
+++ b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMetaTest.java
@@ -18,130 +18,108 @@
package org.apache.hop.pipeline.transforms.metainject;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.util.Collections;
-import java.util.List;
import java.util.Map;
-import org.apache.hop.core.HopClientEnvironment;
-import org.apache.hop.core.exception.HopException;
-import org.apache.hop.core.variables.IVariables;
-import org.apache.hop.core.variables.Variables;
-import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension;
-import org.apache.hop.metadata.api.IHopMetadataProvider;
-import org.apache.hop.pipeline.PipelineMeta;
-import org.apache.hop.pipeline.transform.TransformMeta;
-import org.apache.hop.resource.IResourceNaming;
-import org.apache.hop.resource.ResourceDefinition;
-import org.apache.hop.resource.ResourceReference;
-import org.junit.jupiter.api.BeforeAll;
+import java.util.Set;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaDate;
+import org.apache.hop.core.row.value.ValueMetaInteger;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.row.value.ValueMetaNumber;
+import org.apache.hop.core.row.value.ValueMetaPlugin;
+import org.apache.hop.core.row.value.ValueMetaPluginType;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.metadata.inject.HopMetadataInjector;
+import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
class MetaInjectMetaTest {
- private static final String SOURCE_TRANSFORM_NAME = "SOURCE_TRANSFORM_NAME";
-
- private static final String SOURCE_FIELD_NAME = "SOURCE_TRANSFORM_NAME";
-
- private static final String TARGET_TRANSFORM_NAME = "TARGET_TRANSFORM_NAME";
-
- private static final String TARGET_FIELD_NAME = "TARGET_TRANSFORM_NAME";
-
- private static final String TEST_FILE_NAME = "TEST_FILE_NAME";
-
- private static final String EXPORTED_FILE_NAME = TEST_FILE_NAME;
-
- private static MetaInjectMeta metaInjectMeta;
-
- @RegisterExtension
- static RestoreHopEngineEnvironmentExtension env = new RestoreHopEngineEnvironmentExtension();
-
- @BeforeAll
- static void SetUp() throws Exception {
- if (!HopClientEnvironment.isInitialized()) {
- HopClientEnvironment.init();
- }
- metaInjectMeta = new MetaInjectMeta();
- }
-
@BeforeEach
- void before() {
- metaInjectMeta = new MetaInjectMeta();
- }
-
- @Test
- void getResourceDependencies() {
- PipelineMeta pipelineMeta = mock(PipelineMeta.class);
- TransformMeta transformMeta = mock(TransformMeta.class);
-
- List<ResourceReference> actualResult =
- metaInjectMeta.getResourceDependencies(new Variables(), transformMeta);
- assertEquals(1, actualResult.size());
- ResourceReference reference = actualResult.iterator().next();
- assertEquals(0, reference.getEntries().size());
+ void beforeEach() throws Exception {
+ PluginRegistry registry = PluginRegistry.getInstance();
+ String[] classNames = {
+ ValueMetaString.class.getName(),
+ ValueMetaInteger.class.getName(),
+ ValueMetaDate.class.getName(),
+ ValueMetaNumber.class.getName(),
+ ValueMetaJson.class.getName()
+ };
+ for (String className : classNames) {
+ registry.registerPluginClass(className, ValueMetaPluginType.class, ValueMetaPlugin.class);
+ }
}
@Test
- void getResourceDependencies_with_defined_fileName() {
- PipelineMeta pipelineMeta = mock(PipelineMeta.class);
- TransformMeta transformMeta = mock(TransformMeta.class);
- metaInjectMeta.setFileName("FILE_NAME");
- // doReturn("FILE_NAME_WITH_SUBSTITUTIONS").when(pipelineMeta).environmentSubstitute("FILE_NAME");
-
- List<ResourceReference> actualResult =
- metaInjectMeta.getResourceDependencies(new Variables(), transformMeta);
- assertEquals(1, actualResult.size());
- ResourceReference reference = actualResult.iterator().next();
- assertEquals(1, reference.getEntries().size());
+ void testRoundTrip() throws Exception {
+ MetaInjectMeta meta =
+ TransformSerializationTestUtil.testSerialization("/meta-inject.xml", MetaInjectMeta.class);
+ assertNotNull(meta);
+
+ assertEquals("${PROJECT_HOME}/0038-json-input-template.hpl", meta.getTemplateFileName());
+ assertEquals("local", meta.getRunConfigurationName());
+ assertEquals("${java.io.tmpdir}/json-input.hpl", meta.getTargetFile());
+ assertTrue(meta.isCreateParentFolder());
+ assertTrue(meta.isNoExecution());
+ assertFalse(meta.isAllowEmptyStreamOnExecution());
+ assertEquals("sourceTransform", meta.getSourceTransformName());
+ assertEquals("targetTransform", meta.getStreamTargetTransformName());
+
+ // source output Fields
+ assertEquals(2, meta.getSourceOutputFields().size());
+ MetaInjectOutputField f = meta.getSourceOutputFields().getFirst();
+ assertEquals("f1", f.getName());
+ assertEquals(IValueMeta.TYPE_STRING, f.getType());
+ assertEquals(100, f.getLength());
+ assertEquals(-1, f.getPrecision());
+ f = meta.getSourceOutputFields().getLast();
+ assertEquals("f2", f.getName());
+ assertEquals(IValueMeta.TYPE_NUMBER, f.getType());
+ assertEquals(7, f.getLength());
+ assertEquals(2, f.getPrecision());
+
+ // Mappings
+ assertEquals(38, meta.getMappings().size());
+ // Let's test the first and the last only
+ //
+ MetaInjectMapping m = meta.getMappings().getFirst();
+ assertEquals("JSON input", m.getTargetTransformName());
+ assertEquals("IGNORE_EMPTY_FILE", m.getTargetAttributeKey());
+ assertFalse(m.isTargetDetail());
+ assertEquals("files/json-input.xml", m.getSourceTransformName());
+ assertEquals("IsIgnoreEmptyFile", m.getSourceField());
+
+ m = meta.getMappings().getLast();
+ assertEquals("JSON input", m.getTargetTransformName());
+ assertEquals("FILE_REQUIRED", m.getTargetAttributeKey());
+ assertTrue(m.isTargetDetail());
+ assertEquals("files/json-input.xml files", m.getSourceTransformName());
+ assertEquals("file_required", m.getSourceField());
}
@Test
- void exportResources() throws HopException {
- IVariables variables = mock(IVariables.class);
- IResourceNaming resourceNamingInterface = mock(IResourceNaming.class);
- IHopMetadataProvider metadataProvider = mock(IHopMetadataProvider.class);
-
- MetaInjectMeta injectMetaSpy = spy(metaInjectMeta);
- PipelineMeta pipelineMeta = mock(PipelineMeta.class);
- Map<String, ResourceDefinition> definitions = Collections.emptyMap();
- doReturn(TEST_FILE_NAME)
- .when(pipelineMeta)
- .exportResources(variables, definitions, resourceNamingInterface, metadataProvider);
- doReturn(pipelineMeta).when(injectMetaSpy).loadPipelineMeta(metadataProvider, variables);
-
- String actualExportedFileName =
- injectMetaSpy.exportResources(
- variables, definitions, resourceNamingInterface, metadataProvider);
- assertEquals(TEST_FILE_NAME, actualExportedFileName);
- assertEquals(EXPORTED_FILE_NAME, injectMetaSpy.getFileName());
- verify(pipelineMeta)
- .exportResources(variables, definitions, resourceNamingInterface, metadataProvider);
- }
-
- @Test
- void convertToMap() {
- MetaInjectMapping metaInjectMapping = new MetaInjectMapping();
- metaInjectMapping.setSourceTransform(SOURCE_TRANSFORM_NAME);
- metaInjectMapping.setSourceField(SOURCE_FIELD_NAME);
- metaInjectMapping.setTargetTransform(TARGET_TRANSFORM_NAME);
- metaInjectMapping.setTargetField(TARGET_FIELD_NAME);
-
- Map<TargetTransformAttribute, SourceTransformField> actualResult =
- MetaInjectMeta.convertToMap(Collections.singletonList(metaInjectMapping));
-
- assertEquals(1, actualResult.size());
-
- TargetTransformAttribute targetTransformAttribute = actualResult.keySet().iterator().next();
- assertEquals(TARGET_TRANSFORM_NAME, targetTransformAttribute.getTransformName());
- assertEquals(TARGET_FIELD_NAME, targetTransformAttribute.getAttributeKey());
-
- SourceTransformField sourceTransformField = actualResult.values().iterator().next();
- assertEquals(SOURCE_TRANSFORM_NAME, sourceTransformField.getTransformName());
- assertEquals(SOURCE_FIELD_NAME, sourceTransformField.getField());
+ void testSampleMetaMapping() throws Exception {
+ Map<String, Set<String>> map = HopMetadataInjector.findInjectionGroupKeys(MetaInjectMeta.class);
+ assertNotNull(map);
+ assertEquals(2, map.size());
+ Set<String> fieldKeys = map.get("SOURCE_OUTPUT_FIELDS");
+ assertEquals(4, fieldKeys.size());
+ Set<String> mappingKeys = map.get("MAPPING_FIELDS");
+ assertEquals(5, mappingKeys.size());
+
+ assertTrue(fieldKeys.contains("SOURCE_OUTPUT_NAME"));
+ assertTrue(fieldKeys.contains("SOURCE_OUTPUT_TYPE"));
+ assertTrue(fieldKeys.contains("SOURCE_OUTPUT_LENGTH"));
+ assertTrue(fieldKeys.contains("SOURCE_OUTPUT_PRECISION"));
+ assertTrue(mappingKeys.contains("MAPPING_SOURCE_TRANSFORM"));
+ assertTrue(mappingKeys.contains("MAPPING_SOURCE_FIELD"));
+ assertTrue(mappingKeys.contains("MAPPING_TARGET_TRANSFORM"));
+ assertTrue(mappingKeys.contains("MAPPING_TARGET_FIELD"));
+ assertTrue(mappingKeys.contains("MAPPING_TARGET_DETAIL"));
}
}
diff --git a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectTest.java b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectTest.java
index 262bebcc4d..d49f94ca2c 100644
--- a/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectTest.java
+++ b/plugins/transforms/metainject/src/test/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectTest.java
@@ -21,15 +21,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import org.apache.hop.core.injection.Injection;
import org.apache.hop.core.injection.InjectionSupported;
@@ -65,12 +62,6 @@ class MetaInjectTest {
private static final String UNAVAILABLE_TRANSFORM = "UNAVAILABLE_TRANSFORM";
- private static final TargetTransformAttribute UNAVAILABLE_TARGET_TRANSFORM =
- new TargetTransformAttribute(UNAVAILABLE_TRANSFORM, TEST_ATTR_VALUE, false);
-
- private static final SourceTransformField UNAVAILABLE_SOURCE_TRANSFORM =
- new SourceTransformField(UNAVAILABLE_TRANSFORM, TEST_FIELD);
-
private MetaInject metaInject;
private MetaInjectMeta meta;
@@ -113,91 +104,6 @@ class MetaInjectTest {
doReturn(internalPipelineMeta).when(metaInject).loadPipelineMeta();
}
- @Test
- void getUnavailableSourceTransforms() {
- TargetTransformAttribute targetTransform =
- new TargetTransformAttribute(TEST_TARGET_TRANSFORM_NAME, TEST_ATTR_VALUE, false);
- SourceTransformField unavailableSourceTransform =
- new SourceTransformField(UNAVAILABLE_TRANSFORM, TEST_FIELD);
- Map<TargetTransformAttribute, SourceTransformField> targetMap =
- Collections.singletonMap(targetTransform, unavailableSourceTransform);
- PipelineMeta sourcePipelineMeta = mock(PipelineMeta.class);
- doReturn(new String[0])
- .when(sourcePipelineMeta)
- .getPrevTransformNames(any(TransformMeta.class));
-
- Set<SourceTransformField> actualSet =
- MetaInject.getUnavailableSourceTransforms(
- targetMap, sourcePipelineMeta, mock(TransformMeta.class));
- assertTrue(actualSet.contains(unavailableSourceTransform));
- }
-
- @Test
- void getUnavailableTargetTransforms() {
- TargetTransformAttribute unavailableTargetTransform =
- new TargetTransformAttribute(UNAVAILABLE_TRANSFORM, TEST_ATTR_VALUE, false);
- SourceTransformField sourceTransform =
- new SourceTransformField(TEST_SOURCE_TRANSFORM_NAME, TEST_FIELD);
- Map<TargetTransformAttribute, SourceTransformField> targetMap =
- Collections.singletonMap(unavailableTargetTransform, sourceTransform);
- PipelineMeta injectedPipelineMeta = mock(PipelineMeta.class);
- doReturn(Collections.emptyList()).when(injectedPipelineMeta).getUsedTransforms();
-
- Set<TargetTransformAttribute> actualSet =
- MetaInject.getUnavailableTargetTransforms(targetMap, injectedPipelineMeta);
- assertTrue(actualSet.contains(unavailableTargetTransform));
- }
-
- @Test
- void removeUnavailableTransformsFromMapping_unavailable_source_transform() {
- TargetTransformAttribute unavailableTargetTransform =
- new TargetTransformAttribute(UNAVAILABLE_TRANSFORM, TEST_ATTR_VALUE, false);
- SourceTransformField unavailableSourceTransform =
- new SourceTransformField(UNAVAILABLE_TRANSFORM, TEST_FIELD);
- Map<TargetTransformAttribute, SourceTransformField> targetMap = new HashMap<>();
- targetMap.put(unavailableTargetTransform, unavailableSourceTransform);
-
- Set<SourceTransformField> unavailableSourceTransforms =
- Collections.singleton(UNAVAILABLE_SOURCE_TRANSFORM);
- MetaInject.removeUnavailableTransformsFromMapping(
- targetMap, unavailableSourceTransforms, Collections.emptySet());
- assertTrue(targetMap.isEmpty());
- }
-
- @Test
- void removeUnavailableTransformsFromMapping_unavailable_target_transform() {
- TargetTransformAttribute unavailableTargetTransform =
- new TargetTransformAttribute(UNAVAILABLE_TRANSFORM, TEST_ATTR_VALUE, false);
- SourceTransformField unavailableSourceTransform =
- new SourceTransformField(UNAVAILABLE_TRANSFORM, TEST_FIELD);
- Map<TargetTransformAttribute, SourceTransformField> targetMap = new HashMap<>();
- targetMap.put(unavailableTargetTransform, unavailableSourceTransform);
-
- Set<TargetTransformAttribute> unavailableTargetTransforms =
- Collections.singleton(UNAVAILABLE_TARGET_TRANSFORM);
- MetaInject.removeUnavailableTransformsFromMapping(
- targetMap, Collections.emptySet(), unavailableTargetTransforms);
- assertTrue(targetMap.isEmpty());
- }
-
- @Test
- void removeUnavailableTransformsFromMapping_unavailable_source_target_transform() {
- TargetTransformAttribute unavailableTargetTransform =
- new TargetTransformAttribute(UNAVAILABLE_TRANSFORM, TEST_ATTR_VALUE, false);
- SourceTransformField unavailableSourceTransform =
- new SourceTransformField(UNAVAILABLE_TRANSFORM, TEST_FIELD);
- Map<TargetTransformAttribute, SourceTransformField> targetMap = new HashMap<>();
- targetMap.put(unavailableTargetTransform, unavailableSourceTransform);
-
- Set<TargetTransformAttribute> unavailableTargetTransforms =
- Collections.singleton(UNAVAILABLE_TARGET_TRANSFORM);
- Set<SourceTransformField> unavailableSourceTransforms =
- Collections.singleton(UNAVAILABLE_SOURCE_TRANSFORM);
- MetaInject.removeUnavailableTransformsFromMapping(
- targetMap, unavailableSourceTransforms, unavailableTargetTransforms);
- assertTrue(targetMap.isEmpty());
- }
-
@Test
void convertToUpperCaseSet_null_array() {
Set<String> actualResult = MetaInject.convertToUpperCaseSet(null);
@@ -215,28 +121,6 @@ class MetaInjectTest {
assertEquals(expectedResult, actualResult);
}
- @Test
- void testGetUnavailableTargetKeys() throws Exception {
- final String targetTransformName = "injectable transform name";
- TargetTransformAttribute unavailableTargetAttr =
- new TargetTransformAttribute(targetTransformName, "NOT_THERE", false);
- TargetTransformAttribute availableTargetAttr =
- new TargetTransformAttribute(targetTransformName, "THERE", false);
- SourceTransformField sourceTransform =
- new SourceTransformField(TEST_SOURCE_TRANSFORM_NAME, TEST_FIELD);
-
- Map<TargetTransformAttribute, SourceTransformField> targetMap = new HashMap<>(2);
- targetMap.put(unavailableTargetAttr, sourceTransform);
- targetMap.put(availableTargetAttr, sourceTransform);
-
... 1692 lines suppressed ...