You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ba...@apache.org on 2026/10/16 12:35:22 UTC
(hop) branch main updated: performance improvements, fixes #8965 (#8968)
This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e15375cbf7 performance improvements, fixes #8965 (#8968)
e15375cbf7 is described below
commit e15375cbf7fe55a229992e2507fee0260e936fd9
Author: Hans Van Akelyen <ha...@gmail.com>
AuthorDate: Fri Oct 16 14:35:16 2026 +0200
performance improvements, fixes #8965 (#8968)
---
core/src/main/java/org/apache/hop/core/Const.java | 9 +-
.../main/java/org/apache/hop/core/row/RowMeta.java | 12 +-
.../org/apache/hop/core/row/ValueDataUtil.java | 32 ++++-
.../test/java/org/apache/hop/core/ConstTest.java | 4 +
.../org/apache/hop/core/row/ValueDataUtilTest.java | 25 ++++
.../transforms/concatfields/ConcatFields.java | 2 +-
.../transforms/concatfields/ConcatFieldsData.java | 6 +
.../concatfields/helper/ConcatFieldHelper.java | 129 ++++++++++++---------
.../transforms/fuzzymatch/BoundedLevenshtein.java | 101 ++++++++++++++++
.../pipeline/transforms/fuzzymatch/FuzzyMatch.java | 37 ++++--
.../transforms/fuzzymatch/FuzzyMatchData.java | 8 ++
.../fuzzymatch/BoundedLevenshteinTest.java | 86 ++++++++++++++
.../stringoperations/StringOperations.java | 14 ++-
.../stringoperations/StringOperationsData.java | 6 +
.../stringoperations/StringOperationsTest.java | 37 +++++-
15 files changed, 433 insertions(+), 75 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Const.java b/core/src/main/java/org/apache/hop/core/Const.java
index 284f8fe9d9..79ace80d08 100644
--- a/core/src/main/java/org/apache/hop/core/Const.java
+++ b/core/src/main/java/org/apache/hop/core/Const.java
@@ -1308,7 +1308,8 @@ public class Const {
* @return The converted value or the default.
*/
public static int toInt(String str, int def) {
- if (str == null) {
+ // Integer.parseInt throws for these anyway: skip the exception, callers do this per row
+ if (str == null || str.isBlank()) {
return def;
}
int retval;
@@ -1328,7 +1329,8 @@ public class Const {
* @return The converted value or the default.
*/
public static long toLong(String str, long def) {
- if (str == null) {
+ // Long.parseLong throws for these anyway: skip the exception, callers do this per row
+ if (str == null || str.isBlank()) {
return def;
}
long retval;
@@ -1579,7 +1581,8 @@ public class Const {
* @return The converted value or the default.
*/
public static double toDouble(String str, double def) {
- if (str == null) {
+ // Double.parseDouble throws for these anyway: skip the exception, callers do this per row
+ if (str == null || str.isBlank()) {
return def;
}
double retval;
diff --git a/core/src/main/java/org/apache/hop/core/row/RowMeta.java b/core/src/main/java/org/apache/hop/core/row/RowMeta.java
index a9a15108f8..00a6df3bcf 100644
--- a/core/src/main/java/org/apache/hop/core/row/RowMeta.java
+++ b/core/src/main/java/org/apache/hop/core/row/RowMeta.java
@@ -595,9 +595,19 @@ public class RowMeta implements IRowMeta {
*/
@Override
public Object[] cloneRow(Object[] objects, Object[] newObjects) throws HopValueException {
- List<Integer> list = getOrCreateValuesThatNeedRealClone(valueMetaList);
lock.readLock().lock();
try {
+ // The list is built once; only take the write lock when it still has to be built,
+ // not on every row that gets cloned.
+ List<Integer> list = needRealClone;
+ if (list == null) {
+ lock.readLock().unlock();
+ try {
+ list = getOrCreateValuesThatNeedRealClone(valueMetaList);
+ } finally {
+ lock.readLock().lock();
+ }
+ }
for (Integer i : list) {
IValueMeta valueMeta = valueMetaList.get(i);
newObjects[i] = valueMeta.cloneValueData(objects[i]);
diff --git a/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java b/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
index 23c1bc7df2..3aff35e742 100644
--- a/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
+++ b/core/src/main/java/org/apache/hop/core/row/ValueDataUtil.java
@@ -40,7 +40,6 @@ import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.commons.text.WordUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.hop.core.Const;
@@ -208,11 +207,40 @@ public class ValueDataUtil {
return (new RefinedSoundex()).encode(dataA.toString());
}
+ /**
+ * Lower-case the value, then title-case the first character of every word. Words are separated by
+ * space, underscore, comma, colon, semicolon and dash. Same result as commons-text {@code
+ * WordUtils.capitalizeFully(value, ' ', '_', ',', ':', ';', '-')}, in one pass: that one builds a
+ * set of boxed delimiters on every call, and this runs once per row.
+ */
public static String initCap(Object dataA) {
if (dataA == null) {
return null;
}
- return WordUtils.capitalizeFully(dataA.toString(), new char[] {' ', '_', ',', ':', ';', '-'});
+ String lower = dataA.toString().toLowerCase();
+ StringBuilder result = new StringBuilder(lower.length());
+ boolean capitalizeNext = true;
+ for (int i = 0; i < lower.length(); ) {
+ int codePoint = lower.codePointAt(i);
+ i += Character.charCount(codePoint);
+ if (isInitCapDelimiter(codePoint)) {
+ capitalizeNext = true;
+ result.appendCodePoint(codePoint);
+ } else if (capitalizeNext) {
+ capitalizeNext = false;
+ result.appendCodePoint(Character.toTitleCase(codePoint));
+ } else {
+ result.appendCodePoint(codePoint);
+ }
+ }
+ return result.toString();
+ }
+
+ private static boolean isInitCapDelimiter(int codePoint) {
+ return switch (codePoint) {
+ case ' ', '_', ',', ':', ';', '-' -> true;
+ default -> false;
+ };
}
public static String upperCase(Object dataA) {
diff --git a/core/src/test/java/org/apache/hop/core/ConstTest.java b/core/src/test/java/org/apache/hop/core/ConstTest.java
index 92b9278bcb..792375f244 100644
--- a/core/src/test/java/org/apache/hop/core/ConstTest.java
+++ b/core/src/test/java/org/apache/hop/core/ConstTest.java
@@ -3653,6 +3653,10 @@ class ConstTest {
void testToInt() {
assertEquals(123, Const.toInt("123", -12));
assertEquals(-12, Const.toInt("123f", -12));
+ assertEquals(-12, Const.toInt(null, -12));
+ assertEquals(-12, Const.toInt("", -12));
+ assertEquals(-12, Const.toInt(" ", -12));
+ assertEquals(-12, Const.toInt(" 5", -12));
}
@Test
diff --git a/core/src/test/java/org/apache/hop/core/row/ValueDataUtilTest.java b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilTest.java
index 6fde8f0685..923b223718 100644
--- a/core/src/test/java/org/apache/hop/core/row/ValueDataUtilTest.java
+++ b/core/src/test/java/org/apache/hop/core/row/ValueDataUtilTest.java
@@ -33,6 +33,7 @@ import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.WordUtils;
import org.apache.hop.core.exception.HopFileNotFoundException;
import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.row.value.ValueMetaBigNumber;
@@ -406,6 +407,30 @@ class ValueDataUtilTest {
assertNull(ValueDataUtil.initCap(null));
}
+ /** initCap gives the result WordUtils.capitalizeFully gave it before it got its own loop. */
+ @Test
+ void testInitCapMatchesCapitalizeFully() {
+ char[] delimiters = {' ', '_', ',', ':', ';', '-'};
+ String[] inputs = {
+ "",
+ " ",
+ "HELLO WORLD",
+ "hELLO wORLD ",
+ "first_name,last-name:x;y",
+ "--a__b,,c",
+ "tab\tseparated words",
+ "o'neil mcdonald",
+ "élan ÉCOLE ßtraße",
+ "džemal",
+ "𐐨𐐩 deseret",
+ "123abc 4DEF"
+ };
+ for (String input : inputs) {
+ assertEquals(
+ WordUtils.capitalizeFully(input, delimiters), ValueDataUtil.initCap(input), input);
+ }
+ }
+
@Test
void testUpperCase() {
assertEquals("HELLO", ValueDataUtil.upperCase("hello"));
diff --git a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFields.java b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFields.java
index 7ac956e954..af27aac049 100644
--- a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFields.java
+++ b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFields.java
@@ -44,7 +44,7 @@ public class ConcatFields extends BaseTransform<ConcatFieldsMeta, ConcatFieldsDa
}
@Override
- public synchronized boolean processRow() throws HopException {
+ public boolean processRow() throws HopException {
// This also waits for a row to be finished.
Object[] row = getRow();
if (row == null) {
diff --git a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFieldsData.java b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFieldsData.java
index be89feba10..d26f7339ec 100644
--- a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFieldsData.java
+++ b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/ConcatFieldsData.java
@@ -40,6 +40,12 @@ public class ConcatFieldsData extends BaseTransformData implements ITransformDat
public String[] trimType;
+ /** Per concatenated field: the value written for null, worked out on the first row */
+ public String[] fieldNullStrings;
+
+ /** Per concatenated field: the trim type id to apply, worked out on the first row */
+ public int[] fieldTrimTypes;
+
public int targetFieldLength; // for fast data dump (StringBuilder size)
public ConcatFieldsData() {
diff --git a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/helper/ConcatFieldHelper.java b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/helper/ConcatFieldHelper.java
index e7403ad7e9..716d51f781 100644
--- a/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/helper/ConcatFieldHelper.java
+++ b/plugins/transforms/concatfields/src/main/java/org/apache/hop/pipeline/transforms/concatfields/helper/ConcatFieldHelper.java
@@ -17,9 +17,6 @@
package org.apache.hop.pipeline.transforms.concatfields.helper;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
import lombok.experimental.UtilityClass;
import org.apache.hop.core.Const;
import org.apache.hop.core.exception.HopException;
@@ -66,79 +63,105 @@ public class ConcatFieldHelper {
Object[] inputRowData, ConcatFieldsData data, ConcatFieldsMeta meta, IRowMeta rowMeta)
throws HopException {
Object[] outputRowData = createOutputRow(inputRowData, data, meta);
- List<String> fields = new ArrayList<>(Math.max(data.targetFieldLength, 8));
+ if (data.fieldTrimTypes == null) {
+ prepareFields(data, meta);
+ }
+
+ // This runs for every row: build the target in one pass, no intermediate list or stream
+ String separator = Const.NVL(data.stringSeparator, "");
+ boolean skipEmpty = meta.isSkipValueEmpty();
+ boolean forceEnclosure = meta.isForceEnclosure();
+ String enclosure = meta.getEnclosure();
+ StringBuilder target = new StringBuilder(Math.max(data.targetFieldLength, 16));
+ boolean firstValue = true;
for (int i = 0; i < data.inputFieldIndexes.size(); i++) {
int inputRowIndex = data.inputFieldIndexes.get(i);
IValueMeta valueMeta = rowMeta.getValueMeta(inputRowIndex);
Object valueData = inputRowData[inputRowIndex];
- String nullString;
- String trimType;
- if (meta.getOutputFields().isEmpty()) {
- // No specific null value defined
- nullString = "";
- trimType = ValueMetaBase.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE);
+ String value;
+ if (valueMeta.isNull(valueData)) {
+ value = data.fieldNullStrings[i];
} else {
- ConcatField field = meta.getOutputFields().get(i);
- nullString = Const.NVL(field.getNullString(), "");
- trimType = data.trimType[i];
- // Manage missing trim type. Leave the incoming value as it is
- if (trimType == null) {
- trimType = ValueMetaBase.getTrimTypeCode(IValueMeta.TRIM_TYPE_NONE);
- }
+ value = fieldString(valueMeta, valueData, data.fieldTrimTypes[i]);
}
- handlerField(fields, valueMeta, valueData, nullString, trimType);
+ if (skipEmpty && Utils.isEmpty(value)) {
+ continue;
+ }
+ if (!firstValue) {
+ target.append(separator);
+ }
+ firstValue = false;
+ if (forceEnclosure) {
+ target.append(enclosure).append(value).append(enclosure);
+ } else {
+ target.append(value);
+ }
}
- // list -> string(support separator)
- String separator = Const.NVL(data.stringSeparator, "");
- String collect =
- fields.stream()
- .filter(c -> !(meta.isSkipValueEmpty() && Utils.isEmpty(c)))
- .map(c -> meta.isForceEnclosure() ? meta.getEnclosure() + c + meta.getEnclosure() : c)
- .collect(Collectors.joining(separator));
-
- outputRowData[data.outputRowMeta.size() - 1] = collect;
+ outputRowData[data.outputRowMeta.size() - 1] = target.toString();
return outputRowData;
}
/**
- * Converts a single field value to its string representation and adds it to the provided list of
- * field values.
+ * Work out the null string and trim type of every concatenated field once, rather than for every
+ * row. Without output fields every input field is used, with no null string and no trimming.
+ */
+ private static void prepareFields(ConcatFieldsData data, ConcatFieldsMeta meta) {
+ int nrFields = data.inputFieldIndexes.size();
+ String[] nullStrings = new String[nrFields];
+ int[] trimTypes = new int[nrFields];
+ for (int i = 0; i < nrFields; i++) {
+ if (meta.getOutputFields().isEmpty()) {
+ nullStrings[i] = "";
+ trimTypes[i] = IValueMeta.TRIM_TYPE_NONE;
+ } else {
+ ConcatField field = meta.getOutputFields().get(i);
+ nullStrings[i] = Const.NVL(field.getNullString(), "");
+ String trimType = data.trimType == null ? null : data.trimType[i];
+ // Manage missing trim type. Leave the incoming value as it is
+ trimTypes[i] =
+ trimType == null
+ ? IValueMeta.TRIM_TYPE_NONE
+ : ValueMetaBase.getTrimTypeByCode(trimType);
+ }
+ }
+ data.fieldNullStrings = nullStrings;
+ data.fieldTrimTypes = trimTypes;
+ }
+
+ /**
+ * Converts a non-null field value to its string representation with the trim type of this
+ * transform rather than the one of the incoming field.
*
- * @param fields the list collecting converted field values
* @param valueMeta the value metadata describing the field type
* @param valueData the actual field value
- * @param nullString the replacement value used when the field value is null
- * @param trimType the trim type code to apply
+ * @param trimType the trim type to apply
+ * @return the string value
* @throws HopValueException if the value cannot be converted to a string
*/
- private static void handlerField(
- List<String> fields,
- IValueMeta valueMeta,
- Object valueData,
- String nullString,
- String trimType)
+ private static String fieldString(IValueMeta valueMeta, Object valueData, int trimType)
throws HopValueException {
- if (valueMeta.isNull(valueData)) {
- fields.add(nullString);
- return;
- }
-
- if (trimType != null) {
- // Get the raw string value without applying the incoming field's trim type
- // by temporarily setting trim type to NONE
- int originalTrimType = valueMeta.getTrimType();
+ // Get the raw string value without applying the incoming field's trim type
+ // by temporarily setting trim type to NONE
+ int originalTrimType = valueMeta.getTrimType();
+ String stringValue;
+ if (originalTrimType == IValueMeta.TRIM_TYPE_NONE) {
+ stringValue = valueMeta.getString(valueData);
+ } else {
valueMeta.setTrimType(IValueMeta.TRIM_TYPE_NONE);
- String stringValue = valueMeta.getString(valueData);
- // Restore the original trim type
- valueMeta.setTrimType(originalTrimType);
-
- // Apply only the configured trim type from the transform
- fields.add(Const.trimToType(stringValue, ValueMetaBase.getTrimTypeByCode(trimType)));
+ try {
+ stringValue = valueMeta.getString(valueData);
+ } finally {
+ // Restore the original trim type
+ valueMeta.setTrimType(originalTrimType);
+ }
}
+
+ // Apply only the configured trim type from the transform
+ return Const.trimToType(stringValue, trimType);
}
/**
diff --git a/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshtein.java b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshtein.java
new file mode 100644
index 0000000000..b5865e7c9d
--- /dev/null
+++ b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshtein.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.fuzzymatch;
+
+/**
+ * Levenshtein distance that gives up as soon as the result can no longer be within a maximum. Fuzzy
+ * match drops every distance above its maximal value, so only a band of {@code 2 * max + 1} cells
+ * per row of the matrix is needed, and a row whose cells are all above the maximum ends the
+ * calculation. The two matrix rows are kept between calls. Not thread-safe: one per transform copy.
+ */
+public class BoundedLevenshtein {
+ private int[] previous = new int[0];
+ private int[] current = new int[0];
+
+ /**
+ * @param s the first string
+ * @param t the second string
+ * @param max the largest distance of interest
+ * @return the Levenshtein distance of s and t (counted in UTF-16 code units, like commons-lang3),
+ * or {@link Integer#MAX_VALUE} when it is larger than max
+ */
+ public int distance(CharSequence s, CharSequence t, int max) {
+ if (max < 0) {
+ return Integer.MAX_VALUE;
+ }
+ int n = s.length();
+ int m = t.length();
+ if (Math.abs(n - m) > max) {
+ return Integer.MAX_VALUE;
+ }
+ if (n == 0 || m == 0) {
+ // the other one is at most max long, checked above
+ return Math.max(n, m);
+ }
+
+ // The distance is never above the longer length: cap the band so nothing overflows
+ int limit = Math.min(max, Math.max(n, m));
+ int tooFar = limit + 1;
+
+ if (previous.length < m + 1) {
+ previous = new int[m + 1];
+ current = new int[m + 1];
+ }
+ int[] prev = previous;
+ int[] curr = current;
+
+ for (int j = 0; j <= m; j++) {
+ prev[j] = j;
+ }
+
+ for (int i = 1; i <= n; i++) {
+ char sc = s.charAt(i - 1);
+ int from = Math.max(1, i - limit);
+ int to = Math.min(m, i + limit);
+
+ curr[0] = i;
+ int rowMin = from == 1 ? i : tooFar;
+ if (from > 1) {
+ curr[from - 1] = tooFar;
+ }
+ for (int j = from; j <= to; j++) {
+ int cost = sc == t.charAt(j - 1) ? 0 : 1;
+ int value = Math.min(Math.min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost);
+ if (value > tooFar) {
+ value = tooFar;
+ }
+ curr[j] = value;
+ if (value < rowMin) {
+ rowMin = value;
+ }
+ }
+ if (to < m) {
+ curr[to + 1] = tooFar;
+ }
+ if (rowMin > limit) {
+ return Integer.MAX_VALUE;
+ }
+
+ int[] swap = prev;
+ prev = curr;
+ curr = swap;
+ }
+
+ return prev[m] <= limit ? prev[m] : Integer.MAX_VALUE;
+ }
+}
diff --git a/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatch.java b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatch.java
index dc20a23fbe..c539710d47 100644
--- a/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatch.java
+++ b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatch.java
@@ -240,16 +240,18 @@ public class FuzzyMatch extends BaseTransform<FuzzyMatchMeta, FuzzyMatchData> {
boolean lowerIsBetter = true;
PriorityQueue<ScoredMatch> topK = newTopKHeap(lowerIsBetter);
String lookupValueString = getInputRowMeta().getString(row, data.indexOfMainField);
+ if (data.lookKeys == null) {
+ prepareDistanceKeys();
+ }
- for (Object[] cachedData : data.look) {
- String cacheValue = (String) cachedData[0];
+ String useLookupvalue = lookupValueString;
+ if (!meta.isCaseSensitive()) {
+ useLookupvalue = lookupValueString.toLowerCase();
+ }
- String useCacheValue = cacheValue;
- String useLookupvalue = lookupValueString;
- if (!meta.isCaseSensitive()) {
- useCacheValue = cacheValue.toLowerCase();
- useLookupvalue = lookupValueString.toLowerCase();
- }
+ for (int k = 0; k < data.lookRows.length; k++) {
+ Object[] cachedData = data.lookRows[k];
+ String useCacheValue = data.lookKeys[k];
int cDistance =
switch (meta.getAlgorithm()) {
@@ -257,7 +259,9 @@ public class FuzzyMatch extends BaseTransform<FuzzyMatchMeta, FuzzyMatchData> {
Utils.getDamerauLevenshteinDistance(useCacheValue, useLookupvalue);
case NEEDLEMAN_WUNSH ->
Math.abs((int) new NeedlemanWunsch().score(useCacheValue, useLookupvalue));
- default -> StringUtils.getLevenshteinDistance(useCacheValue, useLookupvalue);
+ // Distances above the maximum are dropped below: stop counting there
+ default ->
+ data.levenshtein.distance(useCacheValue, useLookupvalue, data.maximalDistance);
};
if (data.minimalDistance <= cDistance && cDistance <= data.maximalDistance) {
@@ -268,6 +272,19 @@ public class FuzzyMatch extends BaseTransform<FuzzyMatchMeta, FuzzyMatchData> {
return buildMatchResults(topK, lowerIsBetter);
}
+ /**
+ * The lookup rows are all read before the first main row: take them in the order of the cache
+ * once, and lower-case their keys once instead of for every comparison.
+ */
+ private void prepareDistanceKeys() {
+ data.lookRows = data.look.toArray(new Object[0][]);
+ data.lookKeys = new String[data.lookRows.length];
+ for (int k = 0; k < data.lookRows.length; k++) {
+ String cacheValue = (String) data.lookRows[k][0];
+ data.lookKeys[k] = meta.isCaseSensitive() ? cacheValue : cacheValue.toLowerCase();
+ }
+ }
+
private List<Object[]> doPhonetic(Object[] row) {
boolean lowerIsBetter = true; // equal scores; first-seen kept by heap capacity
PriorityQueue<ScoredMatch> topK = newTopKHeap(lowerIsBetter);
@@ -614,6 +631,8 @@ public class FuzzyMatch extends BaseTransform<FuzzyMatchMeta, FuzzyMatchData> {
@Override
public void dispose() {
data.look.clear();
+ data.lookRows = null;
+ data.lookKeys = null;
super.dispose();
}
diff --git a/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatchData.java b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatchData.java
index 5466397f53..6ca95e61ba 100644
--- a/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatchData.java
+++ b/plugins/transforms/fuzzymatch/src/main/java/org/apache/hop/pipeline/transforms/fuzzymatch/FuzzyMatchData.java
@@ -63,6 +63,14 @@ public class FuzzyMatchData extends BaseTransformData implements ITransformData
public int nrCachedFields;
public IRowMeta infoCache;
+ /** The rows of {@link #look} in its iteration order, taken once the lookup stream is read */
+ public Object[][] lookRows;
+
+ /** The key of every row in {@link #lookRows}, lower-cased once when not case-sensitive */
+ public String[] lookKeys;
+
+ public final BoundedLevenshtein levenshtein = new BoundedLevenshtein();
+
public FuzzyMatchData() {
super();
this.look = new HashSet<>();
diff --git a/plugins/transforms/fuzzymatch/src/test/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshteinTest.java b/plugins/transforms/fuzzymatch/src/test/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshteinTest.java
new file mode 100644
index 0000000000..d2f3955f9e
--- /dev/null
+++ b/plugins/transforms/fuzzymatch/src/test/java/org/apache/hop/pipeline/transforms/fuzzymatch/BoundedLevenshteinTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.fuzzymatch;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Random;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link BoundedLevenshtein}. */
+class BoundedLevenshteinTest {
+
+ @Test
+ void testKnownDistances() {
+ BoundedLevenshtein levenshtein = new BoundedLevenshtein();
+ assertEquals(0, levenshtein.distance("hop", "hop", 2));
+ assertEquals(1, levenshtein.distance("hop", "hip", 2));
+ assertEquals(2, levenshtein.distance("smith", "smyth2", 2));
+ assertEquals(3, levenshtein.distance("kitten", "sitting", 3));
+ assertEquals(Integer.MAX_VALUE, levenshtein.distance("kitten", "sitting", 2));
+ }
+
+ @Test
+ void testEmptyStrings() {
+ BoundedLevenshtein levenshtein = new BoundedLevenshtein();
+ assertEquals(0, levenshtein.distance("", "", 0));
+ assertEquals(2, levenshtein.distance("", "ab", 2));
+ assertEquals(2, levenshtein.distance("ab", "", 5));
+ assertEquals(Integer.MAX_VALUE, levenshtein.distance("", "abc", 2));
+ }
+
+ @Test
+ void testNegativeMaximumMatchesNothing() {
+ assertEquals(Integer.MAX_VALUE, new BoundedLevenshtein().distance("a", "a", -1));
+ }
+
+ @Test
+ void testHugeMaximumDoesNotOverflow() {
+ BoundedLevenshtein levenshtein = new BoundedLevenshtein();
+ assertEquals(3, levenshtein.distance("kitten", "sitting", Integer.MAX_VALUE));
+ assertEquals(7, levenshtein.distance("", "sitting", Integer.MAX_VALUE));
+ }
+
+ /** Same result as the unbounded commons-lang3 distance whenever that one is within the max. */
+ @Test
+ void testMatchesUnboundedDistance() {
+ Random random = new Random(20261016L);
+ BoundedLevenshtein levenshtein = new BoundedLevenshtein();
+ String alphabet = "abcde";
+ for (int run = 0; run < 20_000; run++) {
+ String s = randomString(random, alphabet, random.nextInt(12));
+ String t = randomString(random, alphabet, random.nextInt(12));
+ int max = random.nextInt(8);
+ int expected = StringUtils.getLevenshteinDistance(s, t);
+ int actual = levenshtein.distance(s, t, max);
+ assertEquals(
+ expected <= max ? expected : Integer.MAX_VALUE,
+ actual,
+ () -> "distance(" + s + ", " + t + ", " + max + ")");
+ }
+ }
+
+ private static String randomString(Random random, String alphabet, int length) {
+ StringBuilder builder = new StringBuilder(length);
+ for (int i = 0; i < length; i++) {
+ builder.append(alphabet.charAt(random.nextInt(alphabet.length())));
+ }
+ return builder.toString();
+ }
+}
diff --git a/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperations.java b/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperations.java
index cb439a6dc3..ebf3c970c4 100644
--- a/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperations.java
+++ b/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperations.java
@@ -45,14 +45,15 @@ public class StringOperations extends BaseTransform<StringOperationsMeta, String
super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline);
}
- private String processString(String string, StringOperationsMeta.StringOperation operation) {
+ private String processString(
+ String string, StringOperationsMeta.StringOperation operation, int operationNr) {
String processed = processStringTrim(operation.getTrimType(), string);
processed = processStringLowerUpper(operation.getLowerUpper(), processed);
processed =
processStringPadding(
operation.getPaddingType(),
- operation.getPadChar(),
- Const.toInt(operation.getPadLen(), -1),
+ data.padChars[operationNr],
+ data.padLens[operationNr],
processed);
processed = processStringInitCap(operation.getInitCap(), processed);
processed = processStringMaskXml(operation.getMaskXml(), processed);
@@ -164,7 +165,7 @@ public class StringOperations extends BaseTransform<StringOperationsMeta, String
// Get source value
String value = getInputRowMeta().getString(row, data.inStreamNrs[i]);
// Apply String operations and return result value
- value = processString(value, operation);
+ value = processString(value, operation, i);
if (Utils.isEmpty(operation.getFieldOutStream())) {
// Update field
rowData[data.inStreamNrs[i]] = value;
@@ -227,8 +228,13 @@ public class StringOperations extends BaseTransform<StringOperationsMeta, String
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider);
data.inStreamNrs = new int[meta.getOperations().size()];
+ data.padChars = new String[meta.getOperations().size()];
+ data.padLens = new int[meta.getOperations().size()];
for (int i = 0; i < meta.getOperations().size(); i++) {
StringOperationsMeta.StringOperation operation = meta.getOperations().get(i);
+ // Resolve and parse the padding once, not for every row
+ data.padChars[i] = resolve(operation.getPadChar());
+ data.padLens[i] = Const.toInt(resolve(operation.getPadLen()), -1);
data.inStreamNrs[i] = getInputRowMeta().indexOfValue(operation.getFieldInStream());
if (data.inStreamNrs[i] < 0) { // couldn't find field!
throw new HopTransformException(
diff --git a/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsData.java b/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsData.java
index 7c08705273..4b11b54ee6 100644
--- a/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsData.java
+++ b/plugins/transforms/stringoperations/src/main/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsData.java
@@ -27,6 +27,12 @@ public class StringOperationsData extends BaseTransformData implements ITransfor
public int[] inStreamNrs; // string infields
+ /** Pad character per operation, variables resolved */
+ public String[] padChars;
+
+ /** Pad length per operation, variables resolved; -1 when not set */
+ public int[] padLens;
+
public IRowMeta outputRowMeta;
/** Default constructor. */
diff --git a/plugins/transforms/stringoperations/src/test/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsTest.java b/plugins/transforms/stringoperations/src/test/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsTest.java
index c875802d33..725d1a2667 100644
--- a/plugins/transforms/stringoperations/src/test/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsTest.java
+++ b/plugins/transforms/stringoperations/src/test/java/org/apache/hop/pipeline/transforms/stringoperations/StringOperationsTest.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.when;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Map;
import org.apache.hop.core.Const;
import org.apache.hop.core.QueueRowSet;
import org.apache.hop.core.exception.HopException;
@@ -81,10 +82,20 @@ class StringOperationsTest {
private Object[] processRows(StringOperationsMeta meta, RowMeta inputRowMeta, Object[] inputRow)
throws HopException {
+ return processRows(meta, inputRowMeta, inputRow, Map.of());
+ }
+
+ private Object[] processRows(
+ StringOperationsMeta meta,
+ RowMeta inputRowMeta,
+ Object[] inputRow,
+ Map<String, String> variables)
+ throws HopException {
StringOperationsData data = new StringOperationsData();
StringOperations transform =
new StringOperations(
mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline);
+ variables.forEach(transform::setVariable);
transform.setInputRowMeta(inputRowMeta);
transform.addRowSetToInputRowSets(mockHelper.getMockInputRowSet(inputRow));
QueueRowSet outputRowSet = new QueueRowSet();
@@ -159,6 +170,26 @@ class StringOperationsTest {
assertEquals("#####abc", output[1]);
}
+ @Test
+ void resolvesVariablesInPadCharAndLength() throws HopException {
+ StringOperationsMeta meta = new StringOperationsMeta();
+ StringOperationsMeta.StringOperation op = operationWithDefaults("text");
+ op.setPaddingType(StringOperationsMeta.Padding.RIGHT);
+ op.setPadChar("${PAD_CHAR}");
+ op.setPadLen("${PAD_LEN}");
+ op.setFieldOutStream("padded");
+ meta.getOperations().add(op);
+
+ RowMeta inputRowMeta = new RowMeta();
+ inputRowMeta.addValueMeta(new ValueMetaString("text"));
+
+ Object[] output =
+ processRows(
+ meta, inputRowMeta, new Object[] {"abc"}, Map.of("PAD_CHAR", "*", "PAD_LEN", "6"));
+
+ assertEquals("abc***", output[1]);
+ }
+
@Test
void appliesInitCapInPlace() throws HopException {
StringOperationsMeta meta = new StringOperationsMeta();
@@ -404,12 +435,14 @@ class StringOperationsTest {
StringOperations transform =
new StringOperations(
mockHelper.transformMeta, meta, data, 0, mockHelper.pipelineMeta, mockHelper.pipeline);
+ data.padChars = new String[] {operation.getPadChar()};
+ data.padLens = new int[] {Const.toInt(operation.getPadLen(), -1)};
Method method =
StringOperations.class.getDeclaredMethod(
- "processString", String.class, StringOperationsMeta.StringOperation.class);
+ "processString", String.class, StringOperationsMeta.StringOperation.class, int.class);
method.setAccessible(true);
try {
- return (String) method.invoke(transform, input, operation);
+ return (String) method.invoke(transform, input, operation, 0);
} catch (InvocationTargetException e) {
throw new Exception(e.getCause());
}