You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ba...@apache.org on 2026/09/28 06:20:57 UTC
(hop) branch main updated: Issue #8617 : Limit large String, JSON, Binary, and Avro values in data profiles (#8632)
This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new f69c2e1790 Issue #8617 : Limit large String, JSON, Binary, and Avro values in data profiles (#8632)
f69c2e1790 is described below
commit f69c2e1790ab5d74ef4e5a421e81a8b3ac5c25a3
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Mon Sep 28 08:20:51 2026 +0200
Issue #8617 : Limit large String, JSON, Binary, and Avro values in data profiles (#8632)
* Issue #8617 : Limit large String, JSON, Binary, and Avro values in data profiles
An execution data profile can leave oversized String, JSON, Binary, and Avro
values out of the sample rows it stores. An empty limit keeps today's behavior.
Zero stores none of that type. A longer String or JSON cell is replaced with a
short marker, and Binary or Avro cells over the limit are left empty.
fixes #8617
* Issue #8617 : Store sample markers in the field storage type
Replacing a lazy-conversion value with a Java String made execution data
registration fail for the whole tick. Markers now follow binary-string and
indexed storage. Kept cells are cloned the same way as a normal row copy,
a profiled row is measured once, and a negative or non-numeric limit is logged.
---
.../hop/core/row/value/ValueMetaAvroRecord.java | 46 +++
.../metadata-types/execution-data-profile.png | Bin 0 -> 111201 bytes
.../metadata-types/execution-data-profile.adoc | 8 +
.../execution/profiling/ExecutionDataProfile.java | 117 ++++--
.../execution/sampler/IExecutionDataSampler.java | 16 +
.../hop/execution/sampler/SampledValueLimits.java | 353 ++++++++++++++++++
.../sampler/plugins/ExecutionDataSamplerBase.java | 18 +
.../dataprof/BasicDataProfilingDataSampler.java | 68 +++-
.../first/FirstRowsExecutionDataSampler.java | 2 +-
.../plugins/last/LastRowsExecutionDataSampler.java | 2 +-
.../random/RandomRowsExecutionDataSampler.java | 4 +-
.../engines/local/LocalPipelineEngine.java | 1 +
.../profiling/messages/messages_en_US.properties | 11 +-
.../execution/sampler/SampledValueLimitsTest.java | 415 +++++++++++++++++++++
.../hop/beam/core/transform/TransformBaseFn.java | 6 +-
.../execution/SparkTransformExecutionSampling.java | 8 +-
.../profiling/ExecutionDataProfileEditor.java | 38 ++
17 files changed, 1053 insertions(+), 60 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
index 95ebf68f09..39d66ef5d5 100644
--- a/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
+++ b/core/src/main/java/org/apache/hop/core/row/value/ValueMetaAvroRecord.java
@@ -21,7 +21,9 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
@@ -456,6 +458,33 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
}
}
+ /**
+ * Bytes of the compact schema JSON plus the binary datum. Sample storage uses this as the size of
+ * an Avro value: the schema is part of the payload and {@link #writeData} writes the datum.
+ *
+ * @param object An Avro {@link GenericRecord}
+ * @return The schema and datum size in bytes
+ * @throws HopValueException When the value is not a record or cannot be encoded
+ */
+ public static int storedPayloadBytes(Object object) throws HopValueException {
+ if (!(object instanceof GenericRecord genericRecord) || genericRecord.getSchema() == null) {
+ throw new HopValueException(
+ "An Avro GenericRecord with a schema is needed to measure its size");
+ }
+ Schema recordSchema = genericRecord.getSchema();
+ int schemaBytes = recordSchema.toString(false).getBytes(StandardCharsets.UTF_8).length;
+ CountingOutputStream counting = new CountingOutputStream();
+ try {
+ BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(counting, null);
+ new GenericDatumWriter<GenericRecord>(recordSchema).write(genericRecord, encoder);
+ encoder.flush();
+ } catch (IOException e) {
+ throw new HopValueException("Unable to measure the size of an Avro record", e);
+ }
+ long total = (long) schemaBytes + counting.count;
+ return total > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) total;
+ }
+
public Schema getSchema() {
return schema;
}
@@ -484,4 +513,21 @@ public class ValueMetaAvroRecord extends ValueMetaBase {
return super.getInteger(object);
}
+
+ /** Counts bytes written by an Avro encoder without keeping them. */
+ private static final class CountingOutputStream extends OutputStream {
+ private long count;
+
+ @Override
+ public void write(int b) {
+ count++;
+ }
+
+ @Override
+ public void write(byte[] bytes, int offset, int length) {
+ if (length > 0) {
+ count += length;
+ }
+ }
+ }
}
diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png b/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png
new file mode 100644
index 0000000000..762de0af5d
Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/metadata-types/execution-data-profile.png differ
diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
index 27123d05aa..bac4c439e0 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/execution-data-profile.adoc
@@ -26,14 +26,22 @@ An Apache Hop Execution Data Profile builds data profiles as data flow through p
== Options
+image::metadata-types/execution-data-profile.png[Execution data profile editor, with the large value limits,width="90%"]
+
[options="header"]
|===
|Option |Description
|Name|The name to be used for this Execution Data Profile
|Description|A description to be used for this Execution Data Profile
+|Maximum string length|Maximum number of characters to keep for a String field. A JSON document kept in a String field uses this limit. Empty means no limit. `0` stores none. A longer value is replaced with `<not stored, N characters>`.
+|Maximum JSON length|Maximum number of characters of the compact JSON text to keep for a JSON field. Empty means no limit. `0` stores none. A longer value is replaced with a JSON text `<not stored, N characters>`.
+|Maximum binary size|Maximum number of bytes to keep for a Binary field. Empty means no limit. `0` stores none. A larger value is left empty in the sample.
+|Maximum Avro size|Maximum number of bytes of schema JSON plus Avro datum to keep for an Avro Record field. Empty means no limit. `0` stores none. A larger value is left empty in the sample.
|Data Samplers to use|One or more data samplers to use with this Execution Data Profile. See details below.
|===
+The four size limits apply to every sampler in the profile. Leave a field empty to keep today's behavior for that type. Minimum and maximum values are calculated only from values that were stored. Null counts and string lengths are still recorded for values that are left out. Minimum and maximum length example rows keep the other columns and replace the large value.
+
== Data Samplers
[options="header"]
diff --git a/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java b/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
index 9d36867d5a..f5828f4e15 100644
--- a/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
+++ b/engine/src/main/java/org/apache/hop/execution/profiling/ExecutionDataProfile.java
@@ -20,7 +20,15 @@ package org.apache.hop.execution.profiling;
import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.variables.IVariables;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
import org.apache.hop.metadata.api.HopMetadataCategory;
@@ -32,6 +40,9 @@ import org.apache.hop.metadata.api.IHopMetadata;
* This data profile makes it easy to specify what kind of information you want to capture about the
* output of a transform during its execution.
*/
+@Getter
+@Setter
+@GuiPlugin(description = "Execution data profile widgets")
@HopMetadata(
key = "execution-data-profile",
name = "i18n::ExecutionDataProfile.name",
@@ -44,8 +55,60 @@ import org.apache.hop.metadata.api.IHopMetadata;
public class ExecutionDataProfile extends HopMetadataBase implements IHopMetadata, Cloneable {
public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "ExecutionDataSamplerParent";
+ public static final String GUI_PLUGIN_LIMITS_PARENT_ID = "ExecutionDataProfileLimits";
+
+ private static final String LARGE_VALUES_GROUP = "i18n::ExecutionDataProfile.Group.LargeValues";
+
@HopMetadataProperty private String description;
+ @GuiWidgetElement(
+ id = "stringValueLimit",
+ order = "100",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.StringValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.StringValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String stringValueLimit;
+
+ @GuiWidgetElement(
+ id = "jsonValueLimit",
+ order = "110",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.JsonValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.JsonValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String jsonValueLimit;
+
+ @GuiWidgetElement(
+ id = "binaryValueLimit",
+ order = "120",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.BinaryValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.BinaryValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String binaryValueLimit;
+
+ @GuiWidgetElement(
+ id = "avroValueLimit",
+ order = "130",
+ type = GuiElementType.TEXT,
+ parentId = GUI_PLUGIN_LIMITS_PARENT_ID,
+ label = "i18n::ExecutionDataProfile.Label.AvroValueLimit",
+ toolTip = "i18n::ExecutionDataProfile.Tooltip.AvroValueLimit",
+ groupType = GuiWidgetGroupType.BOXES,
+ group = LARGE_VALUES_GROUP)
+ @HopMetadataProperty
+ private String avroValueLimit;
+
@HopMetadataProperty(groupKey = "samplers", key = "sampler")
private List<IExecutionDataSampler> samplers;
@@ -68,49 +131,35 @@ public class ExecutionDataProfile extends HopMetadataBase implements IHopMetadat
public ExecutionDataProfile(ExecutionDataProfile profile) {
this(profile.name);
this.description = profile.description;
+ this.stringValueLimit = profile.stringValueLimit;
+ this.jsonValueLimit = profile.jsonValueLimit;
+ this.binaryValueLimit = profile.binaryValueLimit;
+ this.avroValueLimit = profile.avroValueLimit;
for (IExecutionDataSampler<?> sampler : profile.samplers) {
this.samplers.add(sampler.clone());
}
}
- @Override
- protected ExecutionDataProfile clone() {
- return new ExecutionDataProfile(this);
- }
-
- /**
- * Gets description
- *
- * @return value of description
- */
- public String getDescription() {
- return description;
- }
-
/**
- * Sets description
+ * Resolve this profile's value limits and attach them to every sampler that will store rows.
*
- * @param description value of description
+ * @param samplers Profile samplers and any extra samplers added by the engine
+ * @param variables Pipeline variables used to resolve the limit fields
*/
- public void setDescription(String description) {
- this.description = description;
- }
-
- /**
- * Gets samplers
- *
- * @return value of samplers
- */
- public List<IExecutionDataSampler> getSamplers() {
- return samplers;
+ public void applyLimits(List<IExecutionDataSampler> samplers, IVariables variables) {
+ if (samplers == null) {
+ return;
+ }
+ SampledValueLimits limits = SampledValueLimits.from(this, variables);
+ for (IExecutionDataSampler sampler : samplers) {
+ if (sampler != null) {
+ sampler.setSampledValueLimits(limits);
+ }
+ }
}
- /**
- * Sets samplers
- *
- * @param samplers value of samplers
- */
- public void setSamplers(List<IExecutionDataSampler> samplers) {
- this.samplers = samplers;
+ @Override
+ protected ExecutionDataProfile clone() {
+ return new ExecutionDataProfile(this);
}
}
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java b/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
index 748526350b..28ed68c938 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/IExecutionDataSampler.java
@@ -18,6 +18,7 @@
package org.apache.hop.execution.sampler;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.core.plugins.PluginRegistry;
@@ -60,6 +61,21 @@ public interface IExecutionDataSampler<Store extends IExecutionDataSamplerStore>
void sampleRow(Store samplerStore, IStream.StreamType streamType, IRowMeta rowMeta, Object[] row)
throws HopException;
+ /**
+ * Limits resolved from the execution data profile before sampling starts. Not metadata: Beam and
+ * Spark must not write this into the sampler JSON.
+ */
+ @JsonIgnore
+ default void setSampledValueLimits(SampledValueLimits limits) {}
+
+ /**
+ * @return The limits for this sampler, or no limits when the profile did not set any.
+ */
+ @JsonIgnore
+ default SampledValueLimits getSampledValueLimits() {
+ return SampledValueLimits.unlimited();
+ }
+
/**
* This object factory is needed to instantiate the correct plugin class based on the value of the
* Object ID which is simply the object ID.
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java b/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java
new file mode 100644
index 0000000000..51240bac8e
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/SampledValueLimits.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.execution.sampler;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import java.nio.charset.StandardCharsets;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopValueException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.value.ValueMetaAvroRecord;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.util.JsonUtil;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.execution.profiling.ExecutionDataProfile;
+
+/**
+ * Resolved limits for String, JSON, Binary, and Avro values kept by an execution data profile. A
+ * blank setting means no limit. A negative or non-numeric setting is logged and then treated as no
+ * limit. Zero stores none of that type.
+ */
+@Getter
+public final class SampledValueLimits {
+ public static final String NOT_STORED = "<not stored>";
+
+ private static final SampledValueLimits UNLIMITED =
+ new SampledValueLimits(null, null, null, null);
+ private static final String CHARACTERS = "characters";
+
+ private final Integer stringLimit;
+ private final Integer jsonLimit;
+ private final Integer binaryLimit;
+ private final Integer avroLimit;
+
+ private SampledValueLimits(
+ Integer stringLimit, Integer jsonLimit, Integer binaryLimit, Integer avroLimit) {
+ this.stringLimit = stringLimit;
+ this.jsonLimit = jsonLimit;
+ this.binaryLimit = binaryLimit;
+ this.avroLimit = avroLimit;
+ }
+
+ public static SampledValueLimits unlimited() {
+ return UNLIMITED;
+ }
+
+ /**
+ * Marker stored in place of a String or JSON value that is over its limit.
+ *
+ * @param size The measured size that was left out
+ * @param unit {@code characters} for String and JSON
+ * @return The short replacement text
+ */
+ public static String notStored(int size, String unit) {
+ return "<not stored, " + size + " " + unit + ">";
+ }
+
+ /**
+ * Resolve the profile fields with {@code variables}. A blank value is unlimited. A negative or
+ * non-numeric value is unlimited and logged, so a typo does not turn the limit off quietly. Zero
+ * stores nothing of that type.
+ */
+ public static SampledValueLimits from(ExecutionDataProfile profile, IVariables variables) {
+ if (profile == null) {
+ return unlimited();
+ }
+ return new SampledValueLimits(
+ parseLimit(resolve(variables, profile.getStringValueLimit())),
+ parseLimit(resolve(variables, profile.getJsonValueLimit())),
+ parseLimit(resolve(variables, profile.getBinaryValueLimit())),
+ parseLimit(resolve(variables, profile.getAvroValueLimit())));
+ }
+
+ public boolean hasLimits() {
+ return stringLimit != null || jsonLimit != null || binaryLimit != null || avroLimit != null;
+ }
+
+ /**
+ * @return true when this value must not be kept as itself. Nulls are kept as null.
+ */
+ public boolean omit(IValueMeta valueMeta, Object value) throws HopValueException {
+ return decide(valueMeta, value).omit;
+ }
+
+ /**
+ * Measure every field once. Profiling calls {@link #copyRow(IRowMeta, Object[], Decision[])}
+ * several times for the same row, and measuring JSON or Avro serialises the value.
+ */
+ public Decision[] decisionsFor(IRowMeta rowMeta, Object[] row) throws HopValueException {
+ if (row == null || rowMeta == null) {
+ return new Decision[0];
+ }
+ int fields = Math.min(rowMeta.size(), row.length);
+ Decision[] decisions = new Decision[fields];
+ for (int i = 0; i < fields; i++) {
+ decisions[i] = decide(rowMeta.getValueMeta(i), row[i]);
+ }
+ return decisions;
+ }
+
+ /**
+ * A new row array for storage. Cells over a limit are replaced. Cells that stay are cloned. The
+ * pipeline row is left unchanged, and a value that is dropped is not copied first.
+ */
+ public Object[] copyRow(IRowMeta rowMeta, Object[] row) throws HopValueException {
+ return copyRow(rowMeta, row, null);
+ }
+
+ /**
+ * Same as {@link #copyRow(IRowMeta, Object[])}, using decisions already computed for this row.
+ */
+ public Object[] copyRow(IRowMeta rowMeta, Object[] row, Decision[] decisions)
+ throws HopValueException {
+ if (row == null) {
+ return null;
+ }
+ if (rowMeta == null || !hasLimits()) {
+ return rowMeta == null ? row.clone() : rowMeta.cloneRow(row);
+ }
+
+ int fields = Math.min(rowMeta.size(), row.length);
+ if (decisions == null) {
+ decisions = decisionsFor(rowMeta, row);
+ } else {
+ fields = Math.min(fields, decisions.length);
+ }
+ boolean anyOmitted = false;
+ for (int i = 0; i < fields; i++) {
+ anyOmitted = anyOmitted || decisions[i].omit();
+ }
+ if (!anyOmitted) {
+ return rowMeta.cloneRow(row);
+ }
+
+ Object[] copy = row.clone();
+ for (int i = 0; i < fields; i++) {
+ if (decisions[i].omit()) {
+ copy[i] = decisions[i].replacement();
+ } else {
+ copy[i] = copyKept(rowMeta.getValueMeta(i), row[i]);
+ }
+ }
+ return copy;
+ }
+
+ private static Object copyKept(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (value == null || valueMeta == null) {
+ return value;
+ }
+ return valueMeta.cloneValueData(value);
+ }
+
+ private Decision decide(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (valueMeta == null || value == null || valueMeta.isNull(value)) {
+ return Decision.keep();
+ }
+ return switch (valueMeta.getType()) {
+ case IValueMeta.TYPE_STRING -> decideString(valueMeta, value);
+ case IValueMeta.TYPE_JSON -> decideJson(valueMeta, value);
+ case IValueMeta.TYPE_BINARY -> decideBinary(valueMeta, value);
+ case IValueMeta.TYPE_AVRO -> decideAvro(value);
+ default -> Decision.keep();
+ };
+ }
+
+ private Decision decideString(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (stringLimit == null) {
+ return Decision.keep();
+ }
+ if (stringLimit == 0) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ int length = stringLength(valueMeta, value);
+ if (length > stringLimit) {
+ return Decision.drop(marker(valueMeta, notStored(length, CHARACTERS)));
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideJson(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (jsonLimit == null) {
+ return Decision.keep();
+ }
+ if (jsonLimit == 0) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ int length;
+ try {
+ length = jsonLength(valueMeta, value);
+ } catch (Exception e) {
+ return Decision.drop(marker(valueMeta, NOT_STORED));
+ }
+ if (length > jsonLimit) {
+ return Decision.drop(marker(valueMeta, notStored(length, CHARACTERS)));
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideBinary(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (binaryLimit == null) {
+ return Decision.keep();
+ }
+ if (binaryLimit == 0) {
+ return Decision.drop(null);
+ }
+ int size;
+ try {
+ size = binarySize(valueMeta, value);
+ } catch (Exception e) {
+ return Decision.drop(null);
+ }
+ if (size > binaryLimit) {
+ return Decision.drop(null);
+ }
+ return Decision.keep();
+ }
+
+ private Decision decideAvro(Object value) {
+ if (avroLimit == null) {
+ return Decision.keep();
+ }
+ if (avroLimit == 0) {
+ return Decision.drop(null);
+ }
+ int size;
+ try {
+ size = ValueMetaAvroRecord.storedPayloadBytes(value);
+ } catch (Exception e) {
+ return Decision.drop(null);
+ }
+ if (size > avroLimit) {
+ return Decision.drop(null);
+ }
+ return Decision.keep();
+ }
+
+ private static int stringLength(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_BINARY_STRING
+ && value instanceof byte[] bytes) {
+ String text = valueMeta.getString(value);
+ return text == null ? bytes.length : text.length();
+ }
+ if (value instanceof String text) {
+ return text.length();
+ }
+ String text = valueMeta.getString(value);
+ return text == null ? 0 : text.length();
+ }
+
+ private static int jsonLength(IValueMeta valueMeta, Object value) throws Exception {
+ if (valueMeta.getStorageType() == IValueMeta.STORAGE_TYPE_BINARY_STRING
+ && value instanceof byte[] bytes) {
+ return new String(bytes, StandardCharsets.UTF_8).length();
+ }
+ JsonNode node;
+ if (value instanceof JsonNode jsonNode) {
+ node = jsonNode;
+ } else if (valueMeta instanceof ValueMetaJson jsonMeta) {
+ node = jsonMeta.getJson(value);
+ } else {
+ node = valueMeta.getJson(value);
+ }
+ if (node == null) {
+ return 0;
+ }
+ String text = JsonUtil.mapJsonToString(node, false);
+ return text == null ? 0 : text.length();
+ }
+
+ private static int binarySize(IValueMeta valueMeta, Object value) throws HopValueException {
+ if (value instanceof byte[] bytes) {
+ return bytes.length;
+ }
+ byte[] bytes = valueMeta.getBinary(value);
+ return bytes == null ? 0 : bytes.length;
+ }
+
+ private static String resolve(IVariables variables, String value) {
+ if (value == null || variables == null) {
+ return value;
+ }
+ return variables.resolve(value);
+ }
+
+ /**
+ * A replacement that {@link org.apache.hop.core.row.value.ValueMetaBase#writeData} can store.
+ * Lazy conversion keeps a {@code byte[]} and an indexed field keeps an index, so a Java String
+ * marker would fail execution-data registration for the whole tick.
+ */
+ private static Object marker(IValueMeta valueMeta, String text) {
+ if (valueMeta == null) {
+ return text;
+ }
+ return switch (valueMeta.getStorageType()) {
+ case IValueMeta.STORAGE_TYPE_BINARY_STRING -> text.getBytes(StandardCharsets.UTF_8);
+ case IValueMeta.STORAGE_TYPE_INDEXED -> null;
+ default -> valueMeta.getType() == IValueMeta.TYPE_JSON ? TextNode.valueOf(text) : text;
+ };
+ }
+
+ private static Integer parseLimit(String raw) {
+ if (StringUtils.isBlank(raw)) {
+ return null;
+ }
+ String trimmed = raw.trim();
+ try {
+ int parsed = Integer.parseInt(trimmed);
+ if (parsed < 0) {
+ LogChannel.GENERAL.logError(
+ "Execution data profile value limit '"
+ + trimmed
+ + "' is negative and is ignored. The value is stored without a limit.");
+ return null;
+ }
+ return parsed;
+ } catch (NumberFormatException e) {
+ LogChannel.GENERAL.logError(
+ "Execution data profile value limit '"
+ + trimmed
+ + "' is not a number and is ignored. The value is stored without a limit.");
+ return null;
+ }
+ }
+
+ /** Whether one cell is kept, and the storage-typed marker used when it is not. */
+ public record Decision(boolean omit, Object replacement) {
+ private static Decision keep() {
+ return new Decision(false, null);
+ }
+
+ private static Decision drop(Object replacement) {
+ return new Decision(true, replacement);
+ }
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
index ba36dd0172..c774a4b500 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/ExecutionDataSamplerBase.java
@@ -18,6 +18,7 @@
package org.apache.hop.execution.sampler.plugins;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Objects;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
@@ -25,6 +26,7 @@ import org.apache.hop.execution.profiling.ExecutionDataProfile;
import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
import org.apache.hop.execution.sampler.IExecutionDataSamplerStore;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.metadata.api.HopMetadataProperty;
public abstract class ExecutionDataSamplerBase<Store extends IExecutionDataSamplerStore>
@@ -39,6 +41,9 @@ public abstract class ExecutionDataSamplerBase<Store extends IExecutionDataSampl
@HopMetadataProperty
protected String sampleSize;
+ /** Resolved from the profile at runtime. Not part of the sampler metadata. */
+ @JsonIgnore protected SampledValueLimits sampledValueLimits;
+
protected String pluginId;
protected String pluginName;
@@ -57,6 +62,7 @@ public abstract class ExecutionDataSamplerBase<Store extends IExecutionDataSampl
this.sampleSize = base.sampleSize;
this.pluginId = base.pluginId;
this.pluginName = base.pluginName;
+ this.sampledValueLimits = base.sampledValueLimits;
}
@Override
@@ -96,6 +102,18 @@ public abstract class ExecutionDataSamplerBase<Store extends IExecutionDataSampl
this.sampleSize = sampleSize;
}
+ @Override
+ @JsonIgnore
+ public void setSampledValueLimits(SampledValueLimits sampledValueLimits) {
+ this.sampledValueLimits = sampledValueLimits;
+ }
+
+ @Override
+ @JsonIgnore
+ public SampledValueLimits getSampledValueLimits() {
+ return sampledValueLimits == null ? SampledValueLimits.unlimited() : sampledValueLimits;
+ }
+
/**
* Gets pluginId
*
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
index 568886d095..7ad676ab9a 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/dataprof/BasicDataProfilingDataSampler.java
@@ -18,9 +18,11 @@
package org.apache.hop.execution.sampler.plugins.dataprof;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.HashMap;
import java.util.Map;
import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.exception.HopValueException;
import org.apache.hop.core.gui.plugin.GuiElementType;
import org.apache.hop.core.gui.plugin.GuiPlugin;
import org.apache.hop.core.gui.plugin.GuiWidgetElement;
@@ -31,6 +33,7 @@ import org.apache.hop.execution.profiling.ExecutionDataProfile;
import org.apache.hop.execution.sampler.ExecutionDataSamplerMeta;
import org.apache.hop.execution.sampler.ExecutionDataSamplerPlugin;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
+import org.apache.hop.execution.sampler.SampledValueLimits;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.pipeline.transform.stream.IStream;
@@ -72,6 +75,9 @@ public class BasicDataProfilingDataSampler
private String pluginId;
private String pluginName;
+ /** Resolved from the profile at runtime. Not part of the sampler metadata. */
+ @JsonIgnore private SampledValueLimits sampledValueLimits;
+
// The metadata fields
//
@@ -163,6 +169,7 @@ public class BasicDataProfilingDataSampler
this.profilingNrNonNull = sampler.profilingNrNonNull;
this.profilingMinLength = sampler.profilingMinLength;
this.profilingMaxLength = sampler.profilingMaxLength;
+ this.sampledValueLimits = sampler.sampledValueLimits;
}
public BasicDataProfilingDataSampler clone() {
@@ -195,6 +202,12 @@ public class BasicDataProfilingDataSampler
try {
+ // Measure each field once. Min, max, length and null samples all store a copy of this row.
+ //
+ SampledValueLimits limits = getSampledValueLimits();
+ SampledValueLimits.Decision[] decisions =
+ limits.hasLimits() ? limits.decisionsFor(rowMeta, row) : null;
+
// Profile all columns
//
for (int i = 0; i < rowMeta.size(); i++) {
@@ -206,66 +219,71 @@ public class BasicDataProfilingDataSampler
if (profilingNrNull) {
long counter = store.getNullCounters().getOrDefault(name, 0L);
store.getNullCounters().put(name, ++counter);
- addSampleRow(store, name, ProfilingType.NrNulls, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.NrNulls, rowMeta, row, decisions);
}
} else {
if (profilingNrNonNull) {
long counter = store.getNonNullCounters().getOrDefault(name, 0L);
store.getNonNullCounters().put(name, ++counter);
- addSampleRow(store, name, ProfilingType.NrNonNulls, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.NrNonNulls, rowMeta, row, decisions);
}
}
+ // A value over its type limit is still counted above. It is not kept as a minimum or
+ // maximum, because that would retain the large value.
+ //
+ boolean storeValue = decisions == null || i >= decisions.length || !decisions[i].omit();
+
// Minimum
//
- if (profilingMinimum) {
+ if (profilingMinimum && storeValue) {
Object oldMin = store.getMinValues().get(name);
if (oldMin == null) {
- store.getMinValues().put(name, valueData);
+ store.getMinValues().put(name, valueMeta.cloneValueData(valueData));
store.getMinMeta().put(name, valueMeta);
} else {
int compare = valueMeta.compare(valueData, oldMin);
if (compare < 0) {
// We have a new minimum
//
- store.getMinValues().put(name, valueData);
+ store.getMinValues().put(name, valueMeta.cloneValueData(valueData));
store.getMinMeta().put(name, valueMeta);
clearSampleRows(store, name, ProfilingType.MinValue);
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row, decisions);
} else if (compare == 0) {
// We found another value at the current minimum
- addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinValue, rowMeta, row, decisions);
}
}
}
// Maximum
//
- if (profilingMaximum) {
+ if (profilingMaximum && storeValue) {
Object oldMax = store.getMaxValues().get(name);
if (oldMax == null) {
- store.getMaxValues().put(name, valueData);
+ store.getMaxValues().put(name, valueMeta.cloneValueData(valueData));
store.getMaxMeta().put(name, valueMeta);
} else {
int compare = valueMeta.compare(valueData, oldMax);
if (compare > 0) {
// We have a new maximum
//
- store.getMaxValues().put(name, valueData);
+ store.getMaxValues().put(name, valueMeta.cloneValueData(valueData));
store.getMaxMeta().put(name, valueMeta);
clearSampleRows(store, name, ProfilingType.MaxValue);
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row, decisions);
} else if (compare == 0) {
// We found another value at the current maximum
- addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxValue, rowMeta, row, decisions);
}
}
}
@@ -292,10 +310,10 @@ public class BasicDataProfilingDataSampler
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MinLength, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinLength, rowMeta, row, decisions);
} else if (length == oldMin) {
// We found another value at the current minimum length
- addSampleRow(store, name, ProfilingType.MinLength, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MinLength, rowMeta, row, decisions);
}
}
}
@@ -319,10 +337,10 @@ public class BasicDataProfilingDataSampler
// Also save the row of data as a sample
//
- addSampleRow(store, name, ProfilingType.MaxLength, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxLength, rowMeta, row, decisions);
} else if (length == oldMax) {
// We found another value at the current maximum length
- addSampleRow(store, name, ProfilingType.MaxLength, rowMeta, row);
+ addSampleRow(store, name, ProfilingType.MaxLength, rowMeta, row, decisions);
}
}
}
@@ -346,7 +364,9 @@ public class BasicDataProfilingDataSampler
String name,
ProfilingType profilingType,
IRowMeta rowMeta,
- Object[] row) {
+ Object[] row,
+ SampledValueLimits.Decision[] decisions)
+ throws HopValueException {
synchronized (store.getProfileSamples()) {
Map<ProfilingType, RowBuffer> typeBufferMap =
store.getProfileSamples().computeIfAbsent(name, k -> new HashMap<>());
@@ -356,7 +376,7 @@ public class BasicDataProfilingDataSampler
// Keep the memory consumption sane
//
if (rowBuffer.size() < store.getMaxRows()) {
- rowBuffer.addRow(row);
+ rowBuffer.addRow(getSampledValueLimits().copyRow(rowMeta, row, decisions));
}
}
}
@@ -544,4 +564,16 @@ public class BasicDataProfilingDataSampler
public void setProfilingMaxLength(boolean profilingMaxLength) {
this.profilingMaxLength = profilingMaxLength;
}
+
+ @Override
+ @JsonIgnore
+ public void setSampledValueLimits(SampledValueLimits sampledValueLimits) {
+ this.sampledValueLimits = sampledValueLimits;
+ }
+
+ @Override
+ @JsonIgnore
+ public SampledValueLimits getSampledValueLimits() {
+ return sampledValueLimits == null ? SampledValueLimits.unlimited() : sampledValueLimits;
+ }
}
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
index 27239e77a8..6cfbe00470 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/first/FirstRowsExecutionDataSampler.java
@@ -79,7 +79,7 @@ public class FirstRowsExecutionDataSampler
if (rows.isEmpty()) {
samplerStore.setRowMeta(rowMeta);
}
- rows.add(rowMeta.cloneRow(row));
+ rows.add(getSampledValueLimits().copyRow(rowMeta, row));
}
}
}
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
index f7d1a707fa..c3ff990ef1 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/last/LastRowsExecutionDataSampler.java
@@ -82,7 +82,7 @@ public class LastRowsExecutionDataSampler
// Add to the front, remove the last on overflow
// This is why we use a linked list
//
- rows.add(0, rowMeta.cloneRow(row));
+ rows.add(0, getSampledValueLimits().copyRow(rowMeta, row));
if (rows.size() > samplerStore.getMaxRows()) {
rows.remove(rows.size() - 1);
}
diff --git a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
index 6f5511551c..0d947422a1 100644
--- a/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
+++ b/engine/src/main/java/org/apache/hop/execution/sampler/plugins/random/RandomRowsExecutionDataSampler.java
@@ -83,11 +83,11 @@ public class RandomRowsExecutionDataSampler
if (rows.isEmpty()) {
samplerStore.setRowMeta(rowMeta);
}
- rows.add(row);
+ rows.add(getSampledValueLimits().copyRow(rowMeta, row));
} else {
int randomIndex = random.nextInt(samplerStore.getMaxRows());
if (randomIndex < samplerStore.getMaxRows()) {
- rows.set(randomIndex, rowMeta.cloneRow(row));
+ rows.set(randomIndex, getSampledValueLimits().copyRow(rowMeta, row));
}
}
}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
index 8359f8c884..7d77daea84 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/engines/local/LocalPipelineEngine.java
@@ -398,6 +398,7 @@ public class LocalPipelineEngine extends Pipeline implements IPipelineEngine<Pip
if (samplers.isEmpty()) {
return;
}
+ profile.applyLimits(samplers, this);
samplerStoresMap = new HashMap<>();
diff --git a/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties b/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
index 04a62301be..a65615d560 100644
--- a/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
+++ b/engine/src/main/resources/org/apache/hop/execution/profiling/messages/messages_en_US.properties
@@ -17,4 +17,13 @@
#
ExecutionDataProfile.name=Execution Data Profile
-ExecutionDataProfile.description=Describes how transform output data is profiled
\ No newline at end of file
+ExecutionDataProfile.description=Describes how transform output data is profiled
+ExecutionDataProfile.Group.LargeValues=Large values
+ExecutionDataProfile.Label.StringValueLimit=Maximum string length
+ExecutionDataProfile.Tooltip.StringValueLimit=Maximum number of characters of a String field to keep in sample rows and in minimum and maximum values. A REST or JSON body stored in a String field uses this limit. Leave empty for no limit. 0 stores none of these values. A longer value is replaced with '<not stored, N characters>'. You can use a variable, for example '${SAMPLE_STRING_LIMIT}'. Minimum and maximum values are calculated only from values that are stored.
+ExecutionDataProfile.Label.JsonValueLimit=Maximum JSON length
+ExecutionDataProfile.Tooltip.JsonValueLimit=Maximum number of characters of the compact JSON text to keep for a JSON field. Leave empty for no limit. 0 stores none of these values. A longer value is replaced with a JSON text '<not stored, N characters>'. You can use a variable, for example '${SAMPLE_JSON_LIMIT}'.
+ExecutionDataProfile.Label.BinaryValueLimit=Maximum binary size (bytes)
+ExecutionDataProfile.Tooltip.BinaryValueLimit=Maximum number of bytes to keep for a Binary field. Leave empty for no limit. 0 stores none of these values. A larger value is left empty in the sample. You can use a variable, for example '${SAMPLE_BINARY_LIMIT}'.
+ExecutionDataProfile.Label.AvroValueLimit=Maximum Avro size (bytes)
+ExecutionDataProfile.Tooltip.AvroValueLimit=Maximum number of bytes of schema JSON plus Avro datum to keep for an Avro Record field. Leave empty for no limit. 0 stores none of these values. A larger value is left empty in the sample. You can use a variable, for example '${SAMPLE_AVRO_LIMIT}'.
\ No newline at end of file
diff --git a/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java b/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java
new file mode 100644
index 0000000000..8f6ba079bf
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/execution/sampler/SampledValueLimitsTest.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.execution.sampler;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.lang.reflect.Field;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.gui.plugin.GuiWidgetGroupType;
+import org.apache.hop.core.json.HopJson;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.row.IValueMeta;
+import org.apache.hop.core.row.RowBuffer;
+import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaAvroRecord;
+import org.apache.hop.core.row.value.ValueMetaBinary;
+import org.apache.hop.core.row.value.ValueMetaJson;
+import org.apache.hop.core.row.value.ValueMetaString;
+import org.apache.hop.core.row.value.ValueMetaTimestamp;
+import org.apache.hop.core.util.JsonUtil;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.execution.ExecutionData;
+import org.apache.hop.execution.profiling.ExecutionDataProfile;
+import org.apache.hop.execution.sampler.plugins.dataprof.BasicDataProfilingDataSampler;
+import org.apache.hop.execution.sampler.plugins.dataprof.BasicDataProfilingDataSamplerStore;
+import org.apache.hop.execution.sampler.plugins.first.FirstRowsExecutionDataSampler;
+import org.apache.hop.execution.sampler.plugins.first.FirstRowsExecutionDataSamplerStore;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.transform.stream.IStream;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class SampledValueLimitsTest {
+
+ @BeforeAll
+ static void initHop() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @Test
+ void unlimitedCopyLeavesEveryValueInANewRow() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ byte[] binary = new byte[] {1, 2, 3, 4};
+ JsonNode json = JsonUtil.jsonMapper().readTree("{\"ok\":true}");
+ GenericRecord avro = avroRecord("hi");
+ Object[] row = new Object[] {"text", json, binary, avro};
+
+ Object[] copy = SampledValueLimits.unlimited().copyRow(rowMeta, row);
+
+ assertNotSame(row, copy);
+ assertEquals("text", copy[0]);
+ assertEquals(json, copy[1]);
+ assertArrayEquals(binary, (byte[]) copy[2]);
+ assertNotSame(binary, copy[2]);
+ assertEquals("hi", ((GenericRecord) copy[3]).get("a").toString());
+ assertSame("text", row[0]);
+ assertSame(json, row[1]);
+ assertSame(binary, row[2]);
+ assertSame(avro, row[3]);
+ }
+
+ @Test
+ void zeroOnOneTypeReplacesOnlyThatType() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ Object[] row =
+ new Object[] {
+ "text", JsonUtil.jsonMapper().readTree("{\"ok\":true}"), new byte[] {9}, avroRecord("hi")
+ };
+
+ Object[] copy = limits(null, null, "0", null).copyRow(rowMeta, row);
+
+ assertEquals("text", copy[0]);
+ assertEquals(row[1], copy[1]);
+ assertNull(copy[2]);
+ assertEquals("hi", ((GenericRecord) copy[3]).get("a").toString());
+ assertArrayEquals(new byte[] {9}, (byte[]) row[2]);
+ }
+
+ @Test
+ void positiveStringLimitKeepsShortValuesAndReplacesLongOnes() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ String longText = "this is longer than five";
+ Object[] shortRow = new Object[] {"short"};
+ Object[] longRow = new Object[] {longText};
+ SampledValueLimits valueLimits = limits("5", null, null, null);
+
+ Object[] shortCopy = valueLimits.copyRow(rowMeta, shortRow);
+ Object[] longCopy = valueLimits.copyRow(rowMeta, longRow);
+
+ assertEquals("short", shortCopy[0]);
+ assertEquals(SampledValueLimits.notStored(longText.length(), "characters"), longCopy[0]);
+ assertSame(longText, longRow[0]);
+ assertNotSame(longRow, longCopy);
+ }
+
+ @Test
+ void binaryAndAvroOverTheLimitAreNullAndSchemaBytesCount() throws Exception {
+ IRowMeta rowMeta = sampleRowMeta();
+ byte[] binary = new byte[] {1, 2, 3, 4, 5};
+ GenericRecord small = avroRecord("hi");
+ int smallSize = ValueMetaAvroRecord.storedPayloadBytes(small);
+ GenericRecord wideSchema = wideSchemaRecord();
+ int wideSize = ValueMetaAvroRecord.storedPayloadBytes(wideSchema);
+ assertTrue(wideSize > smallSize);
+
+ Object[] overBinary =
+ limits(null, null, "4", null)
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), binary, small});
+ assertNull(overBinary[2]);
+ assertNotNull(overBinary[3]);
+
+ Object[] overSchema =
+ limits(null, null, null, Integer.toString(smallSize))
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), new byte[] {1}, wideSchema});
+ assertNull(overSchema[3]);
+
+ Object[] underSchema =
+ limits(null, null, "0", Integer.toString(smallSize))
+ .copyRow(rowMeta, new Object[] {"text", json("{\"a\":1}"), new byte[] {1}, small});
+ assertNull(underSchema[2]);
+ assertEquals("hi", ((GenericRecord) underSchema[3]).get("a").toString());
+ assertNotSame(small, underSchema[3]);
+ }
+
+ @Test
+ void binaryStringAndIndexedMarkersMatchTheFieldStorage() throws Exception {
+ String longText = "this is longer than five";
+ SampledValueLimits valueLimits = limits("5", "5", null, null);
+
+ ValueMetaString lazy = new ValueMetaString("body");
+ lazy.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ lazy.setStorageMetadata(new ValueMetaString("body"));
+ IRowMeta lazyMeta = new RowMeta();
+ lazyMeta.addValueMeta(lazy);
+ Object[] lazyCopy =
+ valueLimits.copyRow(lazyMeta, new Object[] {longText.getBytes(StandardCharsets.UTF_8)});
+ assertInstanceOf(byte[].class, lazyCopy[0]);
+ assertEquals(
+ SampledValueLimits.notStored(longText.length(), "characters"),
+ new String((byte[]) lazyCopy[0], StandardCharsets.UTF_8));
+
+ RowBuffer buffer = new RowBuffer(lazyMeta);
+ buffer.addRow(lazyCopy);
+ ExecutionData executionData = new ExecutionData();
+ executionData.getDataSets().put("rows", buffer);
+ assertNotNull(executionData.getRowsBinaryGzipBase64Encoded());
+
+ ValueMetaJson lazyJson = new ValueMetaJson("payload");
+ lazyJson.setStorageType(IValueMeta.STORAGE_TYPE_BINARY_STRING);
+ lazyJson.setStorageMetadata(new ValueMetaString("payload"));
+ IRowMeta jsonMeta = new RowMeta();
+ jsonMeta.addValueMeta(lazyJson);
+ byte[] jsonBytes = "{\"a\":\"0123456789\"}".getBytes(StandardCharsets.UTF_8);
+ Object[] jsonCopy = valueLimits.copyRow(jsonMeta, new Object[] {jsonBytes});
+ assertInstanceOf(byte[].class, jsonCopy[0]);
+
+ ValueMetaString indexed = new ValueMetaString("body");
+ indexed.setStorageType(IValueMeta.STORAGE_TYPE_INDEXED);
+ indexed.setIndex(new Object[] {"short", longText});
+ indexed.setStorageMetadata(new ValueMetaString("body"));
+ IRowMeta indexedMeta = new RowMeta();
+ indexedMeta.addValueMeta(indexed);
+ Object[] indexedRow = new Object[] {Integer.valueOf(1)};
+ Object[] indexedCopy = valueLimits.copyRow(indexedMeta, indexedRow);
+ assertNull(indexedCopy[0]);
+ assertEquals(Integer.valueOf(1), indexedRow[0]);
+ }
+
+ @Test
+ void aKeptTimestampIsClonedWhenAnotherCellIsOmitted() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ rowMeta.addValueMeta(new ValueMetaTimestamp("when"));
+ Timestamp when = new Timestamp(1_700_000_000_000L);
+ when.setNanos(123_456);
+ Object[] row = new Object[] {"this is longer than five", when};
+
+ Object[] copy = limits("5", null, null, null).copyRow(rowMeta, row);
+
+ assertNotSame(when, copy[1]);
+ assertEquals(when, copy[1]);
+ assertEquals(when.getNanos(), ((Timestamp) copy[1]).getNanos());
+ }
+
+ @Test
+ void nonNumericLimitIsUnlimited() throws Exception {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ String longText = "still stored";
+ Object[] row = new Object[] {longText};
+
+ Object[] copy = limits("many", null, null, null).copyRow(rowMeta, row);
+ Object[] negative = limits("-3", null, null, null).copyRow(rowMeta, row);
+
+ assertEquals(longText, copy[0]);
+ assertEquals(longText, negative[0]);
+ }
+
+ @Test
+ void variableLimitIsResolvedOnce() throws Exception {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit("${MAX}");
+ Variables variables = new Variables();
+ variables.setVariable("MAX", "0");
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+
+ Object[] copy =
+ SampledValueLimits.from(profile, variables).copyRow(rowMeta, new Object[] {"kept?"});
+
+ assertEquals(SampledValueLimits.NOT_STORED, copy[0]);
+ }
+
+ @Test
+ void profileCopyKeepsTheLimitFields() {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit("10");
+ profile.setJsonValueLimit("0");
+ profile.setBinaryValueLimit("20");
+ profile.setAvroValueLimit("30");
+
+ ExecutionDataProfile copy = new ExecutionDataProfile(profile);
+
+ assertEquals("10", copy.getStringValueLimit());
+ assertEquals("0", copy.getJsonValueLimit());
+ assertEquals("20", copy.getBinaryValueLimit());
+ assertEquals("30", copy.getAvroValueLimit());
+ }
+
+ @Test
+ void limitsRoundTripThroughMetadata() throws Exception {
+ ExecutionDataProfile profile = new ExecutionDataProfile("limits");
+ profile.setStringValueLimit("10");
+ profile.setJsonValueLimit("0");
+ profile.setBinaryValueLimit("20");
+ profile.setAvroValueLimit("${SAMPLE_AVRO_LIMIT}");
+ IHopMetadataProvider provider = new MemoryMetadataProvider();
+ provider.getSerializer(ExecutionDataProfile.class).save(profile);
+
+ ExecutionDataProfile loaded = provider.getSerializer(ExecutionDataProfile.class).load("limits");
+
+ assertEquals("10", loaded.getStringValueLimit());
+ assertEquals("0", loaded.getJsonValueLimit());
+ assertEquals("20", loaded.getBinaryValueLimit());
+ assertEquals("${SAMPLE_AVRO_LIMIT}", loaded.getAvroValueLimit());
+ }
+
+ @Test
+ void profileDeclaresTheLimitWidgets() throws Exception {
+ assertNotNull(ExecutionDataProfile.class.getAnnotation(GuiPlugin.class));
+ assertLimitWidget("stringValueLimit");
+ assertLimitWidget("jsonValueLimit");
+ assertLimitWidget("binaryValueLimit");
+ assertLimitWidget("avroValueLimit");
+ }
+
+ private static void assertLimitWidget(String fieldName) throws Exception {
+ Field field = ExecutionDataProfile.class.getDeclaredField(fieldName);
+ GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+ assertNotNull(element, fieldName);
+ assertEquals(fieldName, element.id());
+ assertEquals(GuiElementType.TEXT, element.type());
+ assertEquals(ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID, element.parentId());
+ assertEquals(GuiWidgetGroupType.BOXES, element.groupType());
+ assertEquals("i18n::ExecutionDataProfile.Group.LargeValues", element.group());
+ assertTrue(element.label().startsWith("i18n::"));
+ assertTrue(element.toolTip().startsWith("i18n::"));
+ }
+
+ @Test
+ void jacksonDoesNotWriteTheRuntimeLimits() throws Exception {
+ FirstRowsExecutionDataSampler sampler = new FirstRowsExecutionDataSampler("10");
+ sampler.setSampledValueLimits(limits("0", "0", "0", "0"));
+
+ String json = HopJson.newMapper().writeValueAsString(sampler);
+
+ assertFalse(json.contains("sampledValueLimits"));
+ assertFalse(json.contains("stringLimit"));
+ }
+
+ @Test
+ void profilingKeepsLengthStatsWithoutTheLongString() throws Exception {
+ BasicDataProfilingDataSampler sampler = new BasicDataProfilingDataSampler();
+ sampler.setSampledValueLimits(limits("5", null, null, null));
+ ExecutionDataSamplerMeta samplerMeta =
+ new ExecutionDataSamplerMeta("transform", "0", "log", false, true);
+ BasicDataProfilingDataSamplerStore store = sampler.createSamplerStore(samplerMeta);
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("name"));
+ rowMeta.addValueMeta(new ValueMetaString("body"));
+ store.init(new Variables(), rowMeta, rowMeta);
+
+ String longText = "this is long";
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, new Object[] {"a", "short"});
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, new Object[] {"b", longText});
+
+ assertEquals(2L, store.getNonNullCounters().get("body"));
+ assertEquals(longText.length(), store.getMaxLengths().get("body"));
+ assertEquals("short", store.getMaxValues().get("body"));
+ assertFalse(store.getMaxValues().containsValue(longText));
+
+ RowBuffer lengthRows =
+ store
+ .getProfileSamples()
+ .get("body")
+ .get(BasicDataProfilingDataSampler.ProfilingType.MaxLength);
+ assertEquals(1, lengthRows.size());
+ Object[] example = lengthRows.getBuffer().get(0);
+ assertEquals("b", example[0]);
+ assertEquals(SampledValueLimits.notStored(longText.length(), "characters"), example[1]);
+ }
+
+ @Test
+ void firstRowsReplaceJsonAndKeepTheOtherColumns() throws Exception {
+ FirstRowsExecutionDataSampler sampler = new FirstRowsExecutionDataSampler("10");
+ sampler.setSampledValueLimits(limits(null, "0", null, null));
+ ExecutionDataSamplerMeta samplerMeta =
+ new ExecutionDataSamplerMeta("transform", "0", "log", false, true);
+ FirstRowsExecutionDataSamplerStore store = sampler.createSamplerStore(samplerMeta);
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("id"));
+ rowMeta.addValueMeta(new ValueMetaJson("payload"));
+ store.init(new Variables(), rowMeta, rowMeta);
+ JsonNode payload = json("{\"n\":1}");
+ Object[] row = new Object[] {"row-1", payload};
+
+ sampler.sampleRow(store, IStream.StreamType.OUTPUT, rowMeta, row);
+
+ Object[] stored = store.getRows().get(0);
+ assertNotSame(row, stored);
+ assertEquals("row-1", stored[0]);
+ JsonNode marker = assertInstanceOf(JsonNode.class, stored[1]);
+ assertEquals(SampledValueLimits.NOT_STORED, marker.asText());
+ assertSame(payload, row[1]);
+ }
+
+ private static SampledValueLimits limits(
+ String stringLimit, String jsonLimit, String binaryLimit, String avroLimit) {
+ ExecutionDataProfile profile = new ExecutionDataProfile("profile");
+ profile.setStringValueLimit(stringLimit);
+ profile.setJsonValueLimit(jsonLimit);
+ profile.setBinaryValueLimit(binaryLimit);
+ profile.setAvroValueLimit(avroLimit);
+ return SampledValueLimits.from(profile, new Variables());
+ }
+
+ private static IRowMeta sampleRowMeta() {
+ IRowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("text"));
+ rowMeta.addValueMeta(new ValueMetaJson("json"));
+ rowMeta.addValueMeta(new ValueMetaBinary("binary"));
+ rowMeta.addValueMeta(new ValueMetaAvroRecord("avro"));
+ return rowMeta;
+ }
+
+ private static JsonNode json(String text) throws Exception {
+ return JsonUtil.jsonMapper().readTree(text);
+ }
+
+ private static GenericRecord avroRecord(String value) {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+ "{\"type\":\"record\",\"name\":\"R\",\"fields\":[{\"name\":\"a\",\"type\":\"string\"}]}");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("a", value);
+ return record;
+ }
+
+ private static GenericRecord wideSchemaRecord() {
+ Schema schema =
+ new Schema.Parser()
+ .parse(
+ "{\"type\":\"record\",\"name\":\"Wide\",\"doc\":\""
+ + "x".repeat(400)
+ + "\",\"fields\":[{\"name\":\"a\",\"type\":\"string\"}]}");
+ GenericRecord record = new GenericData.Record(schema);
+ record.put("a", "hi");
+ return record;
+ }
+}
diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
index a7990fdc3f..2f49235912 100644
--- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
+++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/core/transform/TransformBaseFn.java
@@ -166,8 +166,9 @@ public abstract class TransformBaseFn extends DoFn<HopRow, HopRow> {
// If not there's nothing we have to do in this transform really
//
String profileName = runConf.getExecutionDataProfileName();
+ ExecutionDataProfile dataProfile = null;
if (StringUtils.isNotEmpty(profileName)) {
- ExecutionDataProfile dataProfile =
+ dataProfile =
metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
if (dataProfile != null) {
dataSamplers.addAll(dataProfile.getSamplers());
@@ -180,6 +181,9 @@ public abstract class TransformBaseFn extends DoFn<HopRow, HopRow> {
HopJson.newMapper().readValue(dataSamplersJson, IExecutionDataSampler[].class);
dataSamplers.addAll(Arrays.asList(extraSamplers));
}
+ if (dataProfile != null) {
+ dataProfile.applyLimits(dataSamplers, variables);
+ }
executionInfoLocation = location;
diff --git a/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
index 3c6a2c717c..f9064adebd 100644
--- a/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
+++ b/plugins/engines/spark/src/main/java/org/apache/hop/spark/execution/SparkTransformExecutionSampling.java
@@ -168,9 +168,9 @@ public class SparkTransformExecutionSampling {
}
String profileName = runConf.getExecutionDataProfileName();
+ ExecutionDataProfile dataProfile = null;
if (StringUtils.isNotEmpty(profileName)) {
- ExecutionDataProfile dataProfile =
- metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
+ dataProfile = metadataProvider.getSerializer(ExecutionDataProfile.class).load(profileName);
if (dataProfile != null && dataProfile.getSamplers() != null) {
dataSamplers.addAll(dataProfile.getSamplers());
} else {
@@ -201,6 +201,10 @@ public class SparkTransformExecutionSampling {
}
}
+ if (dataProfile != null) {
+ dataProfile.applyLimits(dataSamplers, variables);
+ }
+
if (dataSamplers.isEmpty()) {
LogChannel.GENERAL.logBasic(
"Spark sampling inactive for '"
diff --git a/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java b/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
index b1782a6fdd..2ac60109e2 100644
--- a/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
+++ b/ui/src/main/java/org/apache/hop/ui/execution/profiling/ExecutionDataProfileEditor.java
@@ -44,6 +44,7 @@ import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Group;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.Text;
@@ -66,8 +67,10 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
private Text wDescription;
private org.eclipse.swt.widgets.List wSamplers;
+ private Composite editorComposite;
private Composite wPluginSpecificComp;
private GuiCompositeWidgets guiCompositeWidgets;
+ private GuiCompositeWidgets limitsWidgets;
private Map<String, IExecutionDataSampler> metaMap;
@@ -86,6 +89,7 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
@Override
public void createControl(Composite parent) {
+ this.editorComposite = parent;
PropsUi props = PropsUi.getInstance();
// Create a tabbed interface instead of the confusing left-hand side options
@@ -135,6 +139,18 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
wDescription.setLayoutData(fdDescription);
lastControl = wDescription;
+ limitsWidgets = new GuiCompositeWidgets(manager.getVariables());
+ limitsWidgets.createCompositeWidgets(
+ workingProfile,
+ null,
+ parent,
+ ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID,
+ wDescription);
+ Control limitsGroup = findGroup(parent);
+ if (limitsGroup != null) {
+ lastControl = limitsGroup;
+ }
+
Label wlSamplers = new Label(parent, SWT.LEFT);
PropsUi.setLook(wlSamplers);
wlSamplers.setText(BaseMessages.getString(PKG, "ExecutionDataProfileEditor.label.Samplers"));
@@ -202,6 +218,25 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
wName.addListener(SWT.Modify, modifyListener);
wDescription.addListener(SWT.Modify, modifyListener);
wSamplers.addListener(SWT.Selection, e -> addSamplerPluginWidgets());
+ limitsWidgets.setWidgetsListener(
+ new GuiCompositeWidgetsAdapter() {
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) {
+ setChanged();
+ }
+ });
+ }
+
+ /** The limits group is created on the editor composite, so the sampler list can sit under it. */
+ private static Control findGroup(Composite parent) {
+ Control group = null;
+ for (Control child : parent.getChildren()) {
+ if (child instanceof Group) {
+ group = child;
+ }
+ }
+ return group;
}
private IExecutionDataSampler previousSampler;
@@ -271,6 +306,8 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
wName.setText(Const.NVL(workingProfile.getName(), ""));
wDescription.setText(Const.NVL(workingProfile.getDescription(), ""));
+ limitsWidgets.setWidgetsContents(
+ workingProfile, editorComposite, ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID);
// Add all the sampler plugins in the profile
//
@@ -294,6 +331,7 @@ public class ExecutionDataProfileEditor extends MetadataEditor<ExecutionDataProf
profile.setName(wName.getText());
profile.setDescription(wDescription.getText());
+ limitsWidgets.getWidgetsContents(profile, ExecutionDataProfile.GUI_PLUGIN_LIMITS_PARENT_ID);
savePreviousSampler();