You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2026/04/10 07:21:18 UTC
(streampipes) branch 2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line updated: fix(#2338): enforce raw Unix timestamp validation for file replay adapter
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch 2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line by this push:
new 262bda700f fix(#2338): enforce raw Unix timestamp validation for file replay adapter
262bda700f is described below
commit 262bda700f68d8b639bcfbf16633bd0438435532
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Fri Apr 10 09:21:10 2026 +0200
fix(#2338): enforce raw Unix timestamp validation for file replay adapter
---
.../iiot/protocol/stream/FileReplayAdapter.java | 147 +++++----------------
.../protocol/stream/FileReplayAdapterTest.java | 32 ++++-
2 files changed, 64 insertions(+), 115 deletions(-)
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
index b9cc860138..ce0c70c77b 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
@@ -31,11 +31,7 @@ import org.apache.streampipes.extensions.management.connect.adapter.parser.CsvPa
import org.apache.streampipes.extensions.management.connect.adapter.parser.ImageParser;
import org.apache.streampipes.extensions.management.connect.adapter.parser.JsonParsers;
import org.apache.streampipes.extensions.management.connect.adapter.parser.xml.XmlParser;
-import org.apache.streampipes.model.connect.adapter.AdapterDescription;
import org.apache.streampipes.model.connect.guess.SampleData;
-import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription;
-import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription;
-import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
import org.apache.streampipes.model.extensions.ExtensionAssetType;
import org.apache.streampipes.sdk.StaticProperties;
import org.apache.streampipes.sdk.builder.adapter.AdapterConfigurationBuilder;
@@ -51,7 +47,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -78,13 +73,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
private ScheduledExecutorService executor;
private boolean replaceTimestamp;
private String timestampRuntimeName;
-
- private TimestampTranfsformationRuleDescription timestampTranfsformationRuleDescription;
-
- private String timestampSourceFieldName;
-
private float speedUp;
-
private long timestampLastEvent = -1;
@Override
@@ -132,32 +121,13 @@ public class FileReplayAdapter implements StreamPipesAdapter {
IEventCollector collector,
IAdapterRuntimeContext adapterRuntimeContext
) throws AdapterException {
-
- throwExceptionWhenAddTimestampRuleIsSelected(extractor);
-
boolean replayOnce = extractUserInputsAndReturnValueOfReplayOnce(extractor);
-
determineTimestampRuntimeName(extractor);
-
- determineSourceTimestampField(extractor);
+ validateTimestampFieldInInputEvent(extractor);
startAdapterReplayThread(extractor, collector, adapterRuntimeContext, replayOnce);
}
- protected static void throwExceptionWhenAddTimestampRuleIsSelected(IAdapterParameterExtractor extractor)
- throws AdapterException {
- boolean ruleExists = extractor.getAdapterDescription()
- .getRules()
- .stream()
- .anyMatch(rule -> rule instanceof AddTimestampRuleDescription);
-
- if (ruleExists) {
- throw new AdapterException("The file replay adapter requires a valid timestamp within the file. The add "
- + "timestamp option in the schema editor is not supported. Please edit the "
- + "adater to resolve this problem.");
- }
- }
-
private void startAdapterReplayThread(
IAdapterParameterExtractor extractor,
IEventCollector collector,
@@ -185,13 +155,16 @@ public class FileReplayAdapter implements StreamPipesAdapter {
.getStaticPropertyExtractor()
.selectedSingleValue(REPLAY_ONCE, String.class)
.equals("yes");
+
var replaceTimestampStringList = extractor
.getStaticPropertyExtractor()
.selectedMultiValues(REPLACE_TIMESTAMP, String.class);
replaceTimestamp = !replaceTimestampStringList.isEmpty();
+
var speedUpAlternative = extractor
.getStaticPropertyExtractor()
.selectedAlternativeInternalId(SPEED);
+
speedUp = switch (speedUpAlternative) {
case FASTEST -> Float.MAX_VALUE;
case SPEED_UP_FACTOR -> extractor
@@ -212,57 +185,37 @@ public class FileReplayAdapter implements StreamPipesAdapter {
.findFirst();
if (timestampField.isEmpty()) {
- throw new AdapterException("Could not find a timestamp field in event schema");
+ throw new AdapterException("Could not find a timestamp field in event schema. "
+ + "The file replay adapter requires a Unix timestamp to be present in the "
+ + "original input data.");
} else {
timestampRuntimeName = timestampField.get()
.getRuntimeName();
}
}
- /**
- * Determines the source field name for the timestamp property based
- * on any renaming rules configured within the adapter.
- * It ensures that the correct field name is used when extracting the timestamp from the raw data.
- *
- * @param extractor An instance of {@code IAdapterParameterExtractor} used for extracting adapter parameters.
- * @throws AdapterException If there are multiple renaming rules detected affecting the timestamp property,
- * indicating an invalid configuration.
- */
- private void determineSourceTimestampField(IAdapterParameterExtractor extractor) throws AdapterException {
- var renamingRulesTimestamp = getRenamingRulesTimestamp(extractor);
- if (!renamingRulesTimestamp.isEmpty()) {
- if (renamingRulesTimestamp.size() == 1) {
- timestampSourceFieldName = renamingRulesTimestamp.get(0)
- .getOldRuntimeKey();
- } else {
- throw new AdapterException("Invalid configuration - multiple renaming rules detected which affect the "
- + "timestamp property.");
- }
- } else {
- // no renaming rules can be found, timestamp name does not change
- timestampSourceFieldName = timestampRuntimeName;
+ protected void validateTimestampFieldInInputEvent(IAdapterParameterExtractor extractor)
+ throws AdapterException {
+ var inputEvents = extractor.getAdapterDescription()
+ .getTransformationConfig()
+ .getInputs();
+
+ if (inputEvents == null || inputEvents.isEmpty()) {
+ throw new AdapterException("Could not validate timestamp field in original input event. "
+ + "No sample input event is available. The file replay adapter requires a Unix "
+ + "timestamp to be present in the original input data.");
}
- }
- /**
- * Retrieves a list of {@link RenameRuleDescription} affecting the timestamp property.
- *
- * @param extractor An instance of IAdapterParameterExtractor used to extract adapter description and rules.
- * @return A list of {@link RenameRuleDescription} containing rules affecting timestamp property.
- */
- private List<RenameRuleDescription> getRenamingRulesTimestamp(IAdapterParameterExtractor extractor) {
- var renamingRules =
- extractor.getAdapterDescription()
- .getRules()
- .stream()
- .filter(rule -> rule instanceof RenameRuleDescription)
- .map(rule -> (RenameRuleDescription) rule)
- .toList();
-
- return renamingRules.stream()
- .filter(rule -> rule.getNewRuntimeKey()
- .equals(timestampRuntimeName))
- .toList();
+ var timestampFieldValue = inputEvents.get(0).get(timestampRuntimeName);
+
+ if (!(timestampFieldValue instanceof Number)) {
+ throw new AdapterException("The timestamp field in the original input event must be numeric. "
+ + "The file replay adapter requires a Unix timestamp to be present in the "
+ + "original input data. Field: %s, value: %s".formatted(
+ timestampRuntimeName,
+ timestampFieldValue
+ ));
+ }
}
private void getFileFromEndpointAndParseFile(
@@ -293,8 +246,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
InputStream inputStream,
IAdapterRuntimeContext adapterRuntimeContext
) {
-
- // The parse method does not throw AdapterExceptions, that's why the logging is handeled within the catch blog here
+ // The parse method does not throw AdapterExceptions, so event-level errors are logged in the callback.
parser.parse(inputStream, (event) -> {
try {
processEvent(collector, event);
@@ -312,7 +264,6 @@ public class FileReplayAdapter implements StreamPipesAdapter {
IEventCollector collector,
Map<String, Object> event
) throws AdapterException, InterruptedException {
-
long actualEventTimestamp = getTimestampFromEvent(event);
reduceReplaySpeedIfRequired(actualEventTimestamp);
@@ -327,8 +278,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
protected long getTimestampFromEvent(Map<String, Object> event) throws AdapterException {
long actualEventTimestamp = -1;
-
- var timestampFieldValue = event.get(timestampSourceFieldName);
+ var timestampFieldValue = event.get(timestampRuntimeName);
if (timestampFieldValue instanceof Long) {
actualEventTimestamp = (Long) timestampFieldValue;
@@ -338,7 +288,8 @@ public class FileReplayAdapter implements StreamPipesAdapter {
if (actualEventTimestamp == -1 && !replaceTimestamp) {
throw new AdapterException("Timestamp field could not be parsed, skipping event. "
- + "Value: %s".formatted(event.get(timestampSourceFieldName)));
+ + "The file replay adapter requires a Unix timestamp to be present in the "
+ + "original input data. Value: %s".formatted(event.get(timestampRuntimeName)));
}
return actualEventTimestamp;
@@ -359,7 +310,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
private void replaceTimestampIfRequired(Map<String, Object> event) {
if (replaceTimestamp) {
- event.put(timestampSourceFieldName, System.currentTimeMillis());
+ event.put(timestampRuntimeName, System.currentTimeMillis());
}
}
@@ -390,42 +341,12 @@ public class FileReplayAdapter implements StreamPipesAdapter {
}
}
- protected void setTimestampSourceFieldName(String timestampSourceFieldName) {
- this.timestampSourceFieldName = timestampSourceFieldName;
+ protected void setTimestampRuntimeName(String timestampRuntimeName) {
+ this.timestampRuntimeName = timestampRuntimeName;
}
protected void setReplaceTimestamp(boolean replaceTimestamp) {
this.replaceTimestamp = replaceTimestamp;
}
- protected void setTimestampTranfsformationRuleDescription(
- TimestampTranfsformationRuleDescription timestampTranfsformationRuleDescription
- ) {
- this.timestampTranfsformationRuleDescription = timestampTranfsformationRuleDescription;
- }
-
- /**
- * Removes the timestamp transformation rules from the adapter description.
- *
- * <p>The FileReplay adapter manages timestamp transformations internally to accurately simulate the replay frequency.
- * This is necessary as the timestamp field values are crucial for this simulation. As a result, the timestamp rule
- * description is stored locally within the FileReplay adapter and is applied when the onAdapterStarted method is
- * invoked.</p>
- */
- @Override
- public void preprocessAdapterDescription(AdapterDescription adapterDescription) {
-
- this.timestampTranfsformationRuleDescription = adapterDescription
- .getRules()
- .stream()
- .filter(rule -> rule instanceof TimestampTranfsformationRuleDescription)
- .map(rule -> (TimestampTranfsformationRuleDescription) rule)
- .findFirst()
- .orElse(null);
-
- // remove timestamp preprocessing rule
- adapterDescription
- .getRules()
- .removeIf(rule -> rule instanceof TimestampTranfsformationRuleDescription);
- }
}
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
index 0147e392bf..a11884dd26 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
@@ -22,11 +22,13 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException;
import org.apache.streampipes.extensions.api.connect.IEventCollector;
import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.sdk.helpers.EpProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -51,10 +53,11 @@ class FileReplayAdapterTest {
void setUp() {
collector = mock(IEventCollector.class);
extractor = mock(IAdapterParameterExtractor.class);
- adapterDescription = mock(AdapterDescription.class);
+ adapterDescription = new AdapterDescription();
+ adapterDescription.getEventSchema().addEventProperty(EpProperties.timestampProperty(TIMESTAMP));
when(extractor.getAdapterDescription()).thenReturn(adapterDescription);
fileReplayAdapter = new FileReplayAdapter();
- fileReplayAdapter.setTimestampSourceFieldName(TIMESTAMP);
+ fileReplayAdapter.setTimestampRuntimeName(TIMESTAMP);
event = new HashMap<>();
}
@@ -102,4 +105,29 @@ class FileReplayAdapterTest {
assertEquals(TIMESTAMP_VALUE, actualEventTimestamp);
}
+ @Test
+ void validateTimestampFieldInInputEvent_shouldAcceptLongTimestampInOriginalInput() throws AdapterException {
+ adapterDescription.getTransformationConfig().setInputs(List.of(Map.of(TIMESTAMP, TIMESTAMP_VALUE)));
+
+ fileReplayAdapter.validateTimestampFieldInInputEvent(extractor);
+ }
+
+ @Test
+ void validateTimestampFieldInInputEvent_shouldThrowForStringTimestampInOriginalInput() {
+ adapterDescription.getTransformationConfig().setInputs(List.of(Map.of(TIMESTAMP, "2021-12-24T12:55:12.123+01:00")));
+
+ assertThrows(AdapterException.class, () -> fileReplayAdapter.validateTimestampFieldInInputEvent(extractor));
+ }
+
+ @Test
+ void processEvent_shouldReplaceTimestampWhenConfigured() throws AdapterException, InterruptedException {
+ event.put(TIMESTAMP, TIMESTAMP_VALUE);
+ fileReplayAdapter.setReplaceTimestamp(true);
+
+ fileReplayAdapter.processEvent(collector, event);
+
+ verify(collector, times(1)).collect(event);
+ assertEquals(Long.class, event.get(TIMESTAMP).getClass());
+ }
+
}