You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2026/04/10 07:21:18 UTC

(streampipes) branch 2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line updated: fix(#2338): enforce raw Unix timestamp validation for file replay adapter

This is an automated email from the ASF dual-hosted git repository.

zehnder pushed a commit to branch 2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line
in repository https://gitbox.apache.org/repos/asf/streampipes.git


The following commit(s) were added to refs/heads/2338-dealing-with-files-in-the-file-stream-adapter-that-dont-have-a-timestamp-on-every-line by this push:
     new 262bda700f fix(#2338): enforce raw Unix timestamp validation for file replay adapter
262bda700f is described below

commit 262bda700f68d8b639bcfbf16633bd0438435532
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Fri Apr 10 09:21:10 2026 +0200

    fix(#2338): enforce raw Unix timestamp validation for file replay adapter
---
 .../iiot/protocol/stream/FileReplayAdapter.java    | 147 +++++----------------
 .../protocol/stream/FileReplayAdapterTest.java     |  32 ++++-
 2 files changed, 64 insertions(+), 115 deletions(-)

diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
index b9cc860138..ce0c70c77b 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
@@ -31,11 +31,7 @@ import org.apache.streampipes.extensions.management.connect.adapter.parser.CsvPa
 import org.apache.streampipes.extensions.management.connect.adapter.parser.ImageParser;
 import org.apache.streampipes.extensions.management.connect.adapter.parser.JsonParsers;
 import org.apache.streampipes.extensions.management.connect.adapter.parser.xml.XmlParser;
-import org.apache.streampipes.model.connect.adapter.AdapterDescription;
 import org.apache.streampipes.model.connect.guess.SampleData;
-import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription;
-import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription;
-import org.apache.streampipes.model.connect.rules.value.TimestampTranfsformationRuleDescription;
 import org.apache.streampipes.model.extensions.ExtensionAssetType;
 import org.apache.streampipes.sdk.StaticProperties;
 import org.apache.streampipes.sdk.builder.adapter.AdapterConfigurationBuilder;
@@ -51,7 +47,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -78,13 +73,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
   private ScheduledExecutorService executor;
   private boolean replaceTimestamp;
   private String timestampRuntimeName;
-
-  private TimestampTranfsformationRuleDescription timestampTranfsformationRuleDescription;
-
-  private String timestampSourceFieldName;
-
   private float speedUp;
-
   private long timestampLastEvent = -1;
 
   @Override
@@ -132,32 +121,13 @@ public class FileReplayAdapter implements StreamPipesAdapter {
       IEventCollector collector,
       IAdapterRuntimeContext adapterRuntimeContext
   ) throws AdapterException {
-
-    throwExceptionWhenAddTimestampRuleIsSelected(extractor);
-
     boolean replayOnce = extractUserInputsAndReturnValueOfReplayOnce(extractor);
-
     determineTimestampRuntimeName(extractor);
-
-    determineSourceTimestampField(extractor);
+    validateTimestampFieldInInputEvent(extractor);
 
     startAdapterReplayThread(extractor, collector, adapterRuntimeContext, replayOnce);
   }
 
-  protected static void throwExceptionWhenAddTimestampRuleIsSelected(IAdapterParameterExtractor extractor)
-      throws AdapterException {
-    boolean ruleExists = extractor.getAdapterDescription()
-                                  .getRules()
-                                  .stream()
-                                  .anyMatch(rule -> rule instanceof AddTimestampRuleDescription);
-
-    if (ruleExists) {
-      throw new AdapterException("The file replay adapter requires a valid timestamp within the file. The add "
-                                     + "timestamp option in the schema editor is not supported. Please edit the "
-                                     + "adater to resolve this problem.");
-    }
-  }
-
   private void startAdapterReplayThread(
       IAdapterParameterExtractor extractor,
       IEventCollector collector,
@@ -185,13 +155,16 @@ public class FileReplayAdapter implements StreamPipesAdapter {
         .getStaticPropertyExtractor()
         .selectedSingleValue(REPLAY_ONCE, String.class)
         .equals("yes");
+
     var replaceTimestampStringList = extractor
         .getStaticPropertyExtractor()
         .selectedMultiValues(REPLACE_TIMESTAMP, String.class);
     replaceTimestamp = !replaceTimestampStringList.isEmpty();
+
     var speedUpAlternative = extractor
         .getStaticPropertyExtractor()
         .selectedAlternativeInternalId(SPEED);
+
     speedUp = switch (speedUpAlternative) {
       case FASTEST -> Float.MAX_VALUE;
       case SPEED_UP_FACTOR -> extractor
@@ -212,57 +185,37 @@ public class FileReplayAdapter implements StreamPipesAdapter {
         .findFirst();
 
     if (timestampField.isEmpty()) {
-      throw new AdapterException("Could not find a timestamp field in event schema");
+      throw new AdapterException("Could not find a timestamp field in event schema. "
+                                     + "The file replay adapter requires a Unix timestamp to be present in the "
+                                     + "original input data.");
     } else {
       timestampRuntimeName = timestampField.get()
                                            .getRuntimeName();
     }
   }
 
-  /**
-   * Determines the source field name for the timestamp property based
-   * on any renaming rules configured within the adapter.
-   * It ensures that the correct field name is used when extracting the timestamp from the raw data.
-   *
-   * @param extractor An instance of {@code IAdapterParameterExtractor} used for extracting adapter parameters.
-   * @throws AdapterException If there are multiple renaming rules detected affecting the timestamp property,
-   *                          indicating an invalid configuration.
-   */
-  private void determineSourceTimestampField(IAdapterParameterExtractor extractor) throws AdapterException {
-    var renamingRulesTimestamp = getRenamingRulesTimestamp(extractor);
-    if (!renamingRulesTimestamp.isEmpty()) {
-      if (renamingRulesTimestamp.size() == 1) {
-        timestampSourceFieldName = renamingRulesTimestamp.get(0)
-                                                         .getOldRuntimeKey();
-      } else {
-        throw new AdapterException("Invalid configuration - multiple renaming rules detected which affect the "
-                                       + "timestamp property.");
-      }
-    } else {
-      // no renaming rules can be found, timestamp name does not change
-      timestampSourceFieldName = timestampRuntimeName;
+  protected void validateTimestampFieldInInputEvent(IAdapterParameterExtractor extractor)
+      throws AdapterException {
+    var inputEvents = extractor.getAdapterDescription()
+                               .getTransformationConfig()
+                               .getInputs();
+
+    if (inputEvents == null || inputEvents.isEmpty()) {
+      throw new AdapterException("Could not validate timestamp field in original input event. "
+                                     + "No sample input event is available. The file replay adapter requires a Unix "
+                                     + "timestamp to be present in the original input data.");
     }
-  }
 
-  /**
-   * Retrieves a list of {@link RenameRuleDescription} affecting the timestamp property.
-   *
-   * @param extractor An instance of IAdapterParameterExtractor used to extract adapter description and rules.
-   * @return A list of {@link RenameRuleDescription} containing rules affecting timestamp property.
-   */
-  private List<RenameRuleDescription> getRenamingRulesTimestamp(IAdapterParameterExtractor extractor) {
-    var renamingRules =
-        extractor.getAdapterDescription()
-                 .getRules()
-                 .stream()
-                 .filter(rule -> rule instanceof RenameRuleDescription)
-                 .map(rule -> (RenameRuleDescription) rule)
-                 .toList();
-
-    return renamingRules.stream()
-                        .filter(rule -> rule.getNewRuntimeKey()
-                                            .equals(timestampRuntimeName))
-                        .toList();
+    var timestampFieldValue = inputEvents.get(0).get(timestampRuntimeName);
+
+    if (!(timestampFieldValue instanceof Number)) {
+      throw new AdapterException("The timestamp field in the original input event must be numeric. "
+                                     + "The file replay adapter requires a Unix timestamp to be present in the "
+                                     + "original input data. Field: %s, value: %s".formatted(
+          timestampRuntimeName,
+          timestampFieldValue
+      ));
+    }
   }
 
   private void getFileFromEndpointAndParseFile(
@@ -293,8 +246,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
       InputStream inputStream,
       IAdapterRuntimeContext adapterRuntimeContext
   ) {
-
-    // The parse method does not throw AdapterExceptions, that's why the logging is handeled within the catch blog here
+    // The parse method does not throw AdapterExceptions, so event-level errors are logged in the callback.
     parser.parse(inputStream, (event) -> {
       try {
         processEvent(collector, event);
@@ -312,7 +264,6 @@ public class FileReplayAdapter implements StreamPipesAdapter {
       IEventCollector collector,
       Map<String, Object> event
   ) throws AdapterException, InterruptedException {
-
     long actualEventTimestamp = getTimestampFromEvent(event);
 
     reduceReplaySpeedIfRequired(actualEventTimestamp);
@@ -327,8 +278,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
 
   protected long getTimestampFromEvent(Map<String, Object> event) throws AdapterException {
     long actualEventTimestamp = -1;
-
-    var timestampFieldValue = event.get(timestampSourceFieldName);
+    var timestampFieldValue = event.get(timestampRuntimeName);
 
     if (timestampFieldValue instanceof Long) {
       actualEventTimestamp = (Long) timestampFieldValue;
@@ -338,7 +288,8 @@ public class FileReplayAdapter implements StreamPipesAdapter {
 
     if (actualEventTimestamp == -1 && !replaceTimestamp) {
       throw new AdapterException("Timestamp field could not be parsed, skipping event. "
-                                     + "Value: %s".formatted(event.get(timestampSourceFieldName)));
+                                     + "The file replay adapter requires a Unix timestamp to be present in the "
+                                     + "original input data. Value: %s".formatted(event.get(timestampRuntimeName)));
     }
 
     return actualEventTimestamp;
@@ -359,7 +310,7 @@ public class FileReplayAdapter implements StreamPipesAdapter {
 
   private void replaceTimestampIfRequired(Map<String, Object> event) {
     if (replaceTimestamp) {
-      event.put(timestampSourceFieldName, System.currentTimeMillis());
+      event.put(timestampRuntimeName, System.currentTimeMillis());
     }
   }
 
@@ -390,42 +341,12 @@ public class FileReplayAdapter implements StreamPipesAdapter {
     }
   }
 
-  protected void setTimestampSourceFieldName(String timestampSourceFieldName) {
-    this.timestampSourceFieldName = timestampSourceFieldName;
+  protected void setTimestampRuntimeName(String timestampRuntimeName) {
+    this.timestampRuntimeName = timestampRuntimeName;
   }
 
   protected void setReplaceTimestamp(boolean replaceTimestamp) {
     this.replaceTimestamp = replaceTimestamp;
   }
 
-  protected void setTimestampTranfsformationRuleDescription(
-      TimestampTranfsformationRuleDescription timestampTranfsformationRuleDescription
-  ) {
-    this.timestampTranfsformationRuleDescription = timestampTranfsformationRuleDescription;
-  }
-
-  /**
-   * Removes the timestamp transformation rules from the adapter description.
-   *
-   * <p>The FileReplay adapter manages timestamp transformations internally to accurately simulate the replay frequency.
-   * This is necessary as the timestamp field values are crucial for this simulation. As a result, the timestamp rule
-   * description is stored locally within the FileReplay adapter and is applied when the onAdapterStarted method is
-   * invoked.</p>
-   */
-  @Override
-  public void preprocessAdapterDescription(AdapterDescription adapterDescription) {
-
-    this.timestampTranfsformationRuleDescription = adapterDescription
-        .getRules()
-        .stream()
-        .filter(rule -> rule instanceof TimestampTranfsformationRuleDescription)
-        .map(rule -> (TimestampTranfsformationRuleDescription) rule)
-        .findFirst()
-        .orElse(null);
-
-    // remove timestamp preprocessing rule
-    adapterDescription
-        .getRules()
-        .removeIf(rule -> rule instanceof TimestampTranfsformationRuleDescription);
-  }
 }
diff --git a/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java b/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
index 0147e392bf..a11884dd26 100644
--- a/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
+++ b/streampipes-extensions/streampipes-connect-adapters-iiot/src/test/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapterTest.java
@@ -22,11 +22,13 @@ import org.apache.streampipes.commons.exceptions.connect.AdapterException;
 import org.apache.streampipes.extensions.api.connect.IEventCollector;
 import org.apache.streampipes.extensions.api.extractor.IAdapterParameterExtractor;
 import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.sdk.helpers.EpProperties;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -51,10 +53,11 @@ class FileReplayAdapterTest {
   void setUp() {
     collector = mock(IEventCollector.class);
     extractor = mock(IAdapterParameterExtractor.class);
-    adapterDescription = mock(AdapterDescription.class);
+    adapterDescription = new AdapterDescription();
+    adapterDescription.getEventSchema().addEventProperty(EpProperties.timestampProperty(TIMESTAMP));
     when(extractor.getAdapterDescription()).thenReturn(adapterDescription);
     fileReplayAdapter = new FileReplayAdapter();
-    fileReplayAdapter.setTimestampSourceFieldName(TIMESTAMP);
+    fileReplayAdapter.setTimestampRuntimeName(TIMESTAMP);
     event = new HashMap<>();
   }
 
@@ -102,4 +105,29 @@ class FileReplayAdapterTest {
     assertEquals(TIMESTAMP_VALUE, actualEventTimestamp);
   }
 
+  @Test
+  void validateTimestampFieldInInputEvent_shouldAcceptLongTimestampInOriginalInput() throws AdapterException {
+    adapterDescription.getTransformationConfig().setInputs(List.of(Map.of(TIMESTAMP, TIMESTAMP_VALUE)));
+
+    fileReplayAdapter.validateTimestampFieldInInputEvent(extractor);
+  }
+
+  @Test
+  void validateTimestampFieldInInputEvent_shouldThrowForStringTimestampInOriginalInput() {
+    adapterDescription.getTransformationConfig().setInputs(List.of(Map.of(TIMESTAMP, "2021-12-24T12:55:12.123+01:00")));
+
+    assertThrows(AdapterException.class, () -> fileReplayAdapter.validateTimestampFieldInInputEvent(extractor));
+  }
+
+  @Test
+  void processEvent_shouldReplaceTimestampWhenConfigured() throws AdapterException, InterruptedException {
+    event.put(TIMESTAMP, TIMESTAMP_VALUE);
+    fileReplayAdapter.setReplaceTimestamp(true);
+
+    fileReplayAdapter.processEvent(collector, event);
+
+    verify(collector, times(1)).collect(event);
+    assertEquals(Long.class, event.get(TIMESTAMP).getClass());
+  }
+
 }