You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2026/05/29 11:04:27 UTC
(streampipes) branch add-ignore-timestamp-to-wincc-filewatcher created (now 4f339c37aa)
This is an automated email from the ASF dual-hosted git repository.
dominikriemer pushed a change to branch add-ignore-timestamp-to-wincc-filewatcher
in repository https://gitbox.apache.org/repos/asf/streampipes.git
at 4f339c37aa feat: Add option to ignore last modified timestamp in alarm archive adapter
This branch includes the following new commits:
new 4f339c37aa feat: Add option to ignore last modified timestamp in alarm archive adapter
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(streampipes) 01/01: feat: Add option to ignore last modified timestamp in alarm archive adapter
Posted by ri...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dominikriemer pushed a commit to branch add-ignore-timestamp-to-wincc-filewatcher
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 4f339c37aa66b63ec1b1fd02610d351c6f63fa75
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Fri May 29 13:04:11 2026 +0200
feat: Add option to ignore last modified timestamp in alarm archive adapter
---
.../FileWatcherConnectorsModuleExport.java | 3 +-
.../adapter/WinCCAlarmArchiveAdapter.java | 8 ++-
.../WinCCAlarmArchiveAdapterMigrationV2.java | 58 ++++++++++++++++++++++
.../filewatcher/model/FileWatcherConfig.java | 1 +
.../filewatcher/runtime/FileSetWatcher.java | 31 ++++++++----
.../documentation.md | 5 ++
.../strings.en | 2 +
.../adapter/WinCCAlarmArchiveAdapterTest.java | 3 ++
.../WinCCAlarmArchiveAdapterMigrationV2Test.java} | 37 +++++++-------
.../WinCCAlarmArchiveAdapterVersionedConfig.java | 49 ++++++++++++++++++
.../filewatcher/runtime/FileSetWatcherTest.java | 34 +++++++++++--
11 files changed, 196 insertions(+), 35 deletions(-)
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java
index 44511e8c63..33e72d6ae5 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.extensions.api.migration.IModelMigrator;
import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
import org.apache.streampipes.extensions.connectors.filewatcher.adapter.WinCCAlarmArchiveAdapter;
import org.apache.streampipes.extensions.connectors.filewatcher.migration.WinCCAlarmArchiveAdapterMigrationV1;
+import org.apache.streampipes.extensions.connectors.filewatcher.migration.WinCCAlarmArchiveAdapterMigrationV2;
import java.util.List;
@@ -41,6 +42,6 @@ public class FileWatcherConnectorsModuleExport implements IExtensionModuleExport
@Override
public List<IModelMigrator<?, ?>> migrators() {
- return List.of(new WinCCAlarmArchiveAdapterMigrationV1());
+ return List.of(new WinCCAlarmArchiveAdapterMigrationV1(), new WinCCAlarmArchiveAdapterMigrationV2());
}
}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapter.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapter.java
index b6510c021d..376709a667 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapter.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapter.java
@@ -66,6 +66,7 @@ public class WinCCAlarmArchiveAdapter implements StreamPipesAdapter, IPullAdapte
private static final String SEGMENTED_CIRCULAR_LOG_ENABLED = "segmented-circular-log-enabled";
private static final String ARCHIVE_SEGMENT_COUNT = "archive-segment-count";
public static final String ARCHIVE_SEGMENT_START_INDEX = "archive-segment-start-index";
+ public static final String CONSIDER_LAST_MODIFIED = "consider-last-modified";
private static final String POLL_INTERVAL_SECONDS = "poll-interval-seconds";
private static final String INTER_EVENT_DELAY_MS = "inter-event-delay-ms";
private static final String TIMEZONE_ID = "timezone-id";
@@ -83,7 +84,7 @@ public class WinCCAlarmArchiveAdapter implements StreamPipesAdapter, IPullAdapte
@Override
public IAdapterConfiguration declareConfig() {
return AdapterConfigurationBuilder
- .create(ID, 1, WinCCAlarmArchiveAdapter::new)
+ .create(ID, 2, WinCCAlarmArchiveAdapter::new)
.withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
.withLocales(Locales.EN)
.requiredTextParameter(Labels.withId(
@@ -107,6 +108,9 @@ public class WinCCAlarmArchiveAdapter implements StreamPipesAdapter, IPullAdapte
.requiredIntegerParameter(Labels.withId(
ARCHIVE_SEGMENT_START_INDEX
), 0)
+ .requiredSlideToggle(Labels.withId(
+ CONSIDER_LAST_MODIFIED
+ ), true)
.requiredIntegerParameter(Labels.withId(
POLL_INTERVAL_SECONDS
))
@@ -205,6 +209,7 @@ public class WinCCAlarmArchiveAdapter implements StreamPipesAdapter, IPullAdapte
staticPropertyExtractor.selectedSingleValueInternalName(SEGMENTED_CIRCULAR_LOG_ENABLED, String.class);
var segmentCount = staticPropertyExtractor.singleValueParameter(ARCHIVE_SEGMENT_COUNT, Integer.class);
var segmentStartIndex = staticPropertyExtractor.singleValueParameter(ARCHIVE_SEGMENT_START_INDEX, Integer.class);
+ var considerLastModified = staticPropertyExtractor.slideToggleValue(CONSIDER_LAST_MODIFIED);
var interval = staticPropertyExtractor.singleValueParameter(POLL_INTERVAL_SECONDS, Integer.class);
var interEventDelayMs = staticPropertyExtractor.singleValueParameter(INTER_EVENT_DELAY_MS, Integer.class);
var timeZone = parseTimeZone(staticPropertyExtractor.singleValueParameter(TIMEZONE_ID, String.class));
@@ -225,6 +230,7 @@ public class WinCCAlarmArchiveAdapter implements StreamPipesAdapter, IPullAdapte
WINCC_CSV_SETTINGS,
interval,
SEGMENTED_CIRCULAR_LOG_OFF.equals(segmentedCircularLogMode),
+ considerLastModified,
interEventDelayMs,
timeZone
);
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2.java
new file mode 100644
index 0000000000..fcc575c8dc
--- /dev/null
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.connectors.filewatcher.migration;
+
+import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor;
+import org.apache.streampipes.extensions.api.migration.IAdapterMigrator;
+import org.apache.streampipes.extensions.connectors.filewatcher.adapter.WinCCAlarmArchiveAdapter;
+import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.extensions.svcdiscovery.SpServiceTagPrefix;
+import org.apache.streampipes.model.migration.MigrationResult;
+import org.apache.streampipes.model.migration.ModelMigratorConfig;
+import org.apache.streampipes.model.staticproperty.SlideToggleStaticProperty;
+import org.apache.streampipes.sdk.helpers.Labels;
+
+public class WinCCAlarmArchiveAdapterMigrationV2 implements IAdapterMigrator {
+
+ @Override
+ public ModelMigratorConfig config() {
+ return new ModelMigratorConfig(
+ WinCCAlarmArchiveAdapter.ID,
+ SpServiceTagPrefix.ADAPTER,
+ 1,
+ 2
+ );
+ }
+
+ @Override
+ public MigrationResult<AdapterDescription> migrate(AdapterDescription element,
+ IStaticPropertyExtractor extractor) throws RuntimeException {
+ var label = Labels.withId(WinCCAlarmArchiveAdapter.CONSIDER_LAST_MODIFIED);
+ var staticProperty = new SlideToggleStaticProperty(
+ label.getInternalId(),
+ label.getLabel(),
+ label.getDescription(),
+ true
+ );
+ staticProperty.setSelected(true);
+ element.getConfig().add(5, staticProperty);
+
+ return MigrationResult.success(element);
+ }
+}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/model/FileWatcherConfig.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/model/FileWatcherConfig.java
index f37cfb04ee..68930799be 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/model/FileWatcherConfig.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/model/FileWatcherConfig.java
@@ -27,6 +27,7 @@ public record FileWatcherConfig(Path directory,
CsvParserSettings parserSettings,
int pollIntervalSeconds,
boolean singleFileGrowthMode,
+ boolean considerLastModified,
int interEventDelayMs,
ZoneId timeZone) {
}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcher.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcher.java
index 549285de10..503f7891ea 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcher.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcher.java
@@ -48,12 +48,6 @@ public class FileSetWatcher {
private final EventMapper eventMapper;
private final EventDelayExecutor eventDelayExecutor;
- public FileSetWatcher(FileWatcherConfig config,
- FileWatcherCheckpointStore checkpointStore,
- CsvFileReader csvFileReader) {
- this(config, checkpointStore, csvFileReader, EventMapper.identity());
- }
-
public FileSetWatcher(FileWatcherConfig config,
FileWatcherCheckpointStore checkpointStore,
CsvFileReader csvFileReader,
@@ -122,7 +116,7 @@ public class FileSetWatcher {
}
private long determineStartRecord(FileSlot currentSlot, FileWatcherCheckpoint checkpoint) {
- if (currentSlot.fingerprint().equals(checkpoint.getCurrentFingerprint())) {
+ if (fingerprintsEqual(currentSlot.fingerprint(), checkpoint.getCurrentFingerprint())) {
LOG.debug("File '{}' fingerprint unchanged. Continuing after record {}.",
currentSlot.fileName(), checkpoint.getLastProcessedRecord());
return checkpoint.getLastProcessedRecord() + 1;
@@ -140,7 +134,26 @@ public class FileSetWatcher {
private boolean isAppendedInPlace(FileFingerprint currentFingerprint, FileFingerprint previousFingerprint) {
return currentFingerprint.getSize() > previousFingerprint.getSize()
- && currentFingerprint.getLastModified() >= previousFingerprint.getLastModified();
+ && (!config.considerLastModified()
+ || currentFingerprint.getLastModified() >= previousFingerprint.getLastModified());
+ }
+
+ private boolean fingerprintsEqual(FileFingerprint left, FileFingerprint right) {
+ if (left == right) {
+ return true;
+ }
+ if (left == null || right == null) {
+ return false;
+ }
+
+ boolean sameContent = left.getSize() == right.getSize()
+ && left.getContentHash().equals(right.getContentHash());
+
+ if (!sameContent) {
+ return false;
+ }
+
+ return !config.considerLastModified() || left.getLastModified() == right.getLastModified();
}
private void processFollowingFiles(String adapterElementId,
@@ -152,7 +165,7 @@ public class FileSetWatcher {
for (int offset = 0; offset < files.size(); offset++) {
FileSlot candidate = files.get((startIndex + offset) % files.size());
FileFingerprint processedFingerprint = checkpoint.getProcessedGenerations().get(candidate.fileName());
- if (processedFingerprint != null && processedFingerprint.equals(candidate.fingerprint())) {
+ if (processedFingerprint != null && fingerprintsEqual(processedFingerprint, candidate.fingerprint())) {
LOG.debug("Stopping at file '{}' because this generation was already processed.", candidate.fileName());
break;
}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/documentation.md b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/documentation.md
index b83b548e03..4eab893cf4 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/documentation.md
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/documentation.md
@@ -30,6 +30,7 @@ Configuration is Siemens-specific:
- segmented circular log enabled or disabled
- segment count in segmented circular log mode
- segment start index
+- whether the last modified timestamp should be considered for deduplication
- polling interval
- optional delay between replayed events in milliseconds
- timezone used when WinCC `TimeString` must be converted to Unix time
@@ -42,6 +43,10 @@ segment start index.
In segmented circular log mode the adapter treats reused file names as new generations by fingerprinting file content,
so a segment like `Meldungsarchiv1.csv` can be consumed again after WinCC rotates the archive.
+If another process periodically recopies the same archive files into the watched folder, you can disable the last
+modified timestamp check so unchanged content is not treated as a new generation only because the file timestamp
+changed.
+
Use the inter-event delay when consumers or brokers cannot handle large replay bursts. A value like `1` ms can reduce
event loss on slower consumers, while `0` disables throttling.
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/strings.en b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/strings.en
index 116d327a73..f18811909c 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/strings.en
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/main/resources/org.apache.streampipes.connectors.wincc.alarm.archive/strings.en
@@ -24,6 +24,8 @@ archive-segment-count.title=Segment count
archive-segment-count.description=Number of archive segments in segmented circular log mode
archive-segment-start-index.title=Segment start index
archive-segment-start-index.description=First segment index used in the WinCC file names, for example 0 or 1
+consider-last-modified.title=Consider last modified timestamp
+consider-last-modified.description=If enabled, file timestamps are included when deciding whether an alarm archive file was already processed
poll-interval-seconds.title=Polling interval (seconds)
poll-interval-seconds.description=Interval used to scan the alarm archive files for new entries
inter-event-delay-ms.title=Inter-event delay (ms)
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapterTest.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapterTest.java
index 23fc0f5568..556638fcdf 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapterTest.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/adapter/WinCCAlarmArchiveAdapterTest.java
@@ -79,6 +79,9 @@ class WinCCAlarmArchiveAdapterTest {
if ("selectedSingleValueInternalName".equals(method.getName())) {
return segmentedCircularLogMode;
}
+ if ("slideToggleValue".equals(method.getName())) {
+ return true;
+ }
throw new UnsupportedOperationException("Unexpected extractor method: " + method.getName());
}
);
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2Test.java
similarity index 50%
copy from streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java
copy to streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2Test.java
index 44511e8c63..05ae144e54 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/main/java/org/apache/streampipes/extensions/connectors/filewatcher/FileWatcherConnectorsModuleExport.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/WinCCAlarmArchiveAdapterMigrationV2Test.java
@@ -16,31 +16,30 @@
*
*/
-package org.apache.streampipes.extensions.connectors.filewatcher;
+package org.apache.streampipes.extensions.connectors.filewatcher.migration;
-import org.apache.streampipes.extensions.api.connect.StreamPipesAdapter;
-import org.apache.streampipes.extensions.api.declarer.IExtensionModuleExport;
-import org.apache.streampipes.extensions.api.migration.IModelMigrator;
-import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement;
+import org.apache.streampipes.extensions.api.extractor.IStaticPropertyExtractor;
import org.apache.streampipes.extensions.connectors.filewatcher.adapter.WinCCAlarmArchiveAdapter;
-import org.apache.streampipes.extensions.connectors.filewatcher.migration.WinCCAlarmArchiveAdapterMigrationV1;
+import org.apache.streampipes.extensions.connectors.filewatcher.migration.config.WinCCAlarmArchiveAdapterVersionedConfig;
+import org.apache.streampipes.model.staticproperty.SlideToggleStaticProperty;
-import java.util.List;
+import org.junit.jupiter.api.Test;
-public class FileWatcherConnectorsModuleExport implements IExtensionModuleExport {
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
- @Override
- public List<StreamPipesAdapter> adapters() {
- return List.of(new WinCCAlarmArchiveAdapter());
- }
+class WinCCAlarmArchiveAdapterMigrationV2Test {
- @Override
- public List<IStreamPipesPipelineElement<?>> pipelineElements() {
- return List.of();
- }
+ @Test
+ void shouldInsertConsiderLastModifiedWithDefaultValue() {
+ var migration = new WinCCAlarmArchiveAdapterMigrationV2();
+ var adapterDescription = WinCCAlarmArchiveAdapterVersionedConfig.getWinCCAlarmArchiveAdapterDescriptionV1();
+
+ var migrated = migration.migrate(adapterDescription, (IStaticPropertyExtractor) null).element();
- @Override
- public List<IModelMigrator<?, ?>> migrators() {
- return List.of(new WinCCAlarmArchiveAdapterMigrationV1());
+ assertEquals(9, migrated.getConfig().size());
+ var property = (SlideToggleStaticProperty) migrated.getConfig().get(5);
+ assertEquals(WinCCAlarmArchiveAdapter.CONSIDER_LAST_MODIFIED, property.getInternalName());
+ assertTrue(property.isSelected());
}
}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/config/WinCCAlarmArchiveAdapterVersionedConfig.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/config/WinCCAlarmArchiveAdapterVersionedConfig.java
index 31fda484a8..e0c1a1baeb 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/config/WinCCAlarmArchiveAdapterVersionedConfig.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/migration/config/WinCCAlarmArchiveAdapterVersionedConfig.java
@@ -33,6 +33,8 @@ public class WinCCAlarmArchiveAdapterVersionedConfig {
private static final String ARCHIVE_BASE_NAME = "archive-base-name";
private static final String SEGMENTED_CIRCULAR_LOG_ENABLED = "segmented-circular-log-enabled";
private static final String ARCHIVE_SEGMENT_COUNT = "archive-segment-count";
+ private static final String ARCHIVE_SEGMENT_START_INDEX = "archive-segment-start-index";
+ private static final String CONSIDER_LAST_MODIFIED = "consider-last-modified";
private static final String POLL_INTERVAL_SECONDS = "poll-interval-seconds";
private static final String INTER_EVENT_DELAY_MS = "inter-event-delay-ms";
private static final String TIMEZONE_ID = "timezone-id";
@@ -60,4 +62,51 @@ public class WinCCAlarmArchiveAdapterVersionedConfig {
.buildConfiguration()
.getAdapterDescription();
}
+
+ public static AdapterDescription getWinCCAlarmArchiveAdapterDescriptionV1() {
+ return AdapterConfigurationBuilder
+ .create(WinCCAlarmArchiveAdapter.ID, 1, WinCCAlarmArchiveAdapter::new)
+ .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
+ .withLocales(Locales.EN)
+ .requiredTextParameter(Labels.withId(DIRECTORY_PATH))
+ .requiredTextParameter(Labels.withId(ARCHIVE_BASE_NAME))
+ .requiredSingleValueSelection(
+ Labels.withId(SEGMENTED_CIRCULAR_LOG_ENABLED),
+ Options.from(
+ new Tuple2<>("Enabled", SEGMENTED_CIRCULAR_LOG_ON),
+ new Tuple2<>("Disabled", SEGMENTED_CIRCULAR_LOG_OFF)
+ )
+ )
+ .requiredIntegerParameter(Labels.withId(ARCHIVE_SEGMENT_COUNT))
+ .requiredIntegerParameter(Labels.withId(ARCHIVE_SEGMENT_START_INDEX), 0)
+ .requiredIntegerParameter(Labels.withId(POLL_INTERVAL_SECONDS))
+ .requiredIntegerParameter(Labels.withId(INTER_EVENT_DELAY_MS), 0)
+ .requiredTextParameter(Labels.withId(TIMEZONE_ID), "UTC")
+ .buildConfiguration()
+ .getAdapterDescription();
+ }
+
+ public static AdapterDescription getWinCCAlarmArchiveAdapterDescriptionV2() {
+ return AdapterConfigurationBuilder
+ .create(WinCCAlarmArchiveAdapter.ID, 2, WinCCAlarmArchiveAdapter::new)
+ .withAssets(ExtensionAssetType.DOCUMENTATION, ExtensionAssetType.ICON)
+ .withLocales(Locales.EN)
+ .requiredTextParameter(Labels.withId(DIRECTORY_PATH))
+ .requiredTextParameter(Labels.withId(ARCHIVE_BASE_NAME))
+ .requiredSingleValueSelection(
+ Labels.withId(SEGMENTED_CIRCULAR_LOG_ENABLED),
+ Options.from(
+ new Tuple2<>("Enabled", SEGMENTED_CIRCULAR_LOG_ON),
+ new Tuple2<>("Disabled", SEGMENTED_CIRCULAR_LOG_OFF)
+ )
+ )
+ .requiredIntegerParameter(Labels.withId(ARCHIVE_SEGMENT_COUNT))
+ .requiredIntegerParameter(Labels.withId(ARCHIVE_SEGMENT_START_INDEX), 0)
+ .requiredSlideToggle(Labels.withId(CONSIDER_LAST_MODIFIED), true)
+ .requiredIntegerParameter(Labels.withId(POLL_INTERVAL_SECONDS))
+ .requiredIntegerParameter(Labels.withId(INTER_EVENT_DELAY_MS), 0)
+ .requiredTextParameter(Labels.withId(TIMEZONE_ID), "UTC")
+ .buildConfiguration()
+ .getAdapterDescription();
+ }
}
diff --git a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcherTest.java b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcherTest.java
index 433c01a066..628455483f 100644
--- a/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcherTest.java
+++ b/streampipes-extensions/streampipes-connectors-filewatcher/src/test/java/org/apache/streampipes/extensions/connectors/filewatcher/runtime/FileSetWatcherTest.java
@@ -48,7 +48,7 @@ class FileSetWatcherTest {
write("Meldungsarchiv2", "id,value\n3,c\n");
FileSetWatcher watcher = new FileSetWatcher(
- new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, 0, ZoneId.of("UTC")),
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, true, 0, ZoneId.of("UTC")),
new FileWatcherCheckpointStore(tempDir.resolve("checkpoints")),
new CsvFileReader(),
EventMapper.identity()
@@ -80,7 +80,7 @@ class FileSetWatcherTest {
FileWatcherCheckpointStore checkpointStore = new FileWatcherCheckpointStore(checkpointDir);
FileSetWatcher watcher = new FileSetWatcher(
- new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, 0, ZoneId.of("UTC")),
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, true, 0, ZoneId.of("UTC")),
checkpointStore,
new CsvFileReader(),
EventMapper.identity()
@@ -107,7 +107,7 @@ class FileSetWatcherTest {
write("Meldungsarchiv.csv", "id,value\n1,a\n2,b\n");
FileSetWatcher watcher = new FileSetWatcher(
- new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv\\.csv"), new CsvParserSettings(true, ','), 1, true, 0, ZoneId.of("UTC")),
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv\\.csv"), new CsvParserSettings(true, ','), 1, true, true, 0, ZoneId.of("UTC")),
new FileWatcherCheckpointStore(tempDir.resolve("checkpoints-single")),
new CsvFileReader(),
EventMapper.identity()
@@ -132,7 +132,7 @@ class FileSetWatcherTest {
write("Meldungsarchiv.csv", "id,value\n1,a\n2,b\n3,c\n");
FileSetWatcher watcher = new FileSetWatcher(
- new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv\\.csv"), new CsvParserSettings(true, ','), 1, true, 0, ZoneId.of("UTC")),
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv\\.csv"), new CsvParserSettings(true, ','), 1, true, true, 0, ZoneId.of("UTC")),
new FileWatcherCheckpointStore(tempDir.resolve("checkpoints-replace")),
new CsvFileReader(),
EventMapper.identity()
@@ -157,7 +157,7 @@ class FileSetWatcherTest {
List<Long> appliedDelays = new ArrayList<>();
FileSetWatcher watcher = new FileSetWatcher(
- new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, 5, ZoneId.of("UTC")),
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, true, 5, ZoneId.of("UTC")),
new FileWatcherCheckpointStore(tempDir.resolve("checkpoints-delay")),
new CsvFileReader(),
EventMapper.identity(),
@@ -175,6 +175,30 @@ class FileSetWatcherTest {
assertEquals(List.of(5L, 5L), appliedDelays);
}
+ @Test
+ void shouldIgnoreRecopiedIdenticalFileWhenLastModifiedIsDisabled() throws IOException, InterruptedException {
+ write("Meldungsarchiv1", "id,value\n1,a\n2,b\n");
+
+ FileSetWatcher watcher = new FileSetWatcher(
+ new FileWatcherConfig(tempDir, Pattern.compile("Meldungsarchiv(\\d+)"), new CsvParserSettings(true, ','), 1, false, false, 0, ZoneId.of("UTC")),
+ new FileWatcherCheckpointStore(tempDir.resolve("checkpoints-recopy")),
+ new CsvFileReader(),
+ EventMapper.identity()
+ );
+
+ List<Map<String, Object>> events = new ArrayList<>();
+ watcher.poll("adapter-recopy", events::add);
+
+ assertEquals(2, events.size());
+ events.clear();
+
+ Thread.sleep(5);
+ write("Meldungsarchiv1", "id,value\n1,a\n2,b\n");
+ watcher.poll("adapter-recopy", events::add);
+
+ assertEquals(0, events.size());
+ }
+
private void write(String fileName, String content) throws IOException {
Files.writeString(tempDir.resolve(fileName), content);
}