You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@plc4x.apache.org by jf...@apache.org on 2018/11/24 23:30:41 UTC
[incubator-plc4x] branch develop updated (be1ea77 -> 1c2a5e4)
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git.
from be1ea77 - Fixed a compilation error as InternalPlcConsumerRegistration was not extending InternalPlcMessage
new 92604c0 [plc4j-scraper] Added configuration with yml / json support.
new 1c2a5e4 [plc4j-scraper] Added complete example.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
plc4j/utils/scraper/pom.xml | 23 ++-
.../org/apache/plc4x/java/scraper/ScrapeJob.java} | 50 +++--
.../org/apache/plc4x/java/scraper/Scraper.java | 97 ++++------
.../org/apache/plc4x/java/scraper/ScraperTask.java | 11 +-
.../java/scraper/config/JobConfiguration.java | 65 +++++++
.../java/scraper/config/ScraperConfiguration.java | 119 ++++++++++++
.../scraper/util/PercentageAboveThreshold.java | 58 ++++++
.../java/scraper/ScraperConfigurationTest.java | 202 +++++++++++++++++++++
.../apache/plc4x/java/scraper/ScraperRunner.java} | 33 ++--
.../apache/plc4x/java/scraper/ScraperTaskTest.java | 10 +-
.../org/apache/plc4x/java/scraper/ScraperTest.java | 18 +-
plc4j/utils/scraper/src/test/resources/config.json | 18 ++
plc4j/utils/scraper/src/test/resources/config.yml | 10 +
plc4j/utils/scraper/src/test/resources/example.yml | 25 +++
.../resources/{logback-test.xml => logback.xml} | 0
pom.xml | 1 +
16 files changed, 614 insertions(+), 126 deletions(-)
copy plc4j/utils/{opm/src/main/java/org/apache/plc4x/java/opm/AliasRegistry.java => scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java} (50%)
create mode 100644 plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java
create mode 100644 plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java
create mode 100644 plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java
create mode 100644 plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java
copy plc4j/{protocols/driver-bases/test/src/main/java/org/apache/plc4x/java/mock/MockField.java => utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java} (57%)
create mode 100644 plc4j/utils/scraper/src/test/resources/config.json
create mode 100644 plc4j/utils/scraper/src/test/resources/config.yml
create mode 100644 plc4j/utils/scraper/src/test/resources/example.yml
copy plc4j/utils/scraper/src/test/resources/{logback-test.xml => logback.xml} (100%)
[incubator-plc4x] 02/02: [plc4j-scraper] Added complete example.
Posted by jf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git
commit 1c2a5e4690977ace9f1660393cfdfa3c3c6da6a1
Author: Julian Feinauer <j....@pragmaticminds.de>
AuthorDate: Sun Nov 25 00:25:55 2018 +0100
[plc4j-scraper] Added complete example.
---
.../org/apache/plc4x/java/scraper/Scraper.java | 23 ++++++++++---
.../org/apache/plc4x/java/scraper/ScraperTask.java | 11 +++----
.../apache/plc4x/java/scraper/ScraperRunner.java | 38 ++++++++++++++++++++++
.../apache/plc4x/java/scraper/ScraperTaskTest.java | 10 +++---
.../org/apache/plc4x/java/scraper/ScraperTest.java | 8 ++---
plc4j/utils/scraper/src/test/resources/example.yml | 25 ++++++++++++++
plc4j/utils/scraper/src/test/resources/logback.xml | 36 ++++++++++++++++++++
7 files changed, 130 insertions(+), 21 deletions(-)
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
index dfe86b0..856a73c 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
@@ -47,7 +47,7 @@ public class Scraper {
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(10,
new BasicThreadFactory.Builder()
.namingPattern("scheduler-thread-%d")
- .daemon(true)
+ .daemon(false)
.build()
);
private final ExecutorService handlerPool = Executors.newFixedThreadPool(4,
@@ -56,6 +56,9 @@ public class Scraper {
.daemon(true)
.build()
);
+
+ private final ResultHandler resultHandler;
+
private final MultiValuedMap<ScrapeJob, ScraperTask> tasks = new ArrayListValuedHashMap<>();
private final MultiValuedMap<ScraperTask, ScheduledFuture<?>> futures = new ArrayListValuedHashMap<>();
private final PlcDriverManager driverManager;
@@ -65,17 +68,20 @@ public class Scraper {
* Creates a Scraper instance from a configuration.
* By default a {@link PooledPlcDriverManager} is used.
* @param config Configuration to use.
+ * @param resultHandler
*/
- public Scraper(ScraperConfiguration config) {
- this(new PooledPlcDriverManager(), config.getJobs());
+ public Scraper(ScraperConfiguration config, ResultHandler resultHandler) {
+ this(resultHandler, new PooledPlcDriverManager(), config.getJobs());
}
/**
*
+ * @param resultHandler
* @param driverManager
* @param jobs
*/
- public Scraper(PlcDriverManager driverManager, List<ScrapeJob> jobs) {
+ public Scraper(ResultHandler resultHandler, PlcDriverManager driverManager, List<ScrapeJob> jobs) {
+ this.resultHandler = resultHandler;
Validate.notEmpty(jobs);
this.driverManager = driverManager;
this.jobs = jobs;
@@ -99,7 +105,7 @@ public class Scraper {
tuple.getLeft().getName(), tuple.getMiddle(), tuple.getRight(),
tuple.getLeft().getFields(),
1_000,
- handlerPool);
+ handlerPool, resultHandler);
// Add task to internal list
tasks.put(tuple.getLeft(), task);
ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(task,
@@ -143,4 +149,11 @@ public class Scraper {
futures.clear();
}
+ @FunctionalInterface
+ public interface ResultHandler {
+
+ void handle(Map<String, Object> results);
+
+ }
+
}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
index b762c7a..afbfe07 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
@@ -55,6 +55,7 @@ public class ScraperTask implements Runnable {
private final Map<String, String> fields;
private final long requestTimeoutMs;
private final ExecutorService handlerService;
+ private final Scraper.ResultHandler resultHandler;
private final AtomicLong requestCounter = new AtomicLong(0);
private final AtomicLong successCounter = new AtomicLong(0);
@@ -62,13 +63,14 @@ public class ScraperTask implements Runnable {
private final DescriptiveStatistics failedStatistics = new DescriptiveStatistics(1000);
public ScraperTask(PlcDriverManager driverManager, String jobName, String connectionAlias, String connectionString,
- Map<String, String> fields, long requestTimeoutMs, ExecutorService handlerService) {
+ Map<String, String> fields, long requestTimeoutMs, ExecutorService handlerService, Scraper.ResultHandler resultHandler) {
Validate.notNull(driverManager);
Validate.notBlank(jobName);
Validate.notBlank(connectionAlias);
Validate.notBlank(connectionString);
Validate.notEmpty(fields);
Validate.isTrue(requestTimeoutMs > 0);
+ Validate.notNull(resultHandler);
this.driverManager = driverManager;
this.jobName = jobName;
this.connectionAlias = connectionAlias;
@@ -76,6 +78,7 @@ public class ScraperTask implements Runnable {
this.fields = fields;
this.requestTimeoutMs = requestTimeoutMs;
this.handlerService = handlerService;
+ this.resultHandler = resultHandler;
}
@Override
@@ -120,7 +123,7 @@ public class ScraperTask implements Runnable {
// Validate response
validateResponse(response);
// Handle response (Async)
- CompletableFuture.runAsync(() -> handle(transformResponseToMap(response)), handlerService);
+ CompletableFuture.runAsync(() -> resultHandler.handle(transformResponseToMap(response)), handlerService);
} catch (Exception e) {
LOGGER.debug("Exception during scrape", e);
handleException(e);
@@ -179,10 +182,6 @@ public class ScraperTask implements Runnable {
return 100.0*failedStatistics.getMean();
}
- public void handle(Map<String, Object> result) {
- LOGGER.debug("Handling result on gorgeous pool: {}", result);
- }
-
public void handleException(Exception e) {
LOGGER.debug("Exception: ", e);
failedStatistics.addValue(1.0);
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
new file mode 100644
index 0000000..265abe6
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper;
+
+import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class ScraperRunner {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ScraperRunner.class);
+
+ public static void main(String[] args) throws IOException {
+ ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml");
+ Scraper scraper = new Scraper(configuration, m -> LOGGER.info("Results: {}", m));
+
+ scraper.start();
+ }
+}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTaskTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTaskTest.java
index 77df940..a83a939 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTaskTest.java
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTaskTest.java
@@ -22,7 +22,6 @@ package org.apache.plc4x.java.scraper;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.plc4x.java.PlcDriverManager;
import org.apache.plc4x.java.api.exceptions.PlcConnectionException;
-import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
import org.apache.plc4x.java.api.types.PlcResponseCode;
import org.apache.plc4x.java.base.messages.items.DefaultStringFieldItem;
import org.apache.plc4x.java.mock.MockDevice;
@@ -35,7 +34,6 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Collections;
-import java.util.Map;
import java.util.concurrent.*;
import static org.mockito.ArgumentMatchers.any;
@@ -56,7 +54,7 @@ public class ScraperTaskTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultStringFieldItem("hallo")));
ScraperTask scraperTask = new ScraperTask(driverManager, "job1", "m1", "mock:scraper", Collections.singletonMap("a", "b"),
- 1_000, ForkJoinPool.commonPool());
+ 1_000, ForkJoinPool.commonPool(), m -> {});
scraperTask.run();
}
@@ -73,7 +71,7 @@ public class ScraperTaskTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.NOT_FOUND, new DefaultStringFieldItem("hallo")));
ScraperTask scraperTask = new ScraperTask(driverManager, "job1", "m1",
- "mock:scraper", Collections.singletonMap("a", "b"), 1_000, ForkJoinPool.commonPool());
+ "mock:scraper", Collections.singletonMap("a", "b"), 1_000, ForkJoinPool.commonPool(), m -> {});
// When
scraperTask.run();
@@ -88,7 +86,7 @@ public class ScraperTaskTest implements WithAssertions {
when(driverManager.getConnection(anyString())).thenThrow(new PlcConnectionException("stfu"));
ScraperTask scraperTask = new ScraperTask(driverManager, "job1", "m1", "mock:scraper", Collections.singletonMap("a", "b"),
- 1_000, ForkJoinPool.commonPool());
+ 1_000, ForkJoinPool.commonPool(), m -> {});
ScraperTask spy = spy(scraperTask);
spy.run();
@@ -101,7 +99,7 @@ public class ScraperTaskTest implements WithAssertions {
when(driverManager.getConnection(anyString())).thenThrow(new PlcConnectionException("stfu"));
ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
ScraperTask scraperTask = new ScraperTask(driverManager, "job1", "m1", "mock:scraper", Collections.singletonMap("a", "b"),
- 1_000, ForkJoinPool.commonPool());
+ 1_000, ForkJoinPool.commonPool(), m -> {});
Future<?> future = pool.scheduleAtFixedRate(scraperTask, 0, 10, TimeUnit.MILLISECONDS);
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
index c025946..1822a20 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
@@ -71,7 +71,7 @@ class ScraperTest implements WithAssertions {
return new GenericKeyedObjectPool<>(pooledPlcConnectionFactory, config);
});
- Scraper scraper = new Scraper(driverManager, Arrays.asList(
+ Scraper scraper = new Scraper(m -> {}, driverManager, Arrays.asList(
new ScrapeJob("job1",
10,
Collections.singletonMap("tim", CONN_STRING_TIM),
@@ -95,7 +95,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
- Scraper scraper = new Scraper(driverManager, Collections.singletonList(
+ Scraper scraper = new Scraper(m -> {}, driverManager, Collections.singletonList(
new ScrapeJob("job1",
10,
Collections.singletonMap("m1", "mock:m1"),
@@ -119,7 +119,7 @@ class ScraperTest implements WithAssertions {
void stop_stopsAllJobs() {
PlcDriverManager driverManager = new PlcDriverManager();
- Scraper scraper = new Scraper(driverManager, Collections.singletonList(
+ Scraper scraper = new Scraper(m -> {}, driverManager, Collections.singletonList(
new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
@@ -146,7 +146,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
- Scraper scraper = new Scraper(driverManager, Collections.singletonList(
+ Scraper scraper = new Scraper(m -> {}, driverManager, Collections.singletonList(
new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
diff --git a/plc4j/utils/scraper/src/test/resources/example.yml b/plc4j/utils/scraper/src/test/resources/example.yml
new file mode 100644
index 0000000..b82d837
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/example.yml
@@ -0,0 +1,25 @@
+sources:
+ S7_TIM: s7://10.10.64.22/0/1
+ S7_CHRIS: s7://10.10.64.20/0/1
+
+jobs:
+ - name: pressure-job
+ scrapeRate: 10
+ sources:
+ - S7_TIM
+ fields:
+ pressure: '%DB225:DBW0:INT'
+
+ - name: another-job
+ scrapeRate: 100
+ sources:
+ - S7_TIM
+ fields:
+ temperature: '%DB225:DBW0:INT'
+
+ - name: counter-job
+ scrapeRate: 100
+ sources:
+ - S7_CHRIS
+ fields:
+ counter: '%DB3:DBD32:DINT'
\ No newline at end of file
diff --git a/plc4j/utils/scraper/src/test/resources/logback.xml b/plc4j/utils/scraper/src/test/resources/logback.xml
new file mode 100644
index 0000000..c562020
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/logback.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<configuration xmlns="http://ch.qos.logback/xml/ns/logback"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://ch.qos.logback/xml/ns/logback https://raw.githubusercontent.com/enricopulatzo/logback-XSD/master/src/main/xsd/logback.xsd">
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <!-- encoders are assigned the type
+ ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+</configuration>
\ No newline at end of file
[incubator-plc4x] 01/02: [plc4j-scraper] Added configuration with
yml / json support.
Posted by jf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git
commit 92604c0c7bc650b2449b37c74d292b1f85493baa
Author: Julian Feinauer <j....@pragmaticminds.de>
AuthorDate: Sat Nov 24 23:59:37 2018 +0100
[plc4j-scraper] Added configuration with yml / json support.
---
plc4j/utils/scraper/pom.xml | 23 ++-
.../org/apache/plc4x/java/scraper/ScrapeJob.java | 59 ++++++
.../org/apache/plc4x/java/scraper/Scraper.java | 84 +++------
.../java/scraper/config/JobConfiguration.java | 65 +++++++
.../java/scraper/config/ScraperConfiguration.java | 119 ++++++++++++
.../scraper/util/PercentageAboveThreshold.java | 58 ++++++
.../java/scraper/ScraperConfigurationTest.java | 202 +++++++++++++++++++++
.../org/apache/plc4x/java/scraper/ScraperTest.java | 10 +-
plc4j/utils/scraper/src/test/resources/config.json | 18 ++
plc4j/utils/scraper/src/test/resources/config.yml | 10 +
pom.xml | 1 +
11 files changed, 581 insertions(+), 68 deletions(-)
diff --git a/plc4j/utils/scraper/pom.xml b/plc4j/utils/scraper/pom.xml
index f32a624..c914b19 100644
--- a/plc4j/utils/scraper/pom.xml
+++ b/plc4j/utils/scraper/pom.xml
@@ -31,6 +31,27 @@
<artifactId>plc4j-scraper</artifactId>
<dependencies>
+ <!--Jackson-->
+ <!--<dependency>-->
+ <!--<groupId>com.fasterxml.jackson.core</groupId>-->
+ <!--<artifactId>jackson-core</artifactId>-->
+ <!--<version>${jackson.version}</version>-->
+ <!--</dependency>-->
+ <!--<dependency>-->
+ <!--<groupId>com.fasterxml.jackson.core</groupId>-->
+ <!--<artifactId>jackson-annotations</artifactId>-->
+ <!--<version>${jackson.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
@@ -57,12 +78,10 @@
<groupId>org.apache.plc4x</groupId>
<artifactId>plc4j-connection-pool</artifactId>
<version>0.3.0-SNAPSHOT</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.plc4x</groupId>
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java
new file mode 100644
index 0000000..296032b
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper;
+
+import java.util.Map;
+
+public class ScrapeJob {
+
+ private final String name;
+ private final long scrapeRate;
+ private final Map<String, String> connections;
+ private final Map<String, String> fields;
+
+ public ScrapeJob(String name, long scrapeRate, Map<String, String> connections, Map<String, String> fields) {
+ this.name = name;
+ this.scrapeRate = scrapeRate;
+ this.connections = connections;
+ this.fields = fields;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getScrapeRate() {
+ return scrapeRate;
+ }
+
+ /**
+ * alias -> connection-string
+ */
+ public Map<String, String> getConnections() {
+ return connections;
+ }
+
+ /**
+ * alias -> field-query
+ */
+ public Map<String, String> getFields() {
+ return fields;
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
index e1f9fc1..dfe86b0 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
@@ -24,19 +24,18 @@ import org.apache.commons.collections4.multimap.ArrayListValuedHashMap;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.commons.lang3.tuple.Triple;
-import org.apache.commons.math3.exception.MathIllegalArgumentException;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.commons.math3.stat.descriptive.UnivariateStatistic;
import org.apache.plc4x.java.PlcDriverManager;
+import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.apache.plc4x.java.scraper.util.PercentageAboveThreshold;
+import org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.*;
-import java.util.stream.IntStream;
/**
* Main class that orchestrates scraping.
@@ -62,6 +61,20 @@ public class Scraper {
private final PlcDriverManager driverManager;
private final List<ScrapeJob> jobs;
+ /**
+ * Creates a Scraper instance from a configuration.
+ * By default a {@link PooledPlcDriverManager} is used.
+ * @param config Configuration to use.
+ */
+ public Scraper(ScraperConfiguration config) {
+ this(new PooledPlcDriverManager(), config.getJobs());
+ }
+
+ /**
+ *
+ * @param driverManager
+ * @param jobs
+ */
public Scraper(PlcDriverManager driverManager, List<ScrapeJob> jobs) {
Validate.notEmpty(jobs);
this.driverManager = driverManager;
@@ -75,22 +88,22 @@ public class Scraper {
// Schedule all jobs
LOGGER.info("Starting jobs...");
jobs.stream()
- .flatMap(job -> job.connections.entrySet().stream()
+ .flatMap(job -> job.getConnections().entrySet().stream()
.map(entry -> Triple.of(job, entry.getKey(), entry.getValue()))
)
.forEach(
tuple -> {
LOGGER.debug("Register task for job {} for conn {} ({}) at rate {} ms",
- tuple.getLeft().name, tuple.getMiddle(), tuple.getRight(), tuple.getLeft().scrapeRate);
+ tuple.getLeft().getName(), tuple.getMiddle(), tuple.getRight(), tuple.getLeft().getScrapeRate());
ScraperTask task = new ScraperTask(driverManager,
- tuple.getLeft().name, tuple.getMiddle(), tuple.getRight(),
- tuple.getLeft().fields,
+ tuple.getLeft().getName(), tuple.getMiddle(), tuple.getRight(),
+ tuple.getLeft().getFields(),
1_000,
handlerPool);
// Add task to internal list
tasks.put(tuple.getLeft(), task);
ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(task,
- 0, tuple.getLeft().scrapeRate, TimeUnit.MILLISECONDS);
+ 0, tuple.getLeft().getScrapeRate(), TimeUnit.MILLISECONDS);
// Store the handle for stopping, etc.
futures.put(task, future);
@@ -102,7 +115,7 @@ public class Scraper {
for (Map.Entry<ScrapeJob, ScraperTask> entry : tasks.entries()) {
DescriptiveStatistics statistics = entry.getValue().getLatencyStatistics();
String msg = String.format(Locale.ENGLISH, "Job statistics (%s, %s) number of requests: %d (%d success, %.1f %% failed, %.1f %% too slow), mean latency: %.2f ms, median: %.2f ms",
- entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().scrapeRate * 1e6)), statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
+ entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().getScrapeRate() * 1e6)), statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
LOGGER.info(msg);
}
}, 1_000, 1_000, TimeUnit.MILLISECONDS);
@@ -130,55 +143,4 @@ public class Scraper {
futures.clear();
}
- public static class ScrapeJob {
-
- private final String name;
- private final long scrapeRate;
- /**
- * alias -> connection-string
- */
- private final Map<String, String> connections;
- /**
- * alias -> field-query
- */
- private final Map<String, String> fields;
-
- public ScrapeJob(String name, long scrapeRate, Map<String, String> connections, Map<String, String> fields) {
- this.name = name;
- this.scrapeRate = scrapeRate;
- this.connections = connections;
- this.fields = fields;
- }
- }
-
- private static class PercentageAboveThreshold implements UnivariateStatistic {
-
- private final double threshold;
-
- public PercentageAboveThreshold(double threshold) {
- this.threshold = threshold;
- }
-
- @Override
- public double evaluate(double[] values) throws MathIllegalArgumentException {
- long below = Arrays.stream(values)
- .filter(val -> val <= threshold)
- .count();
- return (double) below / values.length;
- }
-
- @Override
- public double evaluate(double[] values, int begin, int length) throws MathIllegalArgumentException {
- long below = IntStream.range(begin, length)
- .mapToDouble(i -> values[i])
- .filter(val -> val > threshold)
- .count();
- return 100.0 * below / length;
- }
-
- @Override
- public UnivariateStatistic copy() {
- return new PercentageAboveThreshold(threshold);
- }
- }
}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java
new file mode 100644
index 0000000..6cb2ede
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class JobConfiguration {
+
+ private final String name;
+ private final int scrapeRate;
+ private final List<String> sources;
+ private final Map<String, String> fields;
+
+ @JsonCreator
+ JobConfiguration(@JsonProperty(value = "name", required = true) String name,
+ @JsonProperty(value = "scrapeRate", required = true) int scrapeRate,
+ @JsonProperty(value = "sources", required = true) List<String> sources,
+ @JsonProperty(value = "fields", required = true) Map<String, String> fields) {
+ this.name = name;
+ this.scrapeRate = scrapeRate;
+ this.sources = sources;
+ this.fields = fields;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getScrapeRate() {
+ return scrapeRate;
+ }
+
+ public List<String> getSources() {
+ return sources;
+ }
+
+ public Map<String, String> getFields() {
+ return fields;
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java
new file mode 100644
index 0000000..48d688c
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+import org.apache.plc4x.java.scraper.ScrapeJob;
+import org.apache.plc4x.java.scraper.Scraper;
+
+import java.io.*;
+import java.security.InvalidParameterException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Configuration class for {@link Scraper}.
+ */
+
+public class ScraperConfiguration {
+
+ private final Map<String, String> sources;
+ private final List<JobConfiguration> jobConfigurations;
+
+ @JsonCreator
+ ScraperConfiguration(@JsonProperty(value = "sources", required = true) Map<String, String> sources,
+ @JsonProperty(value = "jobs", required = true) List<JobConfiguration> jobConfigurations) {
+ checkNoUnreferencedSources(sources, jobConfigurations);
+ // TODO Warning on too many sources?!
+ this.sources = sources;
+ this.jobConfigurations = jobConfigurations;
+ }
+
+ private void checkNoUnreferencedSources(Map<String, String> sources, List<JobConfiguration> jobConfigurations) {
+ Set<String> unreferencedSources = jobConfigurations.stream()
+ .flatMap(job -> job.getSources().stream())
+ .filter(source -> !sources.containsKey(source))
+ .collect(Collectors.toSet());
+ if (!unreferencedSources.isEmpty()) {
+ throw new PlcRuntimeException("There are the following unreferenced sources: " + unreferencedSources);
+ }
+ }
+
+ public static ScraperConfiguration fromYaml(String yaml) {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ try {
+ return mapper.readValue(yaml, ScraperConfiguration.class);
+ } catch (IOException e) {
+ throw new PlcRuntimeException("Unable to parse given yaml configuration!", e);
+ }
+ }
+
+ public static ScraperConfiguration fromJson(String json) {
+ ObjectMapper mapper = new ObjectMapper(new JsonFactory());
+ try {
+ return mapper.readValue(json, ScraperConfiguration.class);
+ } catch (IOException e) {
+ throw new PlcRuntimeException("Unable to parse given json configuration!", e);
+ }
+ }
+
+ public static ScraperConfiguration fromFile(String path) throws IOException {
+ ObjectMapper mapper;
+ if (path.endsWith("json")) {
+ mapper = new ObjectMapper(new JsonFactory());
+ } else if (path.endsWith("yml") || path.endsWith("yaml")) {
+ mapper = new ObjectMapper(new YAMLFactory());
+ } else {
+ throw new InvalidParameterException("Only files with extensions json, yml or yaml can be read");
+ }
+ return mapper.readValue(new File(path), ScraperConfiguration.class);
+ }
+
+ public Map<String, String> getSources() {
+ return sources;
+ }
+
+ public List<JobConfiguration> getJobConfigurations() {
+ return jobConfigurations;
+ }
+
+ public List<ScrapeJob> getJobs() {
+ return jobConfigurations.stream()
+ .map(conf -> new ScrapeJob(conf.getName(), conf.getScrapeRate(),
+ getSourcesForAliases(conf.getSources()), conf.getFields()))
+ .collect(Collectors.toList());
+ }
+
+ private Map<String, String> getSourcesForAliases(List<String> aliases) {
+ return aliases.stream()
+ .collect(Collectors.toMap(
+ Function.identity(),
+ sources::get
+ ));
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java
new file mode 100644
index 0000000..1cafea1
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.util;
+
+import org.apache.commons.math3.exception.MathIllegalArgumentException;
+import org.apache.commons.math3.stat.descriptive.UnivariateStatistic;
+import org.apache.plc4x.java.scraper.Scraper;
+
+import java.util.Arrays;
+import java.util.stream.IntStream;
+
+public class PercentageAboveThreshold implements UnivariateStatistic {
+
+ private final double threshold;
+
+ public PercentageAboveThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ @Override
+ public double evaluate(double[] values) throws MathIllegalArgumentException {
+ long below = Arrays.stream(values)
+ .filter(val -> val <= threshold)
+ .count();
+ return (double) below / values.length;
+ }
+
+ @Override
+ public double evaluate(double[] values, int begin, int length) throws MathIllegalArgumentException {
+ long below = IntStream.range(begin, length)
+ .mapToDouble(i -> values[i])
+ .filter(val -> val > threshold)
+ .count();
+ return 100.0 * below / length;
+ }
+
+ @Override
+ public UnivariateStatistic copy() {
+ return new PercentageAboveThreshold(threshold);
+ }
+}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java
new file mode 100644
index 0000000..92628d9
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.exc.MismatchedInputException;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+import org.apache.plc4x.java.scraper.config.JobConfiguration;
+import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.util.List;
+
+@ExtendWith(MockitoExtension.class)
+class ScraperConfigurationTest implements WithAssertions {
+
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+
+ @Test
+ void parseJobs_fromString() throws IOException {
+ String yaml = "sources:\n" +
+ " a1: b\n" +
+ " a2: b\n" +
+ " a3: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n" +
+ " - a2\n" +
+ " - a3\n" +
+ " fields:\n" +
+ " a: DBasdf\n" +
+ " b: DBbsdf\n";
+
+ ScraperConfiguration configuration = mapper.readValue(yaml, ScraperConfiguration.class);
+
+ assertThat(configuration.getJobConfigurations()).hasSize(1);
+ JobConfiguration conf = configuration.getJobConfigurations().get(0);
+
+ assertThat(configuration.getSources())
+ .isNotEmpty()
+ .hasSize(3)
+ .containsEntry("a1", "b")
+ .containsEntry("a2", "b")
+ .containsEntry("a3", "b");
+
+ assertThat(conf.getName()).isEqualTo("job1");
+ assertThat(conf.getScrapeRate()).isEqualTo(10);
+ assertThat(conf.getSources())
+ .hasSize(3);
+
+ assertThat(conf.getFields())
+ .hasSize(2)
+ .containsEntry("a", "DBasdf")
+ .containsEntry("b", "DBbsdf");
+ }
+
+ @Test
+ void parseJobs_missingEntries_fails() {
+ String jobs = "sources:\n" +
+ " a: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n";
+
+ assertThatThrownBy(() -> mapper.readValue(jobs, ScraperConfiguration.class))
+ .isInstanceOf(MismatchedInputException.class);
+ }
+
+ @Test
+ void fromYaml_loads() {
+ String yaml = "sources:\n" +
+ " a1: b\n" +
+ " a2: b\n" +
+ " a3: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n" +
+ " - a2\n" +
+ " - a3\n" +
+ " fields:\n" +
+ " a: DBasdf\n" +
+ " b: DBbsdf\n";
+
+ assertThatCode(() -> ScraperConfiguration.fromYaml(yaml))
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void fromString_loads() {
+ String json = "{\n" +
+ " \"sources\": {\n" +
+ " \"a1\": \"b\",\n" +
+ " \"a2\": \"b\",\n" +
+ " \"a3\": \"b\"\n" +
+ " },\n" +
+ " \"jobs\": [\n" +
+ " {\n" +
+ " \"name\": \"job1\",\n" +
+ " \"scrapeRate\": 10,\n" +
+ " \"sources\": [\n" +
+ " \"a1\",\n" +
+ " \"a2\",\n" +
+ " \"a3\"\n" +
+ " ],\n" +
+ " \"fields\": {\n" +
+ " \"a\": \"DBasdf\",\n" +
+ " \"b\": \"DBbsdf\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ assertThatCode(() -> ScraperConfiguration.fromJson(json))
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void new_notAllSourceAliasesAreResolvable_throws() {
+ String yaml = "sources:\n" +
+ " b: c\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - s1\n" +
+ " fields:\n";
+
+ assertThatThrownBy(() -> ScraperConfiguration.fromYaml(yaml))
+ .isInstanceOf(PlcRuntimeException.class)
+ .hasStackTraceContaining("unreferenced sources: [s1]");
+ }
+
+ @Test
+ void generateScrapeJobs_fromConfig() {
+ String yaml = "sources:\n" +
+ " source1: 'connection string'\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - source1\n" +
+ " fields:\n" +
+ " field1: 'DB1 Field 1'\n";
+
+ List<ScrapeJob> jobs = ScraperConfiguration.fromYaml(yaml).getJobs();
+ assertThat(jobs).hasSize(1);
+
+ ScrapeJob job = jobs.get(0);
+
+ assertThat(job.getName()).isEqualTo("job1");
+ assertThat(job.getScrapeRate()).isEqualTo(10);
+ assertThat(job.getConnections())
+ .hasSize(1)
+ .containsEntry("source1", "connection string");
+ assertThat(job.getFields())
+ .hasSize(1)
+ .containsEntry("field1", "DB1 Field 1");
+ }
+
+ @Nested
+ class Files {
+
+ @Test
+ void json() throws IOException {
+ ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.json");
+ }
+
+ @Test
+ void yaml() throws IOException {
+ ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.yml");
+ }
+ }
+}
\ No newline at end of file
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
index ae23245..c025946 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
@@ -72,12 +72,12 @@ class ScraperTest implements WithAssertions {
});
Scraper scraper = new Scraper(driverManager, Arrays.asList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
10,
Collections.singletonMap("tim", CONN_STRING_TIM),
Collections.singletonMap("distance", FIELD_STRING_TIM)
),
- new Scraper.ScrapeJob("job2",
+ new ScrapeJob("job2",
10,
Collections.singletonMap("chris", CONN_STRING_CH),
Collections.singletonMap("counter", FIELD_STRING_CH)
@@ -96,7 +96,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
10,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
@@ -120,7 +120,7 @@ class ScraperTest implements WithAssertions {
PlcDriverManager driverManager = new PlcDriverManager();
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
@@ -147,7 +147,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
diff --git a/plc4j/utils/scraper/src/test/resources/config.json b/plc4j/utils/scraper/src/test/resources/config.json
new file mode 100644
index 0000000..c77d17a
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/config.json
@@ -0,0 +1,18 @@
+{
+ "sources": {
+ "a": "b"
+ },
+ "jobs": [
+ {
+ "name": "job1",
+ "scrapeRate": 10,
+ "sources": [
+ "a"
+ ],
+ "fields": {
+ "a": "DBasdf",
+ "b": "DBbsdf"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/plc4j/utils/scraper/src/test/resources/config.yml b/plc4j/utils/scraper/src/test/resources/config.yml
new file mode 100644
index 0000000..c89df2a
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/config.yml
@@ -0,0 +1,10 @@
+sources:
+ a: b
+jobs:
+ - name: job1
+ scrapeRate: 10
+ sources:
+ - a
+ fields:
+ a: DBasdf
+ b: DBbsdf
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 9310ba6..b9dac77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,6 +98,7 @@
<groovy.version>2.5.3</groovy.version>
<gson.version>2.8.0</gson.version>
<hamcrest.version>1.3</hamcrest.version>
+ <jackson.version>2.9.4</jackson.version>
<junit.version>4.12</junit.version>
<junit.jupiter.version>5.2.0</junit.jupiter.version>
<junit.platform.version>1.2.0</junit.platform.version>