You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@plc4x.apache.org by jf...@apache.org on 2018/11/24 23:30:42 UTC
[incubator-plc4x] 01/02: [plc4j-scraper] Added configuration with
yml / json support.
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git
commit 92604c0c7bc650b2449b37c74d292b1f85493baa
Author: Julian Feinauer <j....@pragmaticminds.de>
AuthorDate: Sat Nov 24 23:59:37 2018 +0100
[plc4j-scraper] Added configuration with yml / json support.
---
plc4j/utils/scraper/pom.xml | 23 ++-
.../org/apache/plc4x/java/scraper/ScrapeJob.java | 59 ++++++
.../org/apache/plc4x/java/scraper/Scraper.java | 84 +++------
.../java/scraper/config/JobConfiguration.java | 65 +++++++
.../java/scraper/config/ScraperConfiguration.java | 119 ++++++++++++
.../scraper/util/PercentageAboveThreshold.java | 58 ++++++
.../java/scraper/ScraperConfigurationTest.java | 202 +++++++++++++++++++++
.../org/apache/plc4x/java/scraper/ScraperTest.java | 10 +-
plc4j/utils/scraper/src/test/resources/config.json | 18 ++
plc4j/utils/scraper/src/test/resources/config.yml | 10 +
pom.xml | 1 +
11 files changed, 581 insertions(+), 68 deletions(-)
diff --git a/plc4j/utils/scraper/pom.xml b/plc4j/utils/scraper/pom.xml
index f32a624..c914b19 100644
--- a/plc4j/utils/scraper/pom.xml
+++ b/plc4j/utils/scraper/pom.xml
@@ -31,6 +31,27 @@
<artifactId>plc4j-scraper</artifactId>
<dependencies>
+ <!--Jackson-->
+ <!--<dependency>-->
+ <!--<groupId>com.fasterxml.jackson.core</groupId>-->
+ <!--<artifactId>jackson-core</artifactId>-->
+ <!--<version>${jackson.version}</version>-->
+ <!--</dependency>-->
+ <!--<dependency>-->
+ <!--<groupId>com.fasterxml.jackson.core</groupId>-->
+ <!--<artifactId>jackson-annotations</artifactId>-->
+ <!--<version>${jackson.version}</version>-->
+ <!--</dependency>-->
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
@@ -57,12 +78,10 @@
<groupId>org.apache.plc4x</groupId>
<artifactId>plc4j-connection-pool</artifactId>
<version>0.3.0-SNAPSHOT</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.plc4x</groupId>
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java
new file mode 100644
index 0000000..296032b
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper;
+
+import java.util.Map;
+
+public class ScrapeJob {
+
+ private final String name;
+ private final long scrapeRate;
+ private final Map<String, String> connections;
+ private final Map<String, String> fields;
+
+ public ScrapeJob(String name, long scrapeRate, Map<String, String> connections, Map<String, String> fields) {
+ this.name = name;
+ this.scrapeRate = scrapeRate;
+ this.connections = connections;
+ this.fields = fields;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getScrapeRate() {
+ return scrapeRate;
+ }
+
+ /**
+ * alias -> connection-string
+ */
+ public Map<String, String> getConnections() {
+ return connections;
+ }
+
+ /**
+ * alias -> field-query
+ */
+ public Map<String, String> getFields() {
+ return fields;
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
index e1f9fc1..dfe86b0 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
@@ -24,19 +24,18 @@ import org.apache.commons.collections4.multimap.ArrayListValuedHashMap;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.commons.lang3.tuple.Triple;
-import org.apache.commons.math3.exception.MathIllegalArgumentException;
import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
-import org.apache.commons.math3.stat.descriptive.UnivariateStatistic;
import org.apache.plc4x.java.PlcDriverManager;
+import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.apache.plc4x.java.scraper.util.PercentageAboveThreshold;
+import org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.*;
-import java.util.stream.IntStream;
/**
* Main class that orchestrates scraping.
@@ -62,6 +61,20 @@ public class Scraper {
private final PlcDriverManager driverManager;
private final List<ScrapeJob> jobs;
+ /**
+ * Creates a Scraper instance from a configuration.
+ * By default a {@link PooledPlcDriverManager} is used.
+ * @param config Configuration to use.
+ */
+ public Scraper(ScraperConfiguration config) {
+ this(new PooledPlcDriverManager(), config.getJobs());
+ }
+
+ /**
+ *
+ * @param driverManager
+ * @param jobs
+ */
public Scraper(PlcDriverManager driverManager, List<ScrapeJob> jobs) {
Validate.notEmpty(jobs);
this.driverManager = driverManager;
@@ -75,22 +88,22 @@ public class Scraper {
// Schedule all jobs
LOGGER.info("Starting jobs...");
jobs.stream()
- .flatMap(job -> job.connections.entrySet().stream()
+ .flatMap(job -> job.getConnections().entrySet().stream()
.map(entry -> Triple.of(job, entry.getKey(), entry.getValue()))
)
.forEach(
tuple -> {
LOGGER.debug("Register task for job {} for conn {} ({}) at rate {} ms",
- tuple.getLeft().name, tuple.getMiddle(), tuple.getRight(), tuple.getLeft().scrapeRate);
+ tuple.getLeft().getName(), tuple.getMiddle(), tuple.getRight(), tuple.getLeft().getScrapeRate());
ScraperTask task = new ScraperTask(driverManager,
- tuple.getLeft().name, tuple.getMiddle(), tuple.getRight(),
- tuple.getLeft().fields,
+ tuple.getLeft().getName(), tuple.getMiddle(), tuple.getRight(),
+ tuple.getLeft().getFields(),
1_000,
handlerPool);
// Add task to internal list
tasks.put(tuple.getLeft(), task);
ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(task,
- 0, tuple.getLeft().scrapeRate, TimeUnit.MILLISECONDS);
+ 0, tuple.getLeft().getScrapeRate(), TimeUnit.MILLISECONDS);
// Store the handle for stopping, etc.
futures.put(task, future);
@@ -102,7 +115,7 @@ public class Scraper {
for (Map.Entry<ScrapeJob, ScraperTask> entry : tasks.entries()) {
DescriptiveStatistics statistics = entry.getValue().getLatencyStatistics();
String msg = String.format(Locale.ENGLISH, "Job statistics (%s, %s) number of requests: %d (%d success, %.1f %% failed, %.1f %% too slow), mean latency: %.2f ms, median: %.2f ms",
- entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().scrapeRate * 1e6)), statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
+ entry.getValue().getJobName(), entry.getValue().getConnectionAlias(), entry.getValue().getRequestCounter(), entry.getValue().getSuccessfullRequestCounter(), entry.getValue().getPercentageFailed(), statistics.apply(new PercentageAboveThreshold(entry.getKey().getScrapeRate() * 1e6)), statistics.getMean() * 1e-6, statistics.getPercentile(50) * 1e-6);
LOGGER.info(msg);
}
}, 1_000, 1_000, TimeUnit.MILLISECONDS);
@@ -130,55 +143,4 @@ public class Scraper {
futures.clear();
}
- public static class ScrapeJob {
-
- private final String name;
- private final long scrapeRate;
- /**
- * alias -> connection-string
- */
- private final Map<String, String> connections;
- /**
- * alias -> field-query
- */
- private final Map<String, String> fields;
-
- public ScrapeJob(String name, long scrapeRate, Map<String, String> connections, Map<String, String> fields) {
- this.name = name;
- this.scrapeRate = scrapeRate;
- this.connections = connections;
- this.fields = fields;
- }
- }
-
- private static class PercentageAboveThreshold implements UnivariateStatistic {
-
- private final double threshold;
-
- public PercentageAboveThreshold(double threshold) {
- this.threshold = threshold;
- }
-
- @Override
- public double evaluate(double[] values) throws MathIllegalArgumentException {
- long below = Arrays.stream(values)
- .filter(val -> val <= threshold)
- .count();
- return (double) below / values.length;
- }
-
- @Override
- public double evaluate(double[] values, int begin, int length) throws MathIllegalArgumentException {
- long below = IntStream.range(begin, length)
- .mapToDouble(i -> values[i])
- .filter(val -> val > threshold)
- .count();
- return 100.0 * below / length;
- }
-
- @Override
- public UnivariateStatistic copy() {
- return new PercentageAboveThreshold(threshold);
- }
- }
}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java
new file mode 100644
index 0000000..6cb2ede
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class JobConfiguration {
+
+ private final String name;
+ private final int scrapeRate;
+ private final List<String> sources;
+ private final Map<String, String> fields;
+
+ @JsonCreator
+ JobConfiguration(@JsonProperty(value = "name", required = true) String name,
+ @JsonProperty(value = "scrapeRate", required = true) int scrapeRate,
+ @JsonProperty(value = "sources", required = true) List<String> sources,
+ @JsonProperty(value = "fields", required = true) Map<String, String> fields) {
+ this.name = name;
+ this.scrapeRate = scrapeRate;
+ this.sources = sources;
+ this.fields = fields;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getScrapeRate() {
+ return scrapeRate;
+ }
+
+ public List<String> getSources() {
+ return sources;
+ }
+
+ public Map<String, String> getFields() {
+ return fields;
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java
new file mode 100644
index 0000000..48d688c
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+import org.apache.plc4x.java.scraper.ScrapeJob;
+import org.apache.plc4x.java.scraper.Scraper;
+
+import java.io.*;
+import java.security.InvalidParameterException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Configuration class for {@link Scraper}.
+ */
+
+public class ScraperConfiguration {
+
+ private final Map<String, String> sources;
+ private final List<JobConfiguration> jobConfigurations;
+
+ @JsonCreator
+ ScraperConfiguration(@JsonProperty(value = "sources", required = true) Map<String, String> sources,
+ @JsonProperty(value = "jobs", required = true) List<JobConfiguration> jobConfigurations) {
+ checkNoUnreferencedSources(sources, jobConfigurations);
+ // TODO Warning on too many sources?!
+ this.sources = sources;
+ this.jobConfigurations = jobConfigurations;
+ }
+
+ private void checkNoUnreferencedSources(Map<String, String> sources, List<JobConfiguration> jobConfigurations) {
+ Set<String> unreferencedSources = jobConfigurations.stream()
+ .flatMap(job -> job.getSources().stream())
+ .filter(source -> !sources.containsKey(source))
+ .collect(Collectors.toSet());
+ if (!unreferencedSources.isEmpty()) {
+ throw new PlcRuntimeException("There are the following unreferenced sources: " + unreferencedSources);
+ }
+ }
+
+ public static ScraperConfiguration fromYaml(String yaml) {
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+ try {
+ return mapper.readValue(yaml, ScraperConfiguration.class);
+ } catch (IOException e) {
+ throw new PlcRuntimeException("Unable to parse given yaml configuration!", e);
+ }
+ }
+
+ public static ScraperConfiguration fromJson(String json) {
+ ObjectMapper mapper = new ObjectMapper(new JsonFactory());
+ try {
+ return mapper.readValue(json, ScraperConfiguration.class);
+ } catch (IOException e) {
+ throw new PlcRuntimeException("Unable to parse given json configuration!", e);
+ }
+ }
+
+ public static ScraperConfiguration fromFile(String path) throws IOException {
+ ObjectMapper mapper;
+ if (path.endsWith("json")) {
+ mapper = new ObjectMapper(new JsonFactory());
+ } else if (path.endsWith("yml") || path.endsWith("yaml")) {
+ mapper = new ObjectMapper(new YAMLFactory());
+ } else {
+ throw new InvalidParameterException("Only files with extensions json, yml or yaml can be read");
+ }
+ return mapper.readValue(new File(path), ScraperConfiguration.class);
+ }
+
+ public Map<String, String> getSources() {
+ return sources;
+ }
+
+ public List<JobConfiguration> getJobConfigurations() {
+ return jobConfigurations;
+ }
+
+ public List<ScrapeJob> getJobs() {
+ return jobConfigurations.stream()
+ .map(conf -> new ScrapeJob(conf.getName(), conf.getScrapeRate(),
+ getSourcesForAliases(conf.getSources()), conf.getFields()))
+ .collect(Collectors.toList());
+ }
+
+ private Map<String, String> getSourcesForAliases(List<String> aliases) {
+ return aliases.stream()
+ .collect(Collectors.toMap(
+ Function.identity(),
+ sources::get
+ ));
+ }
+}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java
new file mode 100644
index 0000000..1cafea1
--- /dev/null
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/util/PercentageAboveThreshold.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper.util;
+
+import org.apache.commons.math3.exception.MathIllegalArgumentException;
+import org.apache.commons.math3.stat.descriptive.UnivariateStatistic;
+import org.apache.plc4x.java.scraper.Scraper;
+
+import java.util.Arrays;
+import java.util.stream.IntStream;
+
+public class PercentageAboveThreshold implements UnivariateStatistic {
+
+ private final double threshold;
+
+ public PercentageAboveThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+
+ @Override
+ public double evaluate(double[] values) throws MathIllegalArgumentException {
+ long below = Arrays.stream(values)
+ .filter(val -> val <= threshold)
+ .count();
+ return (double) below / values.length;
+ }
+
+ @Override
+ public double evaluate(double[] values, int begin, int length) throws MathIllegalArgumentException {
+ long below = IntStream.range(begin, length)
+ .mapToDouble(i -> values[i])
+ .filter(val -> val > threshold)
+ .count();
+ return 100.0 * below / length;
+ }
+
+ @Override
+ public UnivariateStatistic copy() {
+ return new PercentageAboveThreshold(threshold);
+ }
+}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java
new file mode 100644
index 0000000..92628d9
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.plc4x.java.scraper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.exc.MismatchedInputException;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.plc4x.java.api.exceptions.PlcRuntimeException;
+import org.apache.plc4x.java.scraper.config.JobConfiguration;
+import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.assertj.core.api.WithAssertions;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.util.List;
+
+@ExtendWith(MockitoExtension.class)
+class ScraperConfigurationTest implements WithAssertions {
+
+ ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+
+ @Test
+ void parseJobs_fromString() throws IOException {
+ String yaml = "sources:\n" +
+ " a1: b\n" +
+ " a2: b\n" +
+ " a3: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n" +
+ " - a2\n" +
+ " - a3\n" +
+ " fields:\n" +
+ " a: DBasdf\n" +
+ " b: DBbsdf\n";
+
+ ScraperConfiguration configuration = mapper.readValue(yaml, ScraperConfiguration.class);
+
+ assertThat(configuration.getJobConfigurations()).hasSize(1);
+ JobConfiguration conf = configuration.getJobConfigurations().get(0);
+
+ assertThat(configuration.getSources())
+ .isNotEmpty()
+ .hasSize(3)
+ .containsEntry("a1", "b")
+ .containsEntry("a2", "b")
+ .containsEntry("a3", "b");
+
+ assertThat(conf.getName()).isEqualTo("job1");
+ assertThat(conf.getScrapeRate()).isEqualTo(10);
+ assertThat(conf.getSources())
+ .hasSize(3);
+
+ assertThat(conf.getFields())
+ .hasSize(2)
+ .containsEntry("a", "DBasdf")
+ .containsEntry("b", "DBbsdf");
+ }
+
+ @Test
+ void parseJobs_missingEntries_fails() {
+ String jobs = "sources:\n" +
+ " a: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n";
+
+ assertThatThrownBy(() -> mapper.readValue(jobs, ScraperConfiguration.class))
+ .isInstanceOf(MismatchedInputException.class);
+ }
+
+ @Test
+ void fromYaml_loads() {
+ String yaml = "sources:\n" +
+ " a1: b\n" +
+ " a2: b\n" +
+ " a3: b\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - a1\n" +
+ " - a2\n" +
+ " - a3\n" +
+ " fields:\n" +
+ " a: DBasdf\n" +
+ " b: DBbsdf\n";
+
+ assertThatCode(() -> ScraperConfiguration.fromYaml(yaml))
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void fromString_loads() {
+ String json = "{\n" +
+ " \"sources\": {\n" +
+ " \"a1\": \"b\",\n" +
+ " \"a2\": \"b\",\n" +
+ " \"a3\": \"b\"\n" +
+ " },\n" +
+ " \"jobs\": [\n" +
+ " {\n" +
+ " \"name\": \"job1\",\n" +
+ " \"scrapeRate\": 10,\n" +
+ " \"sources\": [\n" +
+ " \"a1\",\n" +
+ " \"a2\",\n" +
+ " \"a3\"\n" +
+ " ],\n" +
+ " \"fields\": {\n" +
+ " \"a\": \"DBasdf\",\n" +
+ " \"b\": \"DBbsdf\"\n" +
+ " }\n" +
+ " }\n" +
+ " ]\n" +
+ "}";
+
+ assertThatCode(() -> ScraperConfiguration.fromJson(json))
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void new_notAllSourceAliasesAreResolvable_throws() {
+ String yaml = "sources:\n" +
+ " b: c\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - s1\n" +
+ " fields:\n";
+
+ assertThatThrownBy(() -> ScraperConfiguration.fromYaml(yaml))
+ .isInstanceOf(PlcRuntimeException.class)
+ .hasStackTraceContaining("unreferenced sources: [s1]");
+ }
+
+ @Test
+ void generateScrapeJobs_fromConfig() {
+ String yaml = "sources:\n" +
+ " source1: 'connection string'\n" +
+ "jobs:\n" +
+ " - name: job1\n" +
+ " scrapeRate: 10\n" +
+ " sources:\n" +
+ " - source1\n" +
+ " fields:\n" +
+ " field1: 'DB1 Field 1'\n";
+
+ List<ScrapeJob> jobs = ScraperConfiguration.fromYaml(yaml).getJobs();
+ assertThat(jobs).hasSize(1);
+
+ ScrapeJob job = jobs.get(0);
+
+ assertThat(job.getName()).isEqualTo("job1");
+ assertThat(job.getScrapeRate()).isEqualTo(10);
+ assertThat(job.getConnections())
+ .hasSize(1)
+ .containsEntry("source1", "connection string");
+ assertThat(job.getFields())
+ .hasSize(1)
+ .containsEntry("field1", "DB1 Field 1");
+ }
+
+ @Nested
+ class Files {
+
+ @Test
+ void json() throws IOException {
+ ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.json");
+ }
+
+ @Test
+ void yaml() throws IOException {
+ ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.yml");
+ }
+ }
+}
\ No newline at end of file
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
index ae23245..c025946 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperTest.java
@@ -72,12 +72,12 @@ class ScraperTest implements WithAssertions {
});
Scraper scraper = new Scraper(driverManager, Arrays.asList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
10,
Collections.singletonMap("tim", CONN_STRING_TIM),
Collections.singletonMap("distance", FIELD_STRING_TIM)
),
- new Scraper.ScrapeJob("job2",
+ new ScrapeJob("job2",
10,
Collections.singletonMap("chris", CONN_STRING_CH),
Collections.singletonMap("counter", FIELD_STRING_CH)
@@ -96,7 +96,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
10,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
@@ -120,7 +120,7 @@ class ScraperTest implements WithAssertions {
PlcDriverManager driverManager = new PlcDriverManager();
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
@@ -147,7 +147,7 @@ class ScraperTest implements WithAssertions {
when(mockDevice.read(any())).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultIntegerFieldItem(1)));
Scraper scraper = new Scraper(driverManager, Collections.singletonList(
- new Scraper.ScrapeJob("job1",
+ new ScrapeJob("job1",
1,
Collections.singletonMap("m1", "mock:m1"),
Collections.singletonMap("field1", "qry1")
diff --git a/plc4j/utils/scraper/src/test/resources/config.json b/plc4j/utils/scraper/src/test/resources/config.json
new file mode 100644
index 0000000..c77d17a
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/config.json
@@ -0,0 +1,18 @@
+{
+ "sources": {
+ "a": "b"
+ },
+ "jobs": [
+ {
+ "name": "job1",
+ "scrapeRate": 10,
+ "sources": [
+ "a"
+ ],
+ "fields": {
+ "a": "DBasdf",
+ "b": "DBbsdf"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/plc4j/utils/scraper/src/test/resources/config.yml b/plc4j/utils/scraper/src/test/resources/config.yml
new file mode 100644
index 0000000..c89df2a
--- /dev/null
+++ b/plc4j/utils/scraper/src/test/resources/config.yml
@@ -0,0 +1,10 @@
+sources:
+ a: b
+jobs:
+ - name: job1
+ scrapeRate: 10
+ sources:
+ - a
+ fields:
+ a: DBasdf
+ b: DBbsdf
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 9310ba6..b9dac77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,6 +98,7 @@
<groovy.version>2.5.3</groovy.version>
<gson.version>2.8.0</gson.version>
<hamcrest.version>1.3</hamcrest.version>
+ <jackson.version>2.9.4</jackson.version>
<junit.version>4.12</junit.version>
<junit.jupiter.version>5.2.0</junit.jupiter.version>
<junit.platform.version>1.2.0</junit.platform.version>