You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@plc4x.apache.org by jf...@apache.org on 2018/12/11 08:54:39 UTC
[incubator-plc4x] branch features/scraper-jmx-support updated
(44cd030 -> 9caa1b2)
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a change to branch features/scraper-jmx-support
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git.
discard 44cd030 [OPM,POOL] Added better defaults for Scrapers PlcPooledDriverManager.
discard 7cb9db0 Added some Logging for Diagnostics in PlcDriverManager.
discard a39af71 [OPM] Improved logging, better Exception handling.
discard 2f12488 Adjusted the content of the failure-email
discard 3a34427 Adjusted the content of the unstable-email
discard ecdbce8 Fixed the build again ...
discard e109903 Fixed the compilation error ... now the build should be unstable due to a failing test ...
discard d35702d PLC4X-78 - Write operations seem to fail - And another ...
discard a707b10 PLC4X-78 - Write operations seem to fail - And another ...
discard c586142 PLC4X-78 - Write operations seem to fail - And another ...
discard fe0b7e9 PLC4X-78 - Write operations seem to fail - And another ... - Intentionally breaking the API module to get faster results ...
discard a88a438 PLC4X-78 - Write operations seem to fail - And another ...
discard 8a197bf PLC4X-78 - Write operations seem to fail - And another ...
discard f5e214b PLC4X-78 - Write operations seem to fail - And another ...
discard 78f0959 PLC4X-78 - Write operations seem to fail - And another ...
discard 81d81b2 PLC4X-78 - Write operations seem to fail - And another ...
discard 7d4fbac PLC4X-78 - Write operations seem to fail - And another ...
discard fa6f124 PLC4X-78 - Write operations seem to fail - And another ...
discard cac784d PLC4X-78 - Write operations seem to fail - And another ...
discard befa846 PLC4X-78 - Write operations seem to fail - Another try of fixing the syntax ...
discard 2be5ee4 PLC4X-78 - Write operations seem to fail - Tried fixing the pipeline syntax.
discard 7a96003 PLC4X-78 - Write operations seem to fail - Inserted a failure - Finetuned the Jenkinsfile to send nicer formatted emails and one email if the build is back to normal
discard f6cbcaa PLC4X-78 - Write operations seem to fail - Made it stable again
discard ddbb7d4 Try to get the email sending working - Intentionally added a test-failure
discard 03210f6 PLC4X-78 - Write operations seem to fail - Fixed a problem that was causing a test to fail / Extended the S7PlcTestConnection to dump the content of packets not matching the expected packet
discard 6c06b62 PLC4X-78 - Write operations seem to fail - Fixed a problem that was causing a test to fail
discard 9d87c16 PLC4X-78 - Write operations seem to fail - Continued work on the write operations
discard 6910790 Tried figuring out how to clear the stash
discard 55101b3 Tried figuring out how to clear the stash
discard 0c57204 Tried figuring out how to clear the stash
discard 66e5bff [plc4x-karaf] remove duplicated plugin management
discard fc64b6d - Fixed the guava version as there were CVEs reported with the 18.0 version transitively referenced from the ethernet-ip module.
discard 1af043b - Removed the commented out slackSending
discard 3be52ca - Fixed some things SonarQube was complaining about. - Tried to get the deleting of the "local-snapshots-dir" working.
discard 2a1f17b - Added the new release.
discard 245c2db - Fixed the build by adding a missing dependency.
discard 458e6ad adjusted connector implementation
discard 1a1000a - Added name and description so the build displays the scraper the same way as all the other modules.
discard cf203be Made the constructor public so it can also be configured programmatically.
discard 329054f Fixed some textual errors.
discard 8130b1e Removed some now obsolete comment.
discard 2266622 PLC4X-77 - When the last item in a request is a DINT, the DefaultS7MessageProcessor dies with an ArrayIndexOutOfFoundsException
discard 186d3b2 PLC4X-77 - When the last item in a request is a DINT, the DefaultS7MessageProcessor dies with an ArrayIndexOutOfFoundsException
discard 6f107e4 Made the example take multiple addresses as input
discard e1d32c9 - Added a cleanup before un-stashing the snapshots to hopefully reduce the deploy-time as currently all previous snapshot versions were deployed too.
discard e82a8ce - Added profiling to the edgent adapter.
discard b86c2d7 PLC4X-76 - When receiving responses with more than 512 byte, the IsoOnTcp protocol doesn't work
discard e299e38 - Added name and description to the pom.xml
discard adbc6d0 - Fixed a bug in the processing of real responses.
discard 47c58b2 - Extended the size of the byte-offset to allow 5 digit offsets.
discard 5cb40cd [PLC4X-75] - Fixing dependency to the wrap url-handler
omit 1f03e11 [plc4j-scraper] Implementation of JMX.
add 826fc1d [PLC4X-75] - Fixing dependency to the wrap url-handler
add fac1b5d Merge pull request #42 from ANierbeck/develop
add 6e78786 - Extended the size of the byte-offset to allow 5 digit offsets.
add 39e69b0 - Fixed a bug in the processing of real responses.
add 50f020a - Added name and description to the pom.xml
add dcca499 PLC4X-76 - When receiving responses with more than 512 byte, the IsoOnTcp protocol doesn't work
add 829cbae - Added profiling to the edgent adapter.
add 85e1cc4 - Added a cleanup before un-stashing the snapshots to hopefully reduce the deploy-time as currently all previous snapshot versions were deployed too.
add a1b3bd4 adjusted connector implementation
add 5806147 Made the example take multiple addresses as input
add 5d03591 PLC4X-77 - When the last item in a request is a DINT, the DefaultS7MessageProcessor dies with an ArrayIndexOutOfFoundsException
add cc5fce9 PLC4X-77 - When the last item in a request is a DINT, the DefaultS7MessageProcessor dies with an ArrayIndexOutOfFoundsException
add fbb8b98 Removed some now obsolete comment.
add 32b0a87 Fixed some textual errors.
add e7cfa1a Made the constructor public so it can also be configured programmatically.
add d248967 - Added name and description so the build displays the scraper the same way as all the other modules.
add cc7c3c9 Merge branch 'develop' of https://gitbox.apache.org/repos/asf/incubator-plc4x into develop
add 3d0222c - Fixed the build by adding a missing dependency.
add 099c1a7 - Added the new release.
add d401780 - Fixed some things SonarQube was complaining about. - Tried to get the deleting of the "local-snapshots-dir" working.
add 4a0709e - Removed the commented out slackSending
add 3ef1a1d - Fixed the guava version as there were CVEs reported with the 18.0 version transitively referenced from the ethernet-ip module.
add 79c57bf [plc4x-karaf] remove duplicated plugin management
add 6fb2583 Tried figuring out how to clear the stash
add 26ca89b Tried figuring out how to clear the stash
add f293a86 Tried figuring out how to clear the stash
add 6a41048 PLC4X-78 - Write operations seem to fail - Continued work on the write operations
add e192c64 PLC4X-78 - Write operations seem to fail - Fixed a problem that was causing a test to fail
add 3d9ba43 PLC4X-78 - Write operations seem to fail - Fixed a problem that was causing a test to fail / Extended the S7PlcTestConnection to dump the content of packets not matching the expected packet
add b28d4a0 Try to get the email sending working - Intentionally added a test-failure
add 5d8237c PLC4X-78 - Write operations seem to fail - Made it stable again
add 1712ddb PLC4X-78 - Write operations seem to fail - Inserted a failure - Finetuned the Jenkinsfile to send nicer formatted emails and one email if the build is back to normal
add 81fa0bd PLC4X-78 - Write operations seem to fail - Tried fixing the pipeline syntax.
add 630e8ba PLC4X-78 - Write operations seem to fail - Another try of fixing the syntax ...
add a5aa4d0 PLC4X-78 - Write operations seem to fail - And another ...
add 359c7b0 PLC4X-78 - Write operations seem to fail - And another ...
add acb7388 PLC4X-78 - Write operations seem to fail - And another ...
add 009a924 PLC4X-78 - Write operations seem to fail - And another ...
add 9fb6273 PLC4X-78 - Write operations seem to fail - And another ...
add 6c1d5da PLC4X-78 - Write operations seem to fail - And another ...
add 07c5234 PLC4X-78 - Write operations seem to fail - And another ...
add c9b39f7 PLC4X-78 - Write operations seem to fail - And another ...
add 4dcffa2 PLC4X-78 - Write operations seem to fail - And another ... - Intentionally breaking the API module to get faster results ...
add b2f5e9b PLC4X-78 - Write operations seem to fail - And another ...
add 96baaa0 PLC4X-78 - Write operations seem to fail - And another ...
add aa0c6df PLC4X-78 - Write operations seem to fail - And another ...
add e9d2ad7 Fixed the compilation error ... now the build should be unstable due to a failing test ...
add 967b6ca Fixed the build again ...
add 8c96312 Adjusted the content of the unstable-email
add 8d190c2 Adjusted the content of the failure-email
add b054c6e [OPM] Improved logging, better Exception handling.
add 2cb21bc Added some Logging for Diagnostics in PlcDriverManager.
add a67f311 [OPM,POOL] Added better defaults for Scrapers PlcPooledDriverManager.
new 9caa1b2 [plc4j-scraper] Implementation of JMX.
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (44cd030)
\
N -- N -- N refs/heads/features/scraper-jmx-support (9caa1b2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[incubator-plc4x] 01/01: [plc4j-scraper] Implementation of JMX.
Posted by jf...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a commit to branch features/scraper-jmx-support
in repository https://gitbox.apache.org/repos/asf/incubator-plc4x.git
commit 9caa1b289d208033ba22664802deced1e55f17f2
Author: Julian Feinauer <j....@pragmaticminds.de>
AuthorDate: Sun Nov 25 17:43:50 2018 +0100
[plc4j-scraper] Implementation of JMX.
---
plc4j/utils/scraper/pom.xml | 6 ++++
.../org/apache/plc4x/java/scraper/Scraper.java | 35 +++++++++++++++++++++-
.../apache/plc4x/java/scraper/ScraperMBean.java} | 19 ++++--------
.../org/apache/plc4x/java/scraper/ScraperTask.java | 26 +++++++++++++++-
.../plc4x/java/scraper/ScraperTaskMBean.java} | 19 +++++-------
.../apache/plc4x/java/scraper/ScraperRunner.java | 19 ++++++++++--
6 files changed, 95 insertions(+), 29 deletions(-)
diff --git a/plc4j/utils/scraper/pom.xml b/plc4j/utils/scraper/pom.xml
index 8b9bf2b..ee65b41 100644
--- a/plc4j/utils/scraper/pom.xml
+++ b/plc4j/utils/scraper/pom.xml
@@ -70,6 +70,12 @@
<artifactId>plc4j-connection-pool</artifactId>
<version>0.3.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.plc4x</groupId>
+ <artifactId>plc4j-driver-simulated</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
index 725e58e..bc23b35 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/Scraper.java
@@ -35,6 +35,8 @@ import org.apache.plc4x.java.utils.connectionpool.PooledPlcDriverManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.management.*;
+import java.lang.management.ManagementFactory;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -43,9 +45,10 @@ import java.util.concurrent.*;
/**
* Main class that orchestrates scraping.
*/
-public class Scraper {
+public class Scraper implements ScraperMBean {
private static final Logger LOGGER = LoggerFactory.getLogger(Scraper.class);
+ public static final String MX_DOMAIN = "org.apache.plc4x.java";
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(10,
new BasicThreadFactory.Builder()
@@ -66,6 +69,7 @@ public class Scraper {
private final MultiValuedMap<ScraperTask, ScheduledFuture<?>> futures = new ArrayListValuedHashMap<>();
private final PlcDriverManager driverManager;
private final List<ScrapeJob> jobs;
+ private MBeanServer mBeanServer;
/**
* Creates a Scraper instance from a configuration.
@@ -103,6 +107,13 @@ public class Scraper {
Validate.notEmpty(jobs);
this.driverManager = driverManager;
this.jobs = jobs;
+ // Register MBean
+ mBeanServer = ManagementFactory.getPlatformMBeanServer();
+ try {
+ mBeanServer.registerMBean(this, new ObjectName(MX_DOMAIN, "scraper", "scraper"));
+ } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException | MalformedObjectNameException e) {
+ LOGGER.debug("Unable to register Scraper as MBean", e);
+ }
}
/**
@@ -124,6 +135,8 @@ public class Scraper {
tuple.getLeft().getFields(),
1_000,
handlerPool, resultHandler);
+ // Register task mxbean
+ registerTaskMBean(task);
// Add task to internal list
tasks.put(tuple.getLeft(), task);
ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(task,
@@ -150,6 +163,18 @@ public class Scraper {
}
/**
+ * Register a task as MBean
+ * @param task task to register
+ */
+ private void registerTaskMBean(ScraperTask task) {
+ try {
+ mBeanServer.registerMBean(task, new ObjectName(MX_DOMAIN + ":type=ScrapeTask,name=" + task.getJobName() + "-" + task.getConnectionAlias()));
+ } catch (InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException | MalformedObjectNameException e) {
+ LOGGER.debug("Unable to register Task as MBean", e);
+ }
+ }
+
+ /**
* For testing.
*/
ScheduledExecutorService getScheduler() {
@@ -171,4 +196,12 @@ public class Scraper {
futures.clear();
}
+ // MBean methods
+
+ @Override
+ public boolean isRunning() {
+ // TODO is this okay so?
+ return !futures.isEmpty();
+ }
+
}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperMBean.java
similarity index 58%
copy from plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
copy to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperMBean.java
index 029a25a..edc2d5a 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperMBean.java
@@ -19,20 +19,13 @@
package org.apache.plc4x.java.scraper;
-import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class ScraperRunner {
+/**
+ * MBean for {@link Scraper}
+ */
+public interface ScraperMBean {
- private static final Logger LOGGER = LoggerFactory.getLogger(ScraperRunner.class);
+ boolean isRunning();
- public static void main(String[] args) throws IOException {
- ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml");
- Scraper scraper = new Scraper(configuration, (j, a, m) -> LOGGER.info("Results from {}/{}: {}", j, a, m));
+ int getNumberOfActiveTasks();
- scraper.start();
- }
}
diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
index f3240cf..ded9a87 100644
--- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTask.java
@@ -32,6 +32,7 @@ import org.apache.plc4x.java.api.types.PlcResponseCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
@@ -46,7 +47,7 @@ import java.util.stream.Collectors;
* One {@link ScrapeJob} gets split into multiple tasks.
* One task for each source that is defined in the {@link org.apache.plc4x.java.scraper.config.JobConfiguration}.
*/
-public class ScraperTask implements Runnable {
+public class ScraperTask implements Runnable, ScraperTaskMBean {
private static final Logger LOGGER = LoggerFactory.getLogger(ScraperTask.class);
@@ -194,4 +195,27 @@ public class ScraperTask implements Runnable {
LOGGER.warn("Handling error responses: {}", failed);
}
+
+ //---------------------------------
+ // JMX Monitoring
+ //---------------------------------
+ @Override
+ public long getScrapesTotal() {
+ return requestCounter.get();
+ }
+
+ @Override
+ public long getScrapesSuccess() {
+ return successCounter.get();
+ }
+
+ @Override
+ public String[] getPercentiles() {
+ String[] percentiles = new String[10];
+ for (int i = 1; i <= 10; i += 1) {
+ percentiles[i - 1] = String.format("%d%%: %s ms", 10 * i, latencyStatistics.getPercentile(10.0 * i) * 1e-6);
+ }
+ return percentiles;
+ }
+
}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskMBean.java
similarity index 58%
copy from plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
copy to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskMBean.java
index 029a25a..08acc43 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
+++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskMBean.java
@@ -19,20 +19,17 @@
package org.apache.plc4x.java.scraper;
-import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+/**
+ * MBean for a scrape job.
+ */
+public interface ScraperTaskMBean {
-import java.io.IOException;
+ long getScrapesTotal();
-public class ScraperRunner {
+ long getScrapesSuccess();
- private static final Logger LOGGER = LoggerFactory.getLogger(ScraperRunner.class);
+ double getPercentageFailed();
- public static void main(String[] args) throws IOException {
- ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml");
- Scraper scraper = new Scraper(configuration, (j, a, m) -> LOGGER.info("Results from {}/{}: {}", j, a, m));
+ String[] getPercentiles();
- scraper.start();
- }
}
diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
index 029a25a..39dbdf7 100644
--- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
+++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java
@@ -20,6 +20,7 @@
package org.apache.plc4x.java.scraper;
import org.apache.plc4x.java.scraper.config.ScraperConfiguration;
+import org.apache.plc4x.java.scraper.config.ScraperConfigurationBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,9 +30,21 @@ public class ScraperRunner {
private static final Logger LOGGER = LoggerFactory.getLogger(ScraperRunner.class);
- public static void main(String[] args) throws IOException {
- ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml");
- Scraper scraper = new Scraper(configuration, (j, a, m) -> LOGGER.info("Results from {}/{}: {}", j, a, m));
+ public static void main(String[] args) {
+ ScraperConfigurationBuilder builder = new ScraperConfigurationBuilder();
+ ScraperConfiguration conf = builder
+ .addSource("source", "test:123")
+ .addSource("source2", "test:456")
+ .job("job1", 100)
+ .source("source")
+ .source("source2")
+ .field("field1", "RANDOM/test:INTEGER")
+ .build()
+ .build();
+
+ // Alternativ: Load from a file.
+ // ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml");
+ Scraper scraper = new Scraper(conf, (j, a, m) -> LOGGER.info("Results from {}/{}: {}", j, a, m));
scraper.start();
}