You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@karaf.apache.org by jb...@apache.org on 2020/04/23 12:01:12 UTC
[karaf-decanter] branch master updated: [KARAF-6647] Add redis
collector
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/karaf-decanter.git
The following commit(s) were added to refs/heads/master by this push:
new a53f1be [KARAF-6647] Add redis collector
new 1030600 Merge pull request #159 from jbonofre/KARAF-6647
a53f1be is described below
commit a53f1bedc10fe032690ef99fca564abcd0f9bd31
Author: jbonofre <jb...@apache.org>
AuthorDate: Mon Apr 20 19:12:43 2020 +0200
[KARAF-6647] Add redis collector
---
appender/redis/pom.xml | 45 ++++++-
.../decanter/appender/redis/RedisAppender.java | 1 -
assembly/src/main/feature/feature.xml | 18 ++-
collector/pom.xml | 1 +
collector/redis/pom.xml | 135 +++++++++++++++++++++
.../org.apache.karaf.decanter.collector.redis.cfg | 67 ++++++++++
.../decanter/collector/redis/RedisCollector.java | 75 ++++++------
.../src/main/asciidoc/user-guide/collectors.adoc | 57 ++++++++-
8 files changed, 357 insertions(+), 42 deletions(-)
diff --git a/appender/redis/pom.xml b/appender/redis/pom.xml
index cda5973..418b39c 100644
--- a/appender/redis/pom.xml
+++ b/appender/redis/pom.xml
@@ -57,11 +57,52 @@
<instructions>
<Export-Package>!*</Export-Package>
<Import-Package>
- com.esotericsoftware.kryo;resolution:=optional
+ !com.github.benmanes.caffeine.cache,
+ !io.micrometer.core*,
+ !io.netty*,
+ !lzma*,
+ !net.bytebuddy*,
+ !org.xerial.snappy*,
+ !reactor*,
+ com.google.protobuf*;resolution:=optional,
+ com.jcraft.jzlib;resolution:=optional,
+ com.ning.compress*;resolution:=optional,
+ com.oracle.svm*;resolution:=optional,
+ com.esotericsoftware.kryo*;resolution:=optional,
+ io.reactivex*;resolution:=optional,
+ javax.cache*;resolution:=optional,
+ jodd.bean*;resolution:=optional,
+ jdk.internal.misc;resolution:=optional,
+ net.jpountz*;resolution:=optional,
+ org.bouncycastle*;resolution:=optional,
+ org.conscrypt*;resolution:=optional,
+ org.eclipse.jetty*;resolution:=optional,
+ org.jboss*;resolution:=optional,
+ com.fasterxml.jackson.dataformat.avro;resolution:=optional,
+ com.fasterxml.jackson.dataformat.cbor;resolution:=optional,
+ com.fasterxml.jackson.dataformat.ion;resolution:=optional,
+ com.fasterxml.jackson.dataformat.smile;resolution:=optional,
+ com.fasterxml.jackson.dataformat.yaml;resolution:=optional,
+ org.msgpack.jackson*;resolution:=optional,
+ org.nustaq*;resolution:=optional,
+ org.reactivestreams*;resolution:=optional,
+ org.springframework*;resolution:=optional,
+ javax.annotation*;resolution:=optional,
+ kotlin*;resolution:=optional,
+ sun*;resolution:=optional,
+ *
</Import-Package>
<Private-Package>
org.redisson*,
- org.apache.karaf.decanter.appender.utils
+ com.github.benmanes.caffeine.cache,
+ io.micrometer.core*,
+ io.netty*,
+ lzma*,
+ net.bytebuddy*,
+ org.xerial.snappy*,
+ reactor*,
+ org.apache.karaf.decanter.appender.utils,
+ org.apache.karaf.decanter.appender.redis
</Private-Package>
<_dsannotations>*</_dsannotations>
</instructions>
diff --git a/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java b/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java
index 34f9ae6..048132f 100644
--- a/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java
+++ b/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java
@@ -65,7 +65,6 @@ public class RedisAppender implements EventHandler {
String address = getValue(config, ADDRESS_PROPERTY, ADDRESS_DEFAULT);
String mode = getValue(config, MODE_PROPERTY, MODE_DEFAULT);
- String map = getValue(config, MAP_PROPERTY, MAP_DEFAULT);
String masterAddress = getValue(config, MASTER_ADDRESS_PROPERTY, MASTER_ADDRESS_DEFAULT);
String masterName = getValue(config, MASTER_NAME_PROPERTY, MASTER_NAME_DEFAULT);
int scanInterval = Integer.parseInt(getValue(config, SCAN_INTERVAL_PROPERTY, SCAN_INTERVAL_DEFAULT));
diff --git a/assembly/src/main/feature/feature.xml b/assembly/src/main/feature/feature.xml
index 1edfbe3..549a7d3 100644
--- a/assembly/src/main/feature/feature.xml
+++ b/assembly/src/main/feature/feature.xml
@@ -220,13 +220,26 @@ org.apache.felix.eventadmin.IgnoreTimeout=org.apache.karaf.decanter.
<feature>decanter-collector-prometheus-core</feature>
</feature>
+ <feature name="decanter-collector-redis-core" version="${project.version}" description="Karaf Decanter Redis Collector core">
+ <feature>decanter-common</feature>
+ <feature>scheduler</feature>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/2.10.2</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/2.10.2</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.2</bundle>
+ <bundle>mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.redis/${project.version}</bundle>
+ </feature>
+
+ <feature name="decanter-collector-redis" version="${project.version}" description="Karaf Decanter Redis Collector">
+ <configfile finalname="/etc/org.apache.karaf.decanter.collector.redis.cfg">mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.redis/${project.version}/cfg</configfile>
+ <feature>decanter-collector-redis-core</feature>
+ </feature>
+
<feature name="decanter-appender-log" version="${project.version}" description="Karaf Decanter Log Appender">
<feature>decanter-common</feature>
<configfile finalname="/etc/org.apache.karaf.decanter.appender.log.cfg">mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.log/${project.version}/cfg</configfile>
<bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.log/${project.version}</bundle>
</feature>
-
<feature name="decanter-appender-elasticsearch-core" version="${project.version}" description="Karaf Decanter Elasticsearch Rest Appender core">
<feature>decanter-common</feature>
<feature>http</feature>
@@ -343,6 +356,9 @@ org.apache.felix.eventadmin.IgnoreTimeout=org.apache.karaf.decanter.
<feature name="decanter-appender-redis-core" version="${project.version}" description="Karaf Decanter Redis Appender core">
<feature>decanter-common</feature>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/2.10.2</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/2.10.2</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-annotations/2.10.2</bundle>
<bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.redis/${project.version}</bundle>
</feature>
diff --git a/collector/pom.xml b/collector/pom.xml
index dca3c42..3cbbec4 100644
--- a/collector/pom.xml
+++ b/collector/pom.xml
@@ -48,6 +48,7 @@
<module>mqtt</module>
<module>oshi</module>
<module>process</module>
+ <module>redis</module>
<module>rest</module>
<module>rest-servlet</module>
<module>soap</module>
diff --git a/collector/redis/pom.xml b/collector/redis/pom.xml
new file mode 100644
index 0000000..d39478b
--- /dev/null
+++ b/collector/redis/pom.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.karaf.decanter</groupId>
+ <artifactId>collector</artifactId>
+ <version>2.4.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.karaf.decanter.collector</groupId>
+ <artifactId>org.apache.karaf.decanter.collector.redis</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Karaf :: Decanter :: Collector :: Redis</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.redisson</groupId>
+ <artifactId>redisson</artifactId>
+ <version>3.12.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.decanter.collector</groupId>
+ <artifactId>org.apache.karaf.decanter.collector.utils</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>src/main/cfg/org.apache.karaf.decanter.collector.redis.cfg</file>
+ <type>cfg</type>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <inherited>true</inherited>
+ <extensions>true</extensions>
+ <configuration>
+ <obrRepository>NONE</obrRepository>
+ <instructions>
+ <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
+ <Export-Package>!*</Export-Package>
+ <Import-Package>
+ com.esotericsoftware.kryo*;resolution:=optional,
+ !com.github.benmanes.caffeine.cache,
+ !io.micrometer.core*,
+ !io.netty*,
+ !lzma*,
+ !net.bytebuddy*,
+ !org.xerial.snappy*,
+ !reactor*,
+ com.google.protobuf*;resolution:=optional,
+ com.jcraft.jzlib;resolution:=optional,
+ com.ning.compress*;resolution:=optional,
+ com.oracle.svm*;resolution:=optional,
+ com.esotericsoftware.kryo*;resolution:=optional,
+ io.reactivex*;resolution:=optional,
+ javax.cache*;resolution:=optional,
+ jodd.bean*;resolution:=optional,
+ jdk.internal.misc;resolution:=optional,
+ net.jpountz*;resolution:=optional,
+ org.bouncycastle*;resolution:=optional,
+ org.conscrypt*;resolution:=optional,
+ org.eclipse.jetty*;resolution:=optional,
+ org.jboss*;resolution:=optional,
+ com.fasterxml.jackson.dataformat.avro;resolution:=optional,
+ com.fasterxml.jackson.dataformat.cbor;resolution:=optional,
+ com.fasterxml.jackson.dataformat.ion;resolution:=optional,
+ com.fasterxml.jackson.dataformat.smile;resolution:=optional,
+ com.fasterxml.jackson.dataformat.yaml;resolution:=optional,
+ org.msgpack.jackson*;resolution:=optional,
+ org.nustaq*;resolution:=optional,
+ org.reactivestreams*;resolution:=optional,
+ org.springframework*;resolution:=optional,
+ javax.annotation*;resolution:=optional,
+ kotlin*;resolution:=optional,
+ sun*;resolution:=optional,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.redisson*,
+ com.github.benmanes.caffeine.cache,
+ io.micrometer.core*,
+ io.netty*,
+ lzma*,
+ net.bytebuddy*,
+ org.xerial.snappy*,
+ reactor*,
+ org.apache.karaf.decanter.collector.redis,
+ org.apache.karaf.decanter.collector.utils
+ </Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/collector/redis/src/main/cfg/org.apache.karaf.decanter.collector.redis.cfg b/collector/redis/src/main/cfg/org.apache.karaf.decanter.collector.redis.cfg
new file mode 100644
index 0000000..0e46744
--- /dev/null
+++ b/collector/redis/src/main/cfg/org.apache.karaf.decanter.collector.redis.cfg
@@ -0,0 +1,67 @@
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+########################################
+# Decanter Redis Collector Configuration
+########################################
+
+#
+# Location of the Redis broker
+# It's possible to use a list of brokers, for instance:
+# host= locahost:6389,localhost:6332,localhost:6419
+#
+# Default is localhost:6379
+#
+address=localhost:6379
+
+#
+# Define the connection mode.
+# Possible modes: Single (default), Master_Slave, Sentinel, Cluster
+#
+mode=Single
+
+#
+# Name of the Redis map
+# Default is Decanter
+#
+map=Decanter
+
+#
+# For Master_Slave mode, we define the location of the master
+# Default is localhost:6379
+#
+#masterAddress=localhost:6379
+
+#
+# For Sentinel model, define the name of the master
+# Default is myMaster
+#
+#masterName=myMaster
+
+#
+# For Cluster mode, define the scan interval of the nodes in the cluster
+# Default value is 2000 (2 seconds).
+#
+#scanInterval=2000
+
+#
+# Key pattern to looking for.
+# Default is *
+#
+#keyPattern=*
\ No newline at end of file
diff --git a/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java b/collector/redis/src/main/java/org/apache/karaf/decanter/collector/redis/RedisCollector.java
similarity index 54%
copy from appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java
copy to collector/redis/src/main/java/org/apache/karaf/decanter/collector/redis/RedisCollector.java
index 34f9ae6..8dff199 100644
--- a/appender/redis/src/main/java/org/apache/karaf/decanter/appender/redis/RedisAppender.java
+++ b/collector/redis/src/main/java/org/apache/karaf/decanter/collector/redis/RedisCollector.java
@@ -14,46 +14,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.karaf.decanter.appender.redis;
+package org.apache.karaf.decanter.collector.redis;
-import org.apache.karaf.decanter.appender.utils.EventFilter;
+import org.apache.karaf.decanter.collector.utils.PropertiesPreparator;
import org.osgi.service.component.ComponentContext;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
import org.osgi.service.event.Event;
-import org.osgi.service.event.EventConstants;
-import org.osgi.service.event.EventHandler;
+import org.osgi.service.event.EventAdmin;
import org.redisson.Redisson;
+import org.redisson.api.RMap;
import org.redisson.api.RedissonClient;
import org.redisson.config.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Dictionary;
+import java.util.HashMap;
import java.util.Map;
-/**
- * Redis appender
- */
@Component(
- configurationPid = "org.apache.karaf.decanter.appender.redis",
- service = EventHandler.class,
- property = {EventConstants.EVENT_TOPIC + "=decanter/collect/*" }
+ name = "org.apache.karaf.decanter.collector.redis",
+ immediate = true,
+ property = { "decanter.collector.name=redis",
+ "scheduler.period:Long=60",
+ "scheduler.concurrent:Boolean=false",
+ "scheduler.name=decanter-collector-redis"}
)
-public class RedisAppender implements EventHandler {
+public class RedisCollector implements Runnable {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(RedisCollector.class);
- public static final String ADDRESS_PROPERTY = "address";
- public static final String MODE_PROPERTY = "mode";
- public static final String MAP_PROPERTY = "map";
- public static final String MASTER_ADDRESS_PROPERTY = "masterAddress";
- public static final String MASTER_NAME_PROPERTY = "masterName";
- public static final String SCAN_INTERVAL_PROPERTY = "scanInterval";
+ @Reference
+ private EventAdmin dispatcher;
public static final String ADDRESS_DEFAULT = "localhost:6379";
public static final String MODE_DEFAULT = "Single";
- public static final String MAP_DEFAULT = "Decanter";
- public static final String MASTER_ADDRESS_DEFAULT = null;
- public static final String MASTER_NAME_DEFAULT = null;
- public static final String SCAN_INTERVAL_DEFAULT = "2000";
private RedissonClient redissonClient;
@@ -63,12 +61,11 @@ public class RedisAppender implements EventHandler {
public void activate(ComponentContext componentContext) {
config = componentContext.getProperties();
- String address = getValue(config, ADDRESS_PROPERTY, ADDRESS_DEFAULT);
- String mode = getValue(config, MODE_PROPERTY, MODE_DEFAULT);
- String map = getValue(config, MAP_PROPERTY, MAP_DEFAULT);
- String masterAddress = getValue(config, MASTER_ADDRESS_PROPERTY, MASTER_ADDRESS_DEFAULT);
- String masterName = getValue(config, MASTER_NAME_PROPERTY, MASTER_NAME_DEFAULT);
- int scanInterval = Integer.parseInt(getValue(config, SCAN_INTERVAL_PROPERTY, SCAN_INTERVAL_DEFAULT));
+ String address = (config.get("address") != null) ? config.get("address").toString() : ADDRESS_DEFAULT;
+ String mode = (config.get("map") != null) ? config.get("map").toString() : MODE_DEFAULT;
+ String masterAddress = (config.get("masterAddress") != null) ? config.get("masterAddress").toString() : null;
+ String masterName = (config.get("masterName") != null) ? config.get("masterName").toString() : null;
+ int scanInterval = (config.get("scanInterval") != null) ? Integer.parseInt(config.get("scanInterval").toString()) : 2000;
Config redissonConfig = new Config();
if (mode.equalsIgnoreCase("Single")) {
@@ -91,17 +88,21 @@ public class RedisAppender implements EventHandler {
}
@Override
- public void handleEvent(Event event) {
- if (EventFilter.match(event, config)) {
- Map<String, Object> redisMap = redissonClient.getMap(getValue(config, MAP_PROPERTY, MAP_DEFAULT));
- for (String name : event.getPropertyNames()) {
- redisMap.put(name, event.getProperty(name));
- }
+ public void run() {
+ Map<String, Object> data = new HashMap<>();
+ data.put("type", "redis");
+ String map = (config.get("map") != null) ? config.get("map").toString() : "Decanter";
+ String keyPattern = (config.get("keyPattern") != null) ? config.get("keyPattern").toString() : "*";
+ RMap rmap = redissonClient.getMap(map);
+ for (Object key : rmap.keySet(keyPattern)) {
+ data.put(key.toString(), rmap.get(key));
}
- }
-
- private String getValue(Dictionary<String, Object> properties, String key, String defaultValue) {
- return (properties.get(key) != null) ? (String) properties.get(key) : defaultValue;
+ try {
+ PropertiesPreparator.prepare(data, config);
+ } catch (Exception e) {
+ LOGGER.warn("Can't prepare data", e);
+ }
+ dispatcher.postEvent(new Event("decanter/collect/redis", data));
}
}
diff --git a/manual/src/main/asciidoc/user-guide/collectors.adoc b/manual/src/main/asciidoc/user-guide/collectors.adoc
index b3bcb64..de2c07d 100644
--- a/manual/src/main/asciidoc/user-guide/collectors.adoc
+++ b/manual/src/main/asciidoc/user-guide/collectors.adoc
@@ -928,7 +928,7 @@ karaf@root()> feature:install decanter-collector-configadmin
==== Prometheus
-The Decanter Prometheus collectors is able to periodically (scheduled collector) read Prometheus servlet output to create events sent in Decanter.
+The Decanter Prometheus collector is able to periodically (scheduled collector) read Prometheus servlet output to create events sent in Decanter.
The `decanter-collector-prometheus` feature installs the Prometheus collector:
@@ -944,6 +944,61 @@ prometheus.url=http://host/prometheus
The `prometheus.url` property is mandatory and define the location of the Prometheus export servlet (that could be provided by the Decanter Prometheus appender for instance).
+==== Redis
+
+The Decanter Redis collector is able to periodically (scheduled collector) read Redis Map to get key/value pairs.
+You can filter the keys you want thanks to key pattern.
+
+The `decanter-collector-redis` feature installs the Redis collector:
+
+----
+karaf@root()> feature:install decanter-collector-redis
+----
+
+The feature also installs the `etc/org.apache.karaf.decanter.collector.redis.cfg` configuration file containing:
+
+----
+address=localhost:6379
+
+#
+# Define the connection mode.
+# Possible modes: Single (default), Master_Slave, Sentinel, Cluster
+#
+mode=Single
+
+#
+# Name of the Redis map
+# Default is Decanter
+#
+map=Decanter
+
+#
+# For Master_Slave mode, we define the location of the master
+# Default is localhost:6379
+#
+#masterAddress=localhost:6379
+
+#
+# For Sentinel model, define the name of the master
+# Default is myMaster
+#
+#masterName=myMaster
+
+#
+# For Cluster mode, define the scan interval of the nodes in the cluster
+# Default value is 2000 (2 seconds).
+#
+#scanInterval=2000
+
+#
+# Key pattern to looking for.
+# Default is *
+#
+#keyPattern=*
+----
+
+You can configure the Redis connection (depending of the topology) and the key pattern in this configuration file.
+
==== Customizing properties in collectors
You can add, rename or remove properties collected by the collectors before sending it to the dispatcher.