You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2018/02/17 16:16:54 UTC
incubator-unomi git commit: UNOMI-162 Add support for ElasticSearch
X-Pack plugin
Repository: incubator-unomi
Updated Branches:
refs/heads/master 80d1b5862 -> 82e1d143d
UNOMI-162 Add support for ElasticSearch X-Pack plugin
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/82e1d143
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/82e1d143
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/82e1d143
Branch: refs/heads/master
Commit: 82e1d143dad282c243b02488c87113902ac0d0e9
Parents: 80d1b58
Author: Serge Huber <sh...@apache.org>
Authored: Sat Feb 17 17:16:50 2018 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Sat Feb 17 17:16:50 2018 +0100
----------------------------------------------------------------------
.../elasticsearch/ChildFirstClassLoader.java | 63 +++++++++++++++
.../ElasticSearchPersistenceServiceImpl.java | 85 ++++++++++++++++++--
.../resources/OSGI-INF/blueprint/blueprint.xml | 6 ++
.../markdown/versions/master/configuration.md | 50 +++++++++++-
4 files changed, 198 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/82e1d143/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java
new file mode 100644
index 0000000..8981df1
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.persistence.elasticsearch;
+import java.net.URL;
+import java.net.URLClassLoader;
+
+/**
+ * This class loader will always try to load classes first from the child URL class loader and will only resort to the
+ * parent class loader if the class coudln't be found.
+ */
+public class ChildFirstClassLoader extends ClassLoader {
+
+ private ChildFirstURLClassLoader childFirstURLClassLoader;
+
+ private static class ChildFirstURLClassLoader extends URLClassLoader {
+ private ClassLoader parentClassLoader;
+
+ public ChildFirstURLClassLoader(URL[] urls, ClassLoader parentClassLoader) {
+ super(urls, null);
+
+ this.parentClassLoader = parentClassLoader;
+ }
+
+ @Override
+ public Class<?> findClass(String name) throws ClassNotFoundException {
+ try {
+ return super.findClass(name);
+ } catch (ClassNotFoundException e) {
+ return parentClassLoader.loadClass(name);
+ }
+ }
+ }
+
+ public ChildFirstClassLoader(ClassLoader parent, URL[] urls) {
+ super(parent);
+ childFirstURLClassLoader = new ChildFirstURLClassLoader(urls, parent);
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve)
+ throws ClassNotFoundException {
+ try {
+ return childFirstURLClassLoader.loadClass(name);
+ } catch (ClassNotFoundException e) {
+ return super.loadClass(name, resolve);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/82e1d143/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 9ba61a7..98b66e6 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -92,9 +92,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.text.ParseException;
@@ -150,6 +154,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String aggregateQueryBucketSize = "5000";
+ private String transportClientClassName = null;
+ private String transportClientProperties = null;
+ private String transportClientJarDirectory = null;
+
private MetricsService metricsService;
private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>();
@@ -246,6 +254,18 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.aggregateQueryBucketSize = aggregateQueryBucketSize;
}
+ public void setTransportClientClassName(String transportClientClassName) {
+ this.transportClientClassName = transportClientClassName;
+ }
+
+ public void setTransportClientProperties(String transportClientProperties) {
+ this.transportClientProperties = transportClientProperties;
+ }
+
+ public void setTransportClientJarDirectory(String transportClientJarDirectory) {
+ this.transportClientJarDirectory = transportClientJarDirectory;
+ }
+
public void setMetricsService(MetricsService metricsService) {
this.metricsService = metricsService;
}
@@ -273,12 +293,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.info("Overriding cluster name from system property=" + clusterName);
}
- Settings transportSettings = Settings.builder()
- .put(CLUSTER_NAME, clusterName).build();
+ Settings.Builder transportSettings = Settings.builder()
+ .put(CLUSTER_NAME, clusterName);
- logger.info("Connecting to ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "...");
-
- client = new PreBuiltTransportClient(transportSettings);
+ if (transportClientClassName != null && transportClientClassName.trim().length() > 0 &&
+ transportClientJarDirectory != null && transportClientJarDirectory.trim().length() > 0) {
+ logger.info("Connecting to ElasticSearch persistence backend using transport class " + transportClientClassName +
+ " with JAR directory "+transportClientJarDirectory +
+ " using cluster name " + clusterName + " and index name " + indexName + "...");
+ client = newTransportClient(transportSettings, transportClientClassName, transportClientJarDirectory, transportClientProperties);
+ } else {
+ logger.info("Connecting to ElasticSearch persistence backend using cluster name " + clusterName + " and index name " + indexName + "...");
+ client = new PreBuiltTransportClient(transportSettings.build());
+ }
for (String elasticSearchAddress : elasticSearchAddressList) {
String[] elasticSearchAddressParts = elasticSearchAddress.split(":");
String elasticSearchHostName = elasticSearchAddressParts[0];
@@ -1759,5 +1786,53 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
+ public TransportClient newTransportClient(Settings.Builder settingsBuilder,
+ String transportClientClassName,
+ String transportClientJarDirectory,
+ String transportClientProperties) {
+
+ ArrayList<URL> urls = new ArrayList<>();
+ File pluginLocationFile = new File(transportClientJarDirectory);
+
+ File[] pluginLocationFiles = pluginLocationFile.listFiles();
+ for (File pluginFile : pluginLocationFiles) {
+ if (pluginFile.getName().toLowerCase().endsWith(".jar")) {
+ try {
+ urls.add(pluginFile.toURI().toURL());
+ } catch (MalformedURLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ ChildFirstClassLoader childFirstClassLoader = new ChildFirstClassLoader(this.getClass().getClassLoader(), urls.toArray(new URL[urls.size()]));
+
+ if (transportClientProperties != null && transportClientProperties.trim().length() > 0) {
+ String[] clientProperties = transportClientProperties.split(",");
+ if (clientProperties.length > 0) {
+ for (String clientProperty : clientProperties) {
+ String[] clientPropertyParts = clientProperty.split("=");
+ settingsBuilder.put(clientPropertyParts[0], clientPropertyParts[1]);
+ }
+ }
+ }
+
+ try {
+ Class<?> transportClientClass = childFirstClassLoader.loadClass(transportClientClassName);
+ Constructor<?> transportClientConstructor = transportClientClass.getConstructor(Settings.class, Class[].class);
+ return (TransportClient) transportClientConstructor.newInstance(settingsBuilder.build(), new Class[0]);
+ } catch (ClassNotFoundException e) {
+ logger.error("Couldn't find class " + transportClientClassName, e);
+ } catch (NoSuchMethodException e) {
+ logger.error("Error creating transport client with class" + transportClientClassName, e);
+ } catch (IllegalAccessException e) {
+ logger.error("Error creating transport client with class" + transportClientClassName, e);
+ } catch (InstantiationException e) {
+ logger.error("Error creating transport client with class" + transportClientClassName, e);
+ } catch (InvocationTargetException e) {
+ logger.error("Error creating transport client with class" + transportClientClassName, e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/82e1d143/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 92871eb..0149a01 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -50,6 +50,9 @@
<cm:property name="aggregateQueryBucketSize" value="5000" />
+ <cm:property name="transportClientClassName" value="" />
+ <cm:property name="transportClientJarDirectory" value="" />
+ <cm:property name="transportClientProperties" value="" />
</cm:default-properties>
</cm:property-placeholder>
@@ -109,6 +112,9 @@
<property name="maximalElasticSearchVersion" value="${es.maximalElasticSearchVersion}" />
<property name="aggregateQueryBucketSize" value="${es.aggregateQueryBucketSize}" />
+ <property name="transportClientClassName" value="${es.transportClientClassName}" />
+ <property name="transportClientJarDirectory" value="${es.transportClientJarDirectory}" />
+ <property name="transportClientProperties" value="${es.transportClientProperties}" />
<property name="metricsService" ref="metricsService" />
</bean>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/82e1d143/src/site/markdown/versions/master/configuration.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/versions/master/configuration.md b/src/site/markdown/versions/master/configuration.md
index a4068e0..a5d97fb 100644
--- a/src/site/markdown/versions/master/configuration.md
+++ b/src/site/markdown/versions/master/configuration.md
@@ -299,4 +299,52 @@ The Apache Karaf SSH console is available inside Apache Unomi, but the port has
ssh -p 8102 karaf@localhost
-or the user/password you have setup to protect the system if you have changed it.
\ No newline at end of file
+or the user/password you have setup to protect the system if you have changed it.
+
+ElasticSearch X-Pack Support
+--------------------------------------
+
+It is now possible to use X-Pack to connect to ElasticSearch. However, for licensing reasons this is not provided out
+of the box. Here is the procedure to install X-Pack with Apache Unomi:
+
+#Important !
+
+Do not start Unomi directly with unomi:start, perform the following steps below first !
+
+#Installation steps
+
+1. Create a directory for all the JARs that you will download, we will call it XPACK_JARS_DIRECTORY
+2. Download https://artifacts.elastic.co/maven/org/elasticsearch/client/x-pack-transport/5.6.3/x-pack-transport-5.6.3.jar to XPACK_JARS_DIRECTORY
+3. Download https://artifacts.elastic.co/maven/org/elasticsearch/plugin/x-pack-api/5.6.3/x-pack-api-5.6.3.jar to XPACK_JARS_DIRECTORY
+4. Download http://central.maven.org/maven2/com/unboundid/unboundid-ldapsdk/3.2.0/unboundid-ldapsdk-3.2.0.jar to XPACK_JARS_DIRECTORY
+5. Download http://central.maven.org/maven2/org/bouncycastle/bcpkix-jdk15on/1.55/bcpkix-jdk15on-1.55.jar to XPACK_JARS_DIRECTORY
+6. Download http://central.maven.org/maven2/org/bouncycastle/bcprov-jdk15on/1.55/bcprov-jdk15on-1.55.jar to XPACK_JARS_DIRECTORY
+7. Download http://central.maven.org/maven2/com/sun/mail/javax.mail/1.5.3/javax.mail-1.5.3.jar to XPACK_JARS_DIRECTORY
+8. Edit etc/org.apache.unomi.persistence.elasticsearch.cfg to add the following settings:
+
+ transportClientClassName=org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
+ transportClientJarDirectory=XPACK_JARS_DIRECTORY
+ transportClientProperties=xpack.security.user=elastic:changeme
+
+ You can setup more properties (for example for SSL/TLS support) by seperating the properties with commas,
+ as in the following example:
+
+ transportClientProperties=xpack.security.user=elastic:changeme,xpack.ssl.key=/home/user/elasticsearch-5.6.3/config/x-pack/localhost/localhost.key,xpack.ssl.certificate=/home/user/elasticsearch-5.6.3/config/x-pack/localhost/localhost.crt,xpack.ssl.certificate_authorities=/home/user/elasticsearch-5.6.3/config/x-pack/ca/ca.crt,xpack.security.transport.ssl.enabled=true
+
+9. Launch Karaf and launch unomi using the command from the shell :
+
+ unomi:start
+
+Alternatively you could edit the configuration directly from the Karaf shell using the following commands:
+
+ config:edit org.apache.unomi.persistence.elasticsearch
+ config:property-set transportClientClassName org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
+ config:property-set transportClientJarDirectory XPACK_JARS_DIRECTORY
+ config:property-set transportClientProperties xpack.security.user=elastic:changeme
+ config:update
+ unomi:start
+
+You can setup more properties (for example for SSL/TLS support) by seperating the properties with commas,
+as in the following example:
+
+ config:property-set transportClientProperties xpack.security.user=elastic:changeme,xpack.ssl.key=/home/user/elasticsearch-5.6.3/config/x-pack/localhost/localhost.key,xpack.ssl.certificate=/home/user/elasticsearch-5.6.3/config/x-pack/localhost/localhost.crt,xpack.ssl.certificate_authorities=/home/user/elasticsearch-5.6.3/config/x-pack/ca/ca.crt,xpack.security.transport.ssl.enabled=true