You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2018/01/13 05:23:56 UTC
[1/2] storm git commit: STORM-2860: Add Kerberos support to Solr bolt
Repository: storm
Updated Branches:
refs/heads/1.x-branch 7afea5807 -> 474b9af67
STORM-2860: Add Kerberos support to Solr bolt
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1d3a875
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1d3a875
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1d3a875
Branch: refs/heads/1.x-branch
Commit: f1d3a875a9dd3e4c5b7d837bc1df10f0da022833
Parents: b8f76af
Author: Manikumar Reddy O <ma...@gmail.com>
Authored: Mon Jan 1 14:43:15 2018 +0530
Committer: Manikumar Reddy O <ma...@gmail.com>
Committed: Mon Jan 8 11:30:00 2018 +0530
----------------------------------------------------------------------
examples/storm-solr-examples/pom.xml | 21 +++++++++++++++
external/storm-solr/README.md | 26 +++++++++++++++++-
external/storm-solr/pom.xml | 27 ++++++++++++-------
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 28 +++++++++++++++-----
.../apache/storm/solr/config/SolrConfig.java | 14 ++++++++++
.../storm/solr/mapper/SolrJsonMapper.java | 1 +
.../schema/builder/RestJsonSchemaBuilder.java | 1 +
.../apache/storm/solr/trident/SolrState.java | 4 +++
8 files changed, 105 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/examples/storm-solr-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-solr-examples/pom.xml b/examples/storm-solr-examples/pom.xml
index b4864c7..4f44fff 100644
--- a/examples/storm-solr-examples/pom.xml
+++ b/examples/storm-solr-examples/pom.xml
@@ -38,6 +38,27 @@
<groupId>org.apache.storm</groupId>
<artifactId>storm-solr</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.1.3</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/README.md
----------------------------------------------------------------------
diff --git a/external/storm-solr/README.md b/external/storm-solr/README.md
index 915258e..c67e55a 100644
--- a/external/storm-solr/README.md
+++ b/external/storm-solr/README.md
@@ -97,6 +97,30 @@ field separates each value with the token % instead of the default | . To use th
.setMultiValueFieldToken("%").build();
```
+##Working with Kerberized Solr
+If your topology is going to interact with kerberized Solr, your bolts/states need to be authenticated by Solr Server. We can enable
+authentication by distributing keytabs for solr user on all worker hosts. We can configure the solr bolt to use keytabs by setting
+SolrConfig.enableKerberos config property. Currently only SolrJsonMapper can be used to interact with kerberized Solr. Please check
+new versions to use SolrFieldsMapper.
+
+On worker hosts the bolt/trident-state code will use the keytab file with principal provided in the jaas config to authenticate with
+Solr. You need to specify a Kerberos principal for the client and a corresponding keytab in the JAAS client configuration file.
+Also make sure the provided principal is configured with required permissions to access solr collections.
+
+Here’s an example JAAS config:
+
+`SolrJClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="/keytabs/solr.keytab"
+ storeKey=true
+ useTicketCache=true
+ debug=true
+ principal="solrclient@EXAMPLE.COM";
+};
+`
+
+
# Build And Run Bundled Examples
To be able to run the examples you must first build the java code in the package `storm-solr`,
and then generate an uber jar with all the dependencies.
@@ -171,7 +195,7 @@ Querying Solr for these patterns, you will see the values that have been indexe
curl -X GET -H "Content-type:application/json" -H "Accept:application/json" http://localhost:8983/solr/gettingstarted_shard1_replica2/select?q=*id_fields_test_val*&wt=json&indent=true
-curl -X GET -H "Content-type: application/json" -H "Accept: application/json" http://localhost:8983/solr/gettingstarted_shard1_replica2/select?q=*id_fields_test_val*&wt=json&indent=true
+curl -X GET -H "Content-type:application/json" -H "Accept:application/json" http://localhost:8983/solr/gettingstarted_shard1_replica2/select?q=*json_test_val*&wt=json&indent=true
You can also see the results by opening the Apache Solr UI and pasting the `id` pattern in the `q` textbox in the queries page
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-solr/pom.xml b/external/storm-solr/pom.xml
index 684cbc5..6d6cf24 100644
--- a/external/storm-solr/pom.xml
+++ b/external/storm-solr/pom.xml
@@ -35,6 +35,10 @@
</developer>
</developers>
+ <properties>
+ <solr.version>5.5.5</solr.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
@@ -46,8 +50,18 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>5.2.1</version>
- <scope>compile</scope>
+ <version>${solr.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
@@ -55,11 +69,6 @@
<version>1.3</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.1</version>
- </dependency>
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>1.4</version>
@@ -67,13 +76,13 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>5.2.1</version>
+ <version>${solr.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
- <version>5.2.1</version>
+ <version>${solr.version}</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
index 4feaaf5..9eb4ade 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
@@ -18,21 +18,22 @@
package org.apache.storm.solr.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.storm.solr.config.CountBasedCommit;
import org.apache.storm.solr.config.SolrCommitStrategy;
import org.apache.storm.solr.config.SolrConfig;
import org.apache.storm.solr.mapper.SolrMapper;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.utils.TupleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,6 +75,8 @@ public class SolrUpdateBolt extends BaseTickTupleAwareRichBolt {
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
+ if (solrConfig.isKerberosEnabled())
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
this.solrClient = new CloudSolrClient(solrConfig.getZkHostString());
this.toCommitTuples = new ArrayList<>(capacity());
}
@@ -153,4 +156,15 @@ public class SolrUpdateBolt extends BaseTickTupleAwareRichBolt {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) { }
+ @Override
+ public void cleanup() {
+ if (solrClient != null) {
+ try {
+ solrClient.close();
+ } catch (IOException e) {
+ LOG.error("Error while closing solrClient", e);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
index 1803a96..ecb3630 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
@@ -29,6 +29,7 @@ import java.io.Serializable;
public class SolrConfig implements Serializable {
private final String zkHostString;
private final int tickTupleInterval;
+ private final boolean enableKerberos;
/**
* @param zkHostString Zookeeper host string as defined in the {@link CloudSolrClient} constructor
@@ -42,8 +43,18 @@ public class SolrConfig implements Serializable {
* @param tickTupleInterval interval for tick tuples
* */
public SolrConfig(String zkHostString, int tickTupleInterval) {
+ this(zkHostString, tickTupleInterval, false);
+ }
+
+ /**
+ * @param zkHostString Zookeeper host string as defined in the {@link CloudSolrClient} constructor
+ * @param tickTupleInterval interval for tick tuples
+ * @param enableKerberos true to enable kerberos else false
+ * */
+ public SolrConfig(String zkHostString, int tickTupleInterval, boolean enableKerberos) {
this.zkHostString = zkHostString;
this.tickTupleInterval = tickTupleInterval;
+ this.enableKerberos = enableKerberos;
}
public String getZkHostString() {
@@ -54,4 +65,7 @@ public class SolrConfig implements Serializable {
return tickTupleInterval;
}
+ public boolean isKerberosEnabled() {
+ return enableKerberos;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
index 704ec2d..0c39cb5 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
@@ -58,6 +58,7 @@ public class SolrJsonMapper implements SolrMapper {
* {@link SolrJsonMapper} builder class.
* @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
* @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
+ * This doesn't work in secure mode, in secure mode we need to pass the collection
*/
public Builder(SolrClient solrClient, String jsonTupleField) {
String defaultCollection = null;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
index 662c768..a9e0b49 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
@@ -31,6 +31,7 @@ import java.util.Scanner;
* Class that builds the {@link Schema} object from the JSON representation of the schema as returned by the
* URL of the form http://localhost:8983/solr/gettingstarted/schema/ . This particular URL returns the schema
* in JSON format for the gettingstarted example running locally.
+ * This doesn't work in kerberos mode. Please check new versions for kerberos support
*/
public class RestJsonSchemaBuilder implements SchemaBuilder {
private static final Logger logger = LoggerFactory.getLogger(RestJsonSchemaBuilder.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1d3a875/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
index d84d140..aeb06b6 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
@@ -18,6 +18,8 @@
package org.apache.storm.solr.trident;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.storm.topology.FailedException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -44,6 +46,8 @@ public class SolrState implements State {
}
protected void prepare() {
+ if (solrConfig.isKerberosEnabled())
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
solrClient = new CloudSolrClient(solrConfig.getZkHostString());
}
[2/2] storm git commit: Merge branch 'kerberos-solr-bolt' of
https://github.com/omkreddy/storm into STORM-2860-1.x-merge
Posted by ka...@apache.org.
Merge branch 'kerberos-solr-bolt' of https://github.com/omkreddy/storm into STORM-2860-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/474b9af6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/474b9af6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/474b9af6
Branch: refs/heads/1.x-branch
Commit: 474b9af67fccc0ef4418c9aa7f12b4483dc78733
Parents: 7afea58 f1d3a87
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sat Jan 13 14:23:00 2018 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Sat Jan 13 14:23:00 2018 +0900
----------------------------------------------------------------------
examples/storm-solr-examples/pom.xml | 21 +++++++++++++++
external/storm-solr/README.md | 26 +++++++++++++++++-
external/storm-solr/pom.xml | 27 ++++++++++++-------
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 28 +++++++++++++++-----
.../apache/storm/solr/config/SolrConfig.java | 14 ++++++++++
.../storm/solr/mapper/SolrJsonMapper.java | 1 +
.../schema/builder/RestJsonSchemaBuilder.java | 1 +
.../apache/storm/solr/trident/SolrState.java | 4 +++
8 files changed, 105 insertions(+), 17 deletions(-)
----------------------------------------------------------------------