You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2019/08/28 16:47:57 UTC
[metron] branch feature/METRON-2088-support-hdp-3.1 updated:
METRON-2225 Upgrade to Solr 7.4.0 (merrimanr via nickwallen) closes
apache/metron#1492
This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch feature/METRON-2088-support-hdp-3.1
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/feature/METRON-2088-support-hdp-3.1 by this push:
new ad71c04 METRON-2225 Upgrade to Solr 7.4.0 (merrimanr via nickwallen) closes apache/metron#1492
ad71c04 is described below
commit ad71c0468888977f1b3ab1aa58fea38b846a1e37
Author: merrimanr <me...@gmail.com>
AuthorDate: Wed Aug 28 12:47:40 2019 -0400
METRON-2225 Upgrade to Solr 7.4.0 (merrimanr via nickwallen) closes apache/metron#1492
---
dependencies_with_url.csv | 4 +-
.../ansible/roles/solr/defaults/main.yml | 2 +-
.../ElasticsearchSearchIntegrationTest.java | 8 ++
.../metron/indexing/dao/SearchIntegrationTest.java | 13 ++--
.../metron-solr/metron-solr-common/README.md | 24 +++---
.../metron-solr/metron-solr-common/pom.xml | 89 ++++++++++++----------
.../java/org/apache/metron/solr/dao/SolrDao.java | 5 +-
...etronSolrClient.java => SolrClientFactory.java} | 60 +++++----------
.../org/apache/metron/solr/writer/SolrWriter.java | 13 ++--
.../src/main/scripts/create_collection.sh | 2 +-
.../src/main/scripts/delete_collection.sh | 3 +
.../src/main/scripts/install_solr.sh | 2 +
.../integration/SolrMetaAlertIntegrationTest.java | 2 +-
.../integration/SolrSearchIntegrationTest.java | 19 +++++
.../solr/integration/components/SolrComponent.java | 27 +++----
.../metron/solr/writer/MetronSolrClientTest.java | 83 --------------------
.../apache/metron/solr/writer/SolrWriterTest.java | 7 +-
.../integration/SolrIndexingIntegrationTest.java | 23 ++++--
pom.xml | 3 +-
19 files changed, 165 insertions(+), 224 deletions(-)
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 9d7b0fa..66c4766 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -136,7 +136,7 @@ org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org
org.slf4j:jcl-over-slf4j:jar:1.7.7:compile,MIT,http://www.slf4j.org
org.slf4j:jcl-over-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
-org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.24:compile,MIT,http://www.slf4j.org
org.slf4j:jul-to-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
org.slf4j:jul-to-slf4j:jar:1.7.25:compile,MIT,http://www.slf4j.org
@@ -328,6 +328,7 @@ org.mortbay.jetty:jsp-2.1:jar:6.1.14:compile,ASLv2,
org.mortbay.jetty:jsp-api-2.1:jar:6.1.14:compile,ASLv2,
org.mortbay.jetty:servlet-api-2.5:jar:6.1.14:compile,ASLv2,
org.noggit:noggit:jar:0.6:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
+org.noggit:noggit:jar:0.8:compile,Apache License, Version 2.0,http://github.com/yonik/noggit
org.scannotation:scannotation:jar:1.0.3:compile,Apache License V2.0,http://scannotation.sf.net
org.slf4j:log4j-over-slf4j:jar:1.6.6:compile,Apache Software Licenses,http://www.slf4j.org
org.springframework.integration:spring-integration-core:jar:3.0.0.RELEASE:compile,The Apache Software License, Version 2.0,http://www.springintegration.org/
@@ -453,6 +454,7 @@ org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://g
org.springframework.kafka:spring-kafka:jar:2.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka
ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java
org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j
+org.locationtech.spatial4j:spatial4j:jar:0.7:compile,ASLv2,https://github.com/locationtech/spatial4j
com.github.luben:zstd-jni:jar:1.3.2-2:compile,BSD,https://github.com/luben/zstd-jni
com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE
io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
diff --git a/metron-deployment/ansible/roles/solr/defaults/main.yml b/metron-deployment/ansible/roles/solr/defaults/main.yml
index 61562f8..ca4e289 100644
--- a/metron-deployment/ansible/roles/solr/defaults/main.yml
+++ b/metron-deployment/ansible/roles/solr/defaults/main.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
---
-solr_version: 6.6.2
+solr_version: 7.4.0
solr_installation_user: "solr"
solr_user_home: /var/solr
solr_home: "{{ solr_user_home }}/solr-{{ solr_version }}"
diff --git a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 7da94e9..26e0529 100644
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -271,6 +271,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
}
@Test
+ public void different_type_facet_query() throws Exception {
+ thrown.expect(Exception.class);
+ SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
+ SearchResponse response = getIndexDao().search(request);
+ Assert.assertEquals(3, response.getTotal());
+ }
+
+ @Test
public void different_type_filter_query() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
diff --git a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index cfe5752..7aa60b7 100644
--- a/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/metron-indexing-common/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -710,14 +710,6 @@ public abstract class SearchIntegrationTest {
}
@Test
- public void different_type_facet_query() throws Exception {
- thrown.expect(Exception.class);
- SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
- SearchResponse response = getIndexDao().search(request);
- Assert.assertEquals(3, response.getTotal());
- }
-
- @Test
public void exceeding_max_results_throws_exception() throws Exception {
thrown.expect(InvalidSearchException.class);
thrown.expectMessage("Search result size must be less than 100");
@@ -934,8 +926,13 @@ public abstract class SearchIntegrationTest {
@Test
public abstract void returns_column_data_for_multiple_indices() throws Exception;
+
@Test
public abstract void returns_column_metadata_for_specified_indices() throws Exception;
+
+ @Test
+ public abstract void different_type_facet_query() throws Exception;
+
@Test
public abstract void different_type_filter_query() throws Exception;
diff --git a/metron-platform/metron-solr/metron-solr-common/README.md b/metron-platform/metron-solr/metron-solr-common/README.md
index 081436a..cf37589 100644
--- a/metron-platform/metron-solr/metron-solr-common/README.md
+++ b/metron-platform/metron-solr/metron-solr-common/README.md
@@ -27,7 +27,7 @@ limitations under the License.
## Introduction
-Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
+Metron ships with Solr 7.4.0 support. Solr Cloud can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
## Configuration
@@ -42,15 +42,15 @@ via the global config. The following settings are possible as part of the globa
* _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
worker failure *may* result data being acknowledged in storm but not written in Solr.
* `solr.commit.soft`
- * This is a boolean which defines whether the writer makes a soft commit or a durable commit. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `false`.
+ * This is a boolean which defines whether the writer makes a soft commit or a durable commit. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `false`.
* _WARNING_: If you set this to `true`, then commits will happen based on the SolrClient's internal mechanism and
worker failure *may* result data being acknowledged in storm but not written in Solr.
* `solr.commit.waitSearcher`
- * This is a boolean which defines whether the writer blocks the commit until the data is available to search. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`.
+ * This is a boolean which defines whether the writer blocks the commit until the data is available to search. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `true`.
* _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
worker failure *may* result data being acknowledged in storm but not written in Solr.
* `solr.commit.waitFlush`
- * This is a boolean which defines whether the writer blocks the commit until the data is flushed. See [here](https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits) The default is `true`.
+ * This is a boolean which defines whether the writer blocks the commit until the data is flushed. See [here](https://lucene.apache.org/solr/guide/7_4/near-real-time-searching.html) The default is `true`.
* _WARNING_: If you set this to `false`, then commits will happen based on the SolrClient's internal mechanism and
worker failure *may* result data being acknowledged in storm but not written in Solr.
* `solr.collection`
@@ -86,15 +86,11 @@ The script performs the following tasks
* Installs Solr
* Starts Solr Cloud
-_Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_
+Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/7_4/taking-solr-to-production.html
Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. After running this script,
Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner of starting Solr
-will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html)
-
-Solr can also be installed using [HDP Search 3](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.4/bk_solr-search-installation/content/ch_hdp_search_30.html). HDP Search 3 sets the Zookeeper root to
-`/solr` so this will need to be added to each url in the comma-separated list in Ambari UI -> Services -> Metron -> Configs -> Index Settings -> Solr Zookeeper Urls. For example, in full dev
-this would be `node1:2181/solr`.
+will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/7_4/getting-started-with-solrcloud.html)
## Enabling Solr
@@ -122,8 +118,8 @@ Any other collections must be created manually before starting the Indexing comp
As of now, we have mapped out the Schemas in `src/main/config/schema`.
Ambari will eventually install these, but at the moment it's manual and
-you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general
-and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
+you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/7_4](here) in general
+and [here](https://lucene.apache.org/solr/guide/7_4/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document.
Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results. For these fields a dynamic field type matching the suffix needs to be added to store the data points.
@@ -146,7 +142,7 @@ If any copy fields are defined, stored and docValues should be set to false.
Convenience scripts are provided with Metron to create and delete collections. Ambari uses these scripts to automatically create collections. To use them outside of Ambari, a few environment variables must be set first:
```
# Path to the zookeeper node used by Solr
-export ZOOKEEPER=node1:2181/solr
+export ZOOKEEPER=node1:9983
# Set to true if Kerberos is enabled
export SECURITY_ENABLED=true
```
@@ -167,4 +163,4 @@ The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/
* error
Additional schemas should be installed in that location if using the `create_collection.sh` script. Any collection can be deleted with the `delete_collection.sh` script.
-These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html).
\ No newline at end of file
+These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/7_4/collections-api.html).
\ No newline at end of file
diff --git a/metron-platform/metron-solr/metron-solr-common/pom.xml b/metron-platform/metron-solr/metron-solr-common/pom.xml
index 0346850..fd9ef01 100644
--- a/metron-platform/metron-solr/metron-solr-common/pom.xml
+++ b/metron-platform/metron-solr/metron-solr-common/pom.xml
@@ -40,6 +40,50 @@
<version>${global_solr_version}</version>
</dependency>
<dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>0.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>4.5.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-test-framework</artifactId>
+ <version>${global_solr_version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>fastutil</artifactId>
+ <groupId>it.unimi.dsi</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>caffeine</artifactId>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
@@ -93,51 +137,14 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
- </exclusions>
- </dependency>
-
- <!-- Test -->
-
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-test-framework</artifactId>
- <version>${global_solr_version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>fastutil</artifactId>
- <groupId>it.unimi.dsi</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
<exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>caffeine</artifactId>
- <groupId>com.github.ben-manes.caffeine</groupId>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${global_log4j_core_version}</version>
- <scope>test</scope>
- </dependency>
+
+ <!-- Test -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index ef68e86..0ef67a0 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -40,7 +40,7 @@ import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.solr.client.SolrClientFactory;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -170,7 +170,8 @@ public class SolrDao implements IndexDao {
}
void enableKerberos() {
- HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
+ Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder();
+ HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder());
}
public SolrSearchDao getSolrSearchDao() {
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java
similarity index 54%
rename from metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java
rename to metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java
index 5c27cce..a662558 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/MetronSolrClient.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrClientFactory.java
@@ -31,21 +31,33 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
-public class MetronSolrClient extends CloudSolrClient {
+public class SolrClientFactory {
private static final Logger LOG = LoggerFactory
- .getLogger(MetronSolrClient.class);
+ .getLogger(SolrClientFactory.class);
- public MetronSolrClient(String zkHost) {
- super(zkHost);
+ public static CloudSolrClient create(String zkHost) {
+ CloudSolrClient.Builder builder = getBuilder(zkHost);
+ return builder.build();
}
- public MetronSolrClient(String zkHost, Map<String, Object> solrHttpConfig) {
- super(zkHost, HttpClientUtil.createClient(toSolrProps(solrHttpConfig)));
+ public static CloudSolrClient create(String zkHost, Map<String, Object> solrHttpConfig) {
+ CloudSolrClient.Builder builder = getBuilder(zkHost);
+ builder.withHttpClient(HttpClientUtil.createClient(toSolrProps(solrHttpConfig)));
+ return builder.build();
+ }
+
+
+ public static CloudSolrClient.Builder getBuilder(String zkHost) {
+ String[] parts = zkHost.split("/");
+ Optional<String> zkChroot = Optional.empty();
+ if (parts.length > 1) {
+ zkChroot = Optional.of("/" + parts[1]);
+ }
+ return new CloudSolrClient.Builder(Arrays.asList(parts[0].split(",")), zkChroot);
}
public static SolrParams toSolrProps(Map<String, Object> config) {
@@ -73,36 +85,4 @@ public class MetronSolrClient extends CloudSolrClient {
}
return ret;
}
-
- public void createCollection(String name, int numShards, int replicationFactor) throws IOException, SolrServerException {
- if (!listCollections().contains(name)) {
- request(getCreateCollectionsRequest(name, numShards, replicationFactor));
- }
- }
-
- public QueryRequest getCreateCollectionsRequest(String name, int numShards, int replicationFactor) {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.CREATE.name());
- params.set(SolrConstants.REQUEST_NAME, name);
- params.set(SolrConstants.REQUEST_NUM_SHARDS, numShards);
- params.set(SolrConstants.REQUEST_REPLICATION_FACTOR, replicationFactor);
- params.set(SolrConstants.REQUEST_COLLECTION_CONFIG_NAME, name);
- QueryRequest request = new QueryRequest(params);
- request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH);
- return request;
- }
-
- @SuppressWarnings("unchecked")
- public List<String> listCollections() throws IOException, SolrServerException {
- NamedList<Object> response = request(getListCollectionsRequest(), null);
- return (List<String>) response.get(SolrConstants.RESPONSE_COLLECTIONS);
- }
-
- public QueryRequest getListCollectionsRequest() {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.LIST.name());
- QueryRequest request = new QueryRequest(params);
- request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH);
- return request;
- }
}
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
index b23a517..242c44a 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/java/org/apache/metron/solr/writer/SolrWriter.java
@@ -46,7 +46,7 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -134,9 +134,9 @@ public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable {
private String defaultCollection;
private Map<String, Object> solrHttpConfig;
- private MetronSolrClient solr;
+ private org.apache.solr.client.solrj.impl.CloudSolrClient solr;
- public SolrWriter withMetronSolrClient(MetronSolrClient solr) {
+ public SolrWriter withCloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient solr) {
this.solr = solr;
return this;
}
@@ -163,9 +163,12 @@ public class SolrWriter implements BulkMessageWriter<JSONObject>, Serializable {
LOG.info("Default Collection: {}", "" + defaultCollection );
if(solr == null) {
if (isKerberosEnabled(stormConf)) {
- HttpClientUtil.addConfigurer(new Krb5HttpClientConfigurer());
+ Krb5HttpClientBuilder krb5HttpClientBuilder = new Krb5HttpClientBuilder();
+ HttpClientUtil.setHttpClientBuilder(krb5HttpClientBuilder.getBuilder());
}
- solr = new MetronSolrClient(zookeeperUrl, solrHttpConfig);
+
+
+ solr = SolrClientFactory.create(zookeeperUrl, solrHttpConfig);
}
solr.setDefaultCollection(defaultCollection);
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh
index 7693646..087a80a 100755
--- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/create_collection.sh
@@ -33,4 +33,4 @@ SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail
zip -rj - $METRON_HOME/config/schema/$1 | curl -X POST $NEGOTIATE --header "Content-Type:text/xml" --data-binary @- "http://$SOLR_NODE/solr/admin/configs?action=UPLOAD&name=$1"
# Create the collection
-curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&numShards=1"
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&collection.configName=$1&numShards=1"
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh
index c8b45e7..3898720 100755
--- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/delete_collection.sh
@@ -31,3 +31,6 @@ SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail
# Delete the collection
curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=DELETE&name=$1"
+
+# Delete the config set
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/configs?action=DELETE&name=$1"
diff --git a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh
index da04557..352833f 100755
--- a/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh
+++ b/metron-platform/metron-solr/metron-solr-common/src/main/scripts/install_solr.sh
@@ -30,6 +30,8 @@
service kibana stop
service elasticsearch stop
+yum install -y wget zip lsof
+
SOLR_VERSION=${global_solr_version}
SOLR_USER=solr
SOLR_SERVICE=$SOLR_USER
diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
index 084660f..5f23f38 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrMetaAlertIntegrationTest.java
@@ -389,7 +389,7 @@ public class SolrMetaAlertIntegrationTest extends MetaAlertIntegrationTest {
@Override
protected void commit() throws IOException {
try {
- List<String> collections = solr.getSolrClient().listCollections();
+ List<String> collections = solr.listCollections();
for (String collection : collections) {
solr.getSolrClient().commit(collection);
}
diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
index de01df9..6c28a79 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/SolrSearchIntegrationTest.java
@@ -226,6 +226,25 @@ public class SolrSearchIntegrationTest extends SearchIntegrationTest {
}
@Test
+ public void different_type_facet_query() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFacetQuery, SearchRequest.class);
+ SearchResponse response = getIndexDao().search(request);
+ Assert.assertEquals(10, response.getTotal());
+ Assert.assertTrue(response.getFacetCounts().containsKey("ttl"));
+ Map<String, Long> facetCounts = response.getFacetCounts().get("ttl");
+ Assert.assertEquals(1L, facetCounts.get("1").longValue());
+ Assert.assertEquals(1L, facetCounts.get("2").longValue());
+ Assert.assertEquals(1L, facetCounts.get("3").longValue());
+ Assert.assertEquals(1L, facetCounts.get("4").longValue());
+ Assert.assertEquals(1L, facetCounts.get("5").longValue());
+ Assert.assertEquals(1L, facetCounts.get("data 1").longValue());
+ Assert.assertEquals(1L, facetCounts.get("data 2").longValue());
+ Assert.assertEquals(1L, facetCounts.get("data 3").longValue());
+ Assert.assertEquals(1L, facetCounts.get("data 4").longValue());
+ Assert.assertEquals(1L, facetCounts.get("data 5").longValue());
+ }
+
+ @Test
public void different_type_filter_query() throws Exception {
thrown.expect(InvalidSearchException.class);
SearchRequest request = JSONUtils.INSTANCE.load(differentTypeFilterQuery, SearchRequest.class);
diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 4bc9f8a..c91c43a 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -21,12 +21,12 @@ import com.google.common.base.Function;
import java.util.Collection;
import java.util.Map.Entry;
import java.util.stream.Collectors;
-import org.apache.metron.common.Constants;
+
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
import org.apache.metron.solr.dao.SolrUtilities;
-import org.apache.metron.solr.writer.MetronSolrClient;
+import org.apache.metron.solr.writer.SolrClientFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
@@ -105,7 +105,7 @@ public class SolrComponent implements InMemoryComponent {
for(String name: collections.keySet()) {
String configPath = collections.get(name);
miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
- CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient());
+ CollectionAdminRequest.createCollection(name, name,1, 1).process(miniSolrCloudCluster.getSolrClient());
}
if (postStartCallback != null) {
postStartCallback.apply(this);
@@ -134,8 +134,8 @@ public class SolrComponent implements InMemoryComponent {
}
}
- public MetronSolrClient getSolrClient() {
- return new MetronSolrClient(getZookeeperUrl());
+ public CloudSolrClient getSolrClient() {
+ return SolrClientFactory.create(getZookeeperUrl());
}
public MiniSolrCloudCluster getMiniSolrCloudCluster() {
@@ -149,19 +149,16 @@ public class SolrComponent implements InMemoryComponent {
public void addCollection(String name, String configPath)
throws InterruptedException, IOException, KeeperException, SolrServerException {
miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
- CollectionAdminRequest.createCollection(name, 1, 1)
+ CollectionAdminRequest.createCollection(name, name,1, 1)
.process(miniSolrCloudCluster.getSolrClient());
}
- public boolean hasCollection(String collection) {
- MetronSolrClient solr = getSolrClient();
- boolean collectionFound = false;
- try {
- collectionFound = solr.listCollections().contains(collection);
- } catch (Exception e) {
- e.printStackTrace();
- }
- return collectionFound;
+ public List<String> listCollections() throws IOException, SolrServerException {
+ return CollectionAdminRequest.listCollections(miniSolrCloudCluster.getSolrClient());
+ }
+
+ public boolean hasCollection(String collection) throws IOException, SolrServerException {
+ return listCollections().contains(collection);
}
public List<Map<String, Object>> getAllIndexedDocs(String collection) {
diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java
deleted file mode 100644
index 37807e6..0000000
--- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/MetronSolrClientTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.solr.writer;
-
-import org.apache.metron.solr.writer.MetronSolrClient;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.util.NamedList;
-import org.hamcrest.Description;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.util.ArrayList;
-
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.isNull;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-public class MetronSolrClientTest {
-
- static class CollectionRequestMatcher extends ArgumentMatcher<QueryRequest> {
-
- private String name;
-
- public CollectionRequestMatcher(String name) {
- this.name = name;
- }
-
- @Override
- public boolean matches(Object o) {
- QueryRequest queryRequest = (QueryRequest) o;
- return name.equals(queryRequest.getParams().get("action"));
- }
-
- @Override
- public void describeTo(Description description) {
- description.appendText(name);
- }
- }
-
- @Test
- public void testClient() throws Exception {
-
- final String collection = "metron";
- String zookeeperUrl = "zookeeperUrl";
- MetronSolrClient metronSolrClient = Mockito.spy(new MetronSolrClient(zookeeperUrl));
-
- Mockito.doReturn(new NamedList<Object>() {{
- add("collections", new ArrayList<String>() {{
- add(collection);
- }});
- }}).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull());
- metronSolrClient.createCollection(collection, 1, 1);
- verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull());
- verify(metronSolrClient, times(0)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull());
-
- metronSolrClient = Mockito.spy(new MetronSolrClient(zookeeperUrl));
- Mockito.doReturn(new NamedList<Object>() {{
- add("collections", new ArrayList<String>());
- }}).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull());
- Mockito.doReturn(new NamedList<>()).when(metronSolrClient).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull());
- metronSolrClient.createCollection(collection, 1, 1);
- verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.LIST.name())), (String) isNull());
- verify(metronSolrClient, times(1)).request(argThat(new CollectionRequestMatcher(CollectionParams.CollectionAction.CREATE.name())), (String) isNull());
- }
-}
diff --git a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
index 515f2f8..f6ac329 100644
--- a/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
+++ b/metron-platform/metron-solr/metron-solr-common/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
@@ -34,6 +34,7 @@ import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration;
import org.apache.metron.common.writer.BulkMessage;
import org.apache.metron.enrichment.integration.utils.SampleUtil;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.common.SolrInputDocument;
import org.hamcrest.Description;
@@ -119,8 +120,8 @@ public class SolrWriterTest {
messages.add(new BulkMessage<>("message2", message2));
String collection = "metron";
- MetronSolrClient solr = Mockito.mock(MetronSolrClient.class);
- SolrWriter writer = new SolrWriter().withMetronSolrClient(solr);
+ CloudSolrClient solr = Mockito.mock(CloudSolrClient.class);
+ SolrWriter writer = new SolrWriter().withCloudSolrClient(solr);
writer.init(null,new IndexingWriterConfiguration("solr", configurations));
verify(solr, times(1)).setDefaultCollection(collection);
@@ -128,7 +129,7 @@ public class SolrWriterTest {
Map<String, Object> globalConfig = configurations.getGlobalConfig();
globalConfig.put("solr.collection", collection);
configurations.updateGlobalConfig(globalConfig);
- writer = new SolrWriter().withMetronSolrClient(solr);
+ writer = new SolrWriter().withCloudSolrClient(solr);
writer.init(null, new IndexingWriterConfiguration("solr", configurations));
verify(solr, times(1)).setDefaultCollection(collection);
diff --git a/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
index 2fb48b7..420cdb2 100644
--- a/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
+++ b/metron-platform/metron-solr/metron-solr-storm/src/test/java/org/apache/metron/indexing/integration/SolrIndexingIntegrationTest.java
@@ -18,6 +18,8 @@
package org.apache.metron.indexing.integration;
import com.google.common.base.Function;
+
+import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -39,6 +41,7 @@ import org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.solr.SolrConstants;
import org.apache.metron.solr.integration.components.SolrComponent;
+import org.apache.solr.client.solrj.SolrServerException;
public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
@@ -85,18 +88,22 @@ public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
public ReadinessState process(ComponentRunner runner) {
SolrComponent solrComponent = runner.getComponent("search", SolrComponent.class);
KafkaComponent kafkaComponent = runner.getComponent("kafka", KafkaComponent.class);
- if (solrComponent.hasCollection(collection)) {
- docs = solrComponent.getAllIndexedDocs(collection);
- if (docs.size() < inputMessages.size() ) {
- errors = kafkaComponent.readMessages(ERROR_TOPIC);
- if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){
+ try {
+ if (solrComponent.hasCollection(collection)) {
+ docs = solrComponent.getAllIndexedDocs(collection);
+ if (docs.size() < inputMessages.size() ) {
+ errors = kafkaComponent.readMessages(ERROR_TOPIC);
+ if(errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()){
+ return ReadinessState.READY;
+ }
+ return ReadinessState.NOT_READY;
+ } else {
return ReadinessState.READY;
}
- return ReadinessState.NOT_READY;
} else {
- return ReadinessState.READY;
+ return ReadinessState.NOT_READY;
}
- } else {
+ } catch (IOException | SolrServerException e) {
return ReadinessState.NOT_READY;
}
}
diff --git a/pom.xml b/pom.xml
index 73782ec..3cbd2bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,7 +115,6 @@
<global_json_schema_validator_version>2.2.5</global_json_schema_validator_version>
<global_opencsv_version>3.7</global_opencsv_version>
<global_java_version>1.8</global_java_version>
- <global_solr_version>6.6.2</global_solr_version>
<global_mockito_version>1.10.19</global_mockito_version>
<global_powermock_version>1.7.0</global_powermock_version>
<global_shade_version>3.2.0</global_shade_version>
@@ -148,6 +147,7 @@
<global_slf4j_version>1.7.25</global_slf4j_version>
<global_storm_version>1.2.1</global_storm_version>
<global_zeppelin_version>0.8.0</global_zeppelin_version>
+ <global_solr_version>7.4.0</global_solr_version>
</properties>
</profile>
<profile>
@@ -161,6 +161,7 @@
<global_storm_version>${base_storm_version}.${hdp_version}-${build_number}</global_storm_version>
<global_kafka_version>${base_kafka_version}.${hdp_version}-${build_number}</global_kafka_version>
<global_zeppelin_version>0.7.3</global_zeppelin_version>
+ <global_solr_version>6.6.2</global_solr_version>
</properties>
</profile>
</profiles>