You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/04/26 13:02:07 UTC
metron git commit: METRON-1526 Location field types cause
DocValuesField appear more than once error (merrimanr via justinleet) closes
apache/metron#995
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1416-upgrade-solr 7ef4b7703 -> f8d7843e9
METRON-1526 Location field types cause DocValuesField appear more than once error (merrimanr via justinleet) closes apache/metron#995
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f8d7843e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f8d7843e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f8d7843e
Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: f8d7843e93e410147257cca40d83b2c2d4963f1b
Parents: 7ef4b77
Author: merrimanr <me...@gmail.com>
Authored: Thu Apr 26 09:01:03 2018 -0400
Committer: leet <le...@apache.org>
Committed: Thu Apr 26 09:01:03 2018 -0400
----------------------------------------------------------------------
.../indexing/dao/UpdateIntegrationTest.java | 21 +++++++++---
metron-platform/metron-solr/README.md | 16 +++++++++
.../src/main/config/schema/bro/schema.xml | 1 +
.../src/main/config/schema/snort/schema.xml | 1 +
.../src/main/config/schema/yaf/schema.xml | 1 +
.../integration/SolrUpdateIntegrationTest.java | 34 ++++++++++++++++++--
.../integration/components/SolrComponent.java | 7 +++-
7 files changed, 74 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index 369fa79..471acf6 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -31,11 +31,14 @@ import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
import org.apache.metron.integration.InMemoryComponent;
+import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
public abstract class UpdateIntegrationTest {
private static final int MAX_RETRIES = 10;
@@ -46,6 +49,7 @@ public abstract class UpdateIntegrationTest {
private static String index;
private static MockHTable table;
private static IndexDao hbaseDao;
+ private static AccessConfig accessConfig;
protected static MultiIndexDao dao;
protected static InMemoryComponent indexComponent;
@@ -62,20 +66,24 @@ public abstract class UpdateIntegrationTest {
table = (MockHTable)tableProvider.getTable(config, TABLE_NAME);
hbaseDao = new HBaseDao();
- AccessConfig accessConfig = new AccessConfig();
+ accessConfig = new AccessConfig();
accessConfig.setTableProvider(tableProvider);
Map<String, Object> globalConfig = createGlobalConfig();
globalConfig.put(HBaseDao.HBASE_TABLE, TABLE_NAME);
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
-
- dao = new MultiIndexDao(hbaseDao, createDao());
- dao.init(accessConfig);
}
}
+ protected AccessConfig getAccessConfig() {
+ return accessConfig;
+ }
+
@Test
public void test() throws Exception {
+ dao = new MultiIndexDao(hbaseDao, createDao());
+ dao.init(getAccessConfig());
+
List<Map<String, Object>> inputData = new ArrayList<>();
for(int i = 0; i < 10;++i) {
final String name = "message" + i;
@@ -181,6 +189,11 @@ public abstract class UpdateIntegrationTest {
}
}
+ @After
+ public void reset() throws Exception {
+ indexComponent.reset();
+ }
+
@AfterClass
public static void teardown() {
if(indexComponent != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index 4321114..0d525ec 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -95,3 +95,19 @@ As of now, we have mapped out the Schemas in `src/main/config/schema`.
Ambari will eventually install these, but at the moment it's manual and
you should refer to the Solr documentation [https://lucene.apache.org/solr/guide/6_6](here) in general
and [here](https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html) if you'd like to know more about schemas in Solr.
+
+In Metron's Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document.
+Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results. For these fields a dynamic field type matching the suffix needs to be added to store the data points.
+Solr 6+ comes with a new LatLonPointSpatialField field type that should be used instead of LatLonType if possible. Otherwise, a LatLongType field should be defined as:
+```
+<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+<dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
+<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+```
+A PointType field should be defined as:
+```
+<dynamicField name="*.point" type="point" multiValued="false" docValues="false"/>
+<dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/>
+<fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/>
+```
+If any copy fields are defined, stored and docValues should be set to false.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
index 4ce8d6b..b463366 100644
--- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
@@ -659,6 +659,7 @@
<!-- Geo Enrichment Fields -->
<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+ <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
<dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
index ae13b8d..129c0f0 100644
--- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -52,6 +52,7 @@
<!-- Geo Enrichment Fields -->
<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+ <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
<dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
index edccd9d..f3abb14 100644
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -58,6 +58,7 @@
<!-- Geo Enrichment Fields -->
<dynamicField name="*.location_point" type="location" multiValued="false" docValues="false"/>
+ <dynamicField name="*_coordinate" type="pdouble" indexed="true" stored="false" docValues="false"/>
<dynamicField name="*.country" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.city" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.locID" type="string" multiValued="false" docValues="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index faa4ec4..232c727 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -20,15 +20,21 @@ package org.apache.metron.solr.integration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
import org.apache.metron.indexing.dao.UpdateIntegrationTest;
+import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
+import org.junit.Assert;
+import org.junit.Test;
public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
- SolrComponent solrComponent;
+ protected static SolrComponent solrComponent;
@Override
protected String getIndexName() {
@@ -49,7 +55,7 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
@Override
protected InMemoryComponent startIndex() throws Exception {
- solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/test/resources/config/test/conf").build();
+ solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro").build();
solrComponent.start();
return solrComponent;
}
@@ -70,4 +76,28 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
throws Exception {
return solrComponent.getAllIndexedDocs(indexName);
}
+
+ @Test
+ public void suppress_expanded_fields() throws Exception {
+ dao = new MultiIndexDao(createDao());
+ dao.init(getAccessConfig());
+
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "bro_1");
+ fields.put("source.type", SENSOR_NAME);
+ fields.put("ip_src_port", 8010);
+ fields.put("long_field", 10000);
+ fields.put("latitude", 48.5839);
+ fields.put("score", 10.0);
+ fields.put("is_alert", true);
+ fields.put("field.location_point", "48.5839,7.7455");
+
+ Document document = new Document(fields, "bro_1", SENSOR_NAME, 0L);
+ dao.update(document, Optional.of(SENSOR_NAME));
+
+ Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
+
+ // assert no extra expanded fields are included
+ Assert.assertEquals(8, indexedDocument.getDocument().size());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f8d7843e/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 85d14f4..1cf8c34 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -116,7 +116,12 @@ public class SolrComponent implements InMemoryComponent {
@Override
public void reset() {
try {
- miniSolrCloudCluster.deleteCollection("metron");
+ MetronSolrClient solr = getSolrClient();
+ for (String collection: collections.keySet()) {
+ solr.setDefaultCollection(collection);
+ solr.deleteByQuery("*:*");
+ }
+ solr.commit();
} catch (SolrServerException | IOException e) {
// Do nothing
}