You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:20 UTC
[04/50] [abbrv] metron git commit: METRON-1567 Large error message
can' t be written in Solr (justinleet) closes apache/metron#1020
METRON-1567 Large error message can't be written in Solr (justinleet) closes apache/metron#1020
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bc16e682
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bc16e682
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bc16e682
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: bc16e68252de98846bba58aafb2d7d445caac9c1
Parents: 832296f
Author: justinleet <ju...@gmail.com>
Authored: Fri May 18 11:34:30 2018 -0400
Committer: leet <le...@apache.org>
Committed: Fri May 18 11:34:30 2018 -0400
----------------------------------------------------------------------
.../src/main/config/schema/error/schema.xml | 4 +-
.../integration/SolrUpdateIntegrationTest.java | 49 ++++++++++++++++++--
2 files changed, 48 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
index 5f9f9a3..4aa80ef 100644
--- a/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/error/schema.xml
@@ -29,7 +29,6 @@
<!-- The message in string form (this may be garbage if the message is a bytes rather than text -->
<field name="message" type="string" indexed="true" stored="true" />
- <field name="raw_message" type="string" indexed="false" stored="true" />
<field name="raw_message_bytes" type="bytes" indexed="false" stored="true" />
<field name="error_fields" type="string" indexed="true" stored="true" />
<field name="error_hash" type="string" indexed="true" stored="true" />
@@ -40,6 +39,8 @@
<field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<uniqueKey>guid</uniqueKey>
+ <!-- Raw messages can be split into multiple fields -->
+ <dynamicField name="raw_message*" type="text_raw" indexed="false" stored="true"/>
<!-- Catch all, if we don't know about it, it gets dropped. -->
<dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
@@ -55,4 +56,5 @@
<fieldType name="ip" stored="true" indexed="true" multiValued="false" class="solr.StrField" sortMissingLast="true" docValues="false"/>
<fieldType name="timestamp" stored="true" indexed="true" multiValued="false" class="solr.TrieLongField" sortMissingLast="false" docValues="true"/>
<fieldType name="ignored" stored="true" indexed="true" multiValued="true" class="solr.StrField" sortMissingLast="false" docValues="false"/>
+ <fieldType name="text_raw" class="solr.TextField"/>
</schema>
http://git-wip-us.apache.org/repos/asf/metron/blob/bc16e682/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
index 232c727..dc08967 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java
@@ -17,11 +17,14 @@
*/
package org.apache.metron.solr.integration;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MultiIndexDao;
import org.apache.metron.indexing.dao.UpdateIntegrationTest;
@@ -29,10 +32,13 @@ import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.solr.dao.SolrDao;
import org.apache.metron.solr.integration.components.SolrComponent;
-import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
protected static SolrComponent solrComponent;
@@ -55,7 +61,9 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
@Override
protected InMemoryComponent startIndex() throws Exception {
- solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro").build();
+ solrComponent = new SolrComponent.Builder().addCollection(SENSOR_NAME, "../metron-solr/src/main/config/schema/bro")
+ .addCollection("error", "../metron-solr/src/main/config/schema/error")
+ .build();
solrComponent.start();
return solrComponent;
}
@@ -98,6 +106,39 @@ public class SolrUpdateIntegrationTest extends UpdateIntegrationTest {
Document indexedDocument = dao.getLatest("bro_1", SENSOR_NAME);
// assert no extra expanded fields are included
- Assert.assertEquals(8, indexedDocument.getDocument().size());
+ assertEquals(8, indexedDocument.getDocument().size());
+ }
+
+ @Test
+ public void testHugeErrorFields() throws Exception {
+ dao = new MultiIndexDao(createDao());
+ dao.init(getAccessConfig());
+
+ String hugeString = StringUtils.repeat("test ", 1_000_000);
+ String hugeStringTwo = hugeString + "-2";
+
+ Map<String, Object> documentMap = new HashMap<>();
+ documentMap.put("guid", "error_guid");
+ // Needs to be over 32kb
+ documentMap.put("raw_message", hugeString);
+ documentMap.put("raw_message_1", hugeStringTwo);
+ Document errorDoc = new Document(documentMap, "error", "error", 0L);
+ dao.update(errorDoc, Optional.of("error"));
+
+ // Ensure that the huge string is returned when not a string field
+ Document latest = dao.getLatest("error_guid", "error");
+ @SuppressWarnings("unchecked")
+ String actual = (String) latest.getDocument().get("raw_message");
+ assertEquals(actual, hugeString);
+ String actualTwo = (String) latest.getDocument().get("raw_message_1");
+ assertEquals(actualTwo, hugeStringTwo);
+
+ // Validate that error occurs for string fields.
+ documentMap.put("error_hash", hugeString);
+ errorDoc = new Document(documentMap, "error", "error", 0L);
+
+ exception.expect(IOException.class);
+ exception.expectMessage("Document contains at least one immense term in field=\"error_hash\"");
+ dao.update(errorDoc, Optional.of("error"));
}
}