You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:22 UTC
[06/50] [abbrv] metron git commit: METRON-1421 Create a
SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
index 1cf8c34..4bc9f8a 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/components/SolrComponent.java
@@ -19,9 +19,13 @@ package org.apache.metron.solr.integration.components;
import com.google.common.base.Function;
import java.util.Collection;
+import java.util.Map.Entry;
import java.util.stream.Collectors;
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.integration.InMemoryComponent;
import org.apache.metron.integration.UnableToStartException;
+import org.apache.metron.solr.dao.SolrUtilities;
import org.apache.metron.solr.writer.MetronSolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -29,6 +33,7 @@ import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrDocument;
@@ -40,13 +45,15 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.zookeeper.KeeperException;
public class SolrComponent implements InMemoryComponent {
public static class Builder {
+
private int port = 8983;
private String solrXmlPath = "../metron-solr/src/test/resources/solr/solr.xml";
- private Map<String, String> collections = new HashMap<>();
+ private Map<String, String> initialCollections = new HashMap<>();
private Function<SolrComponent, Void> postStartCallback;
public Builder withPort(int port) {
@@ -59,8 +66,8 @@ public class SolrComponent implements InMemoryComponent {
return this;
}
- public Builder addCollection(String name, String configPath) {
- collections.put(name, configPath);
+ public Builder addInitialCollection(String name, String configPath) {
+ initialCollections.put(name, configPath);
return this;
}
@@ -69,9 +76,8 @@ public class SolrComponent implements InMemoryComponent {
return this;
}
- public SolrComponent build() throws Exception {
- if (collections.isEmpty()) throw new Exception("Must add at least 1 collection");
- return new SolrComponent(port, solrXmlPath, collections, postStartCallback);
+ public SolrComponent build() {
+ return new SolrComponent(port, solrXmlPath, initialCollections, postStartCallback);
}
}
@@ -81,7 +87,8 @@ public class SolrComponent implements InMemoryComponent {
private MiniSolrCloudCluster miniSolrCloudCluster;
private Function<SolrComponent, Void> postStartCallback;
- private SolrComponent(int port, String solrXmlPath, Map<String, String> collections, Function<SolrComponent, Void> postStartCallback) throws Exception {
+ private SolrComponent(int port, String solrXmlPath, Map<String, String> collections,
+ Function<SolrComponent, Void> postStartCallback) {
this.port = port;
this.solrXmlPath = solrXmlPath;
this.collections = collections;
@@ -93,14 +100,17 @@ public class SolrComponent implements InMemoryComponent {
try {
File baseDir = Files.createTempDirectory("solrcomponent").toFile();
baseDir.deleteOnExit();
- miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(), JettyConfig.builder().setPort(port).build());
+ miniSolrCloudCluster = new MiniSolrCloudCluster(1, baseDir.toPath(),
+ JettyConfig.builder().setPort(port).build());
for(String name: collections.keySet()) {
String configPath = collections.get(name);
miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
CollectionAdminRequest.createCollection(name, 1, 1).process(miniSolrCloudCluster.getSolrClient());
}
- if (postStartCallback != null) postStartCallback.apply(this);
- } catch(Exception e) {
+ if (postStartCallback != null) {
+ postStartCallback.apply(this);
+ }
+ } catch (Exception e) {
throw new UnableToStartException(e.getMessage(), e);
}
}
@@ -108,21 +118,18 @@ public class SolrComponent implements InMemoryComponent {
@Override
public void stop() {
try {
+ miniSolrCloudCluster.deleteAllCollections();
miniSolrCloudCluster.shutdown();
} catch (Exception e) {
+ // Do nothing
}
}
@Override
public void reset() {
try {
- MetronSolrClient solr = getSolrClient();
- for (String collection: collections.keySet()) {
- solr.setDefaultCollection(collection);
- solr.deleteByQuery("*:*");
- }
- solr.commit();
- } catch (SolrServerException | IOException e) {
+ miniSolrCloudCluster.deleteAllCollections();
+ } catch (Exception e) {
// Do nothing
}
}
@@ -139,12 +146,19 @@ public class SolrComponent implements InMemoryComponent {
return miniSolrCloudCluster.getZkServer().getZkAddress();
}
+ public void addCollection(String name, String configPath)
+ throws InterruptedException, IOException, KeeperException, SolrServerException {
+ miniSolrCloudCluster.uploadConfigSet(new File(configPath).toPath(), name);
+ CollectionAdminRequest.createCollection(name, 1, 1)
+ .process(miniSolrCloudCluster.getSolrClient());
+ }
+
public boolean hasCollection(String collection) {
MetronSolrClient solr = getSolrClient();
boolean collectionFound = false;
try {
collectionFound = solr.listCollections().contains(collection);
- } catch(Exception e) {
+ } catch (Exception e) {
e.printStackTrace();
}
return collectionFound;
@@ -155,12 +169,21 @@ public class SolrComponent implements InMemoryComponent {
CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
solr.setDefaultCollection(collection);
SolrQuery parameters = new SolrQuery();
- parameters.set("q", "*:*");
+
+ // If it's metaalert, we need to adjust the query. We want child docs with the parent,
+ // not separate.
+ if (collection.equals("metaalert")) {
+ parameters.setQuery("source.type:metaalert")
+ .setFields("*", "[child parentFilter=source.type:metaalert limit=999]");
+ } else {
+ parameters.set("q", "*:*");
+ }
try {
solr.commit();
QueryResponse response = solr.query(parameters);
for (SolrDocument solrDocument : response.getResults()) {
- docs.add(solrDocument);
+ // Use the utils to make sure we get child docs.
+ docs.add(SolrUtilities.toDocument(solrDocument).getDocument());
}
} catch (SolrServerException | IOException e) {
e.printStackTrace();
@@ -174,9 +197,36 @@ public class SolrComponent implements InMemoryComponent {
solr.setDefaultCollection(collection);
Collection<SolrInputDocument> solrInputDocuments = docs.stream().map(doc -> {
SolrInputDocument solrInputDocument = new SolrInputDocument();
- doc.forEach(solrInputDocument::addField);
+ for (Entry<String, Object> entry : doc.entrySet()) {
+ // If the entry itself is a map, add it as a child document. Handle one level of nesting.
+ if (entry.getValue() instanceof List && !entry.getKey().equals(
+ MetaAlertConstants.METAALERT_FIELD)) {
+ for (Object entryItem : (List)entry.getValue()) {
+ if (entryItem instanceof Map) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> childDoc = (Map<String, Object>) entryItem;
+ SolrInputDocument childInputDoc = new SolrInputDocument();
+ for (Entry<String, Object> childEntry : childDoc.entrySet()) {
+ childInputDoc.addField(childEntry.getKey(), childEntry.getValue());
+ }
+ solrInputDocument.addChildDocument(childInputDoc);
+ }
+ }
+ } else {
+ solrInputDocument.addField(entry.getKey(), entry.getValue());
+ }
+ }
return solrInputDocument;
}).collect(Collectors.toList());
- solr.add(collection, solrInputDocuments);
+
+ checkUpdateResponse(solr.add(collection, solrInputDocuments));
+ // Make sure to commit so things show up
+ checkUpdateResponse(solr.commit(true, true));
+ }
+
+ protected void checkUpdateResponse(UpdateResponse result) throws IOException {
+ if (result.getStatus() != 0) {
+ throw new IOException("Response error received while adding documents: " + result);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
index e655428..73113f6 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
@@ -25,9 +25,12 @@ import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.metron.solr.integration.components.SolrComponent;
import org.apache.metron.solr.writer.SolrWriter;
import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.storm.tuple.Tuple;
+import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import java.io.File;
@@ -38,7 +41,6 @@ import java.util.*;
import static org.mockito.Mockito.mock;
public class SchemaValidationIntegrationTest {
-
public static Iterable<String> getData(String sensor) throws IOException {
return Iterables.filter(
Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()),
@@ -53,9 +55,7 @@ public class SchemaValidationIntegrationTest {
}
public static SolrComponent createSolrComponent(String sensor) throws Exception {
- return new SolrComponent.Builder()
- .addCollection(String.format("%s", sensor), String.format("src/main/config/schema/%s", sensor))
- .build();
+ return new SolrComponent.Builder().build();
}
@Test
@@ -92,6 +92,7 @@ public class SchemaValidationIntegrationTest {
try {
component = createSolrComponent(sensorType);
component.start();
+ component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType));
Map<String, Object> globalConfig = getGlobalConfig(sensorType, component);
List<JSONObject> inputs = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
index 685c5fd..7b7d208 100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/writer/SolrWriterTest.java
@@ -17,11 +17,19 @@
*/
package org.apache.metron.solr.writer;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.EnrichmentConfigurations;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.writer.IndexingWriterConfiguration;
import org.apache.metron.enrichment.integration.utils.SampleUtil;
@@ -34,17 +42,6 @@ import org.junit.Test;
import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.argThat;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
public class SolrWriterTest {
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
index 85d6915..bb2de59 100644
--- a/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
+++ b/metron-platform/metron-solr/src/test/resources/config/test/conf/managed-schema
@@ -16,53 +16,59 @@
limitations under the License.
-->
<schema name="example" version="1.6">
- <field name="_version_" type="plong" indexed="false" stored="false"/>
+ <field name="_version_" type="plong" indexed="false" stored="false"/>
- <!-- points to the root document of a block of nested documents. Required for nested
- document support, may be removed otherwise
- -->
- <field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
+ <!-- points to the root document of a block of nested documents. Required for nested
+ document support, may be removed otherwise
+ -->
+ <field name="_root_" type="string" indexed="true" stored="false" docValues="false"/>
- <!-- Only remove the "id" field if you have a very good reason to. While not strictly
- required, it is highly recommended. A <uniqueKey> is present in almost all Solr
- installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id".
- Do NOT change the type and apply index-time analysis to the <uniqueKey> as it will likely
- make routing in SolrCloud and document replacement in general fail. Limited _query_ time
- analysis is possible as long as the indexing process is guaranteed to index the term
- in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce multiple
- tokens
- -->
- <field name="guid" type="string" indexed="true" stored="true" required="true" multiValued="false" />
+ <!-- Only remove the "id" field if you have a very good reason to. While not strictly
+ required, it is highly recommended. A <uniqueKey> is present in almost all Solr
+ installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id".
+ Do NOT change the type and apply index-time analysis to the <uniqueKey> as it will likely
+ make routing in SolrCloud and document replacement in general fail. Limited _query_ time
+ analysis is possible as long as the indexing process is guaranteed to index the term
+ in a compatible way. Any analysis applied to the <uniqueKey> should _not_ produce multiple
+ tokens
+ -->
+ <field name="guid" type="string" indexed="true" stored="true" required="true"
+ multiValued="false"/>
- <field name="source:type" type="string" indexed="true" stored="true"/>
- <field name="name" type="string" indexed="true" stored="true" />
- <field name="timestamp" type="plong" indexed="true" stored="true" />
- <field name="new-field" type="string" indexed="true" stored="true" />
+ <field name="source.type" type="string" indexed="true" stored="true"/>
+ <field name="name" type="string" indexed="true" stored="true"/>
+ <field name="timestamp" type="plong" indexed="true" stored="true"/>
+ <field name="new-field" type="string" indexed="true" stored="true"/>
+ <field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
+ <field name="threat:triage:score" type="pdouble" indexed="true" stored="true"/>
+ <field name="score" type="pdouble" indexed="true" stored="true"/>
- <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
+ <dynamicField name="*" type="ignored" multiValued="false" docValues="true"/>
- <!-- Field to use to determine and enforce document uniqueness.
- Unless this field is marked with required="false", it will be a required field
- -->
- <uniqueKey>guid</uniqueKey>
-
- <!-- field type definitions. The "name" attribute is
- just a label to be used by field definitions. The "class"
- attribute and any other attributes determine the real
- behavior of the fieldType.
- Class names starting with "solr" refer to java classes in a
- standard package such as org.apache.solr.analysis
+ <!-- Field to use to determine and enforce document uniqueness.
+ Unless this field is marked with required="false", it will be a required field
-->
+ <uniqueKey>guid</uniqueKey>
+
+
+ <!-- field type definitions. The "name" attribute is
+ just a label to be used by field definitions. The "class"
+ attribute and any other attributes determine the real
+ behavior of the fieldType.
+ Class names starting with "solr" refer to java classes in a
+ standard package such as org.apache.solr.analysis
+ -->
- <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
- <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
- <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
- <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
- <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
- <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
+ <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
+ <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
+ <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
+ <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
+ <fieldType name="ignored" stored="false" indexed="false" multiValued="true"
+ class="solr.StrField"/>
</schema>