You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:36 UTC
[20/50] [abbrv] metron git commit: Merge branch 'master' into
feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/41708876
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/41708876
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/41708876
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 41708876336d009b3a2ff2c533bdf0294af1477b
Parents: f241f87 32d132b
Author: cstella <ce...@gmail.com>
Authored: Mon Jun 11 17:50:48 2018 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Jun 11 17:50:48 2018 -0400
----------------------------------------------------------------------
KEYS | 59 +
Upgrading.md | 2 +-
dev-utilities/release-utils/metron-rc-check | 77 +-
.../release-utils/validate-jira-for-release | 75 +-
metron-analytics/metron-maas-common/pom.xml | 2 +-
metron-analytics/metron-maas-service/pom.xml | 2 +-
metron-analytics/metron-profiler-client/pom.xml | 2 +-
.../profiler/client/stellar/GetProfileTest.java | 3 +-
.../client/stellar/ProfilerFunctionsTest.java | 15 +-
metron-analytics/metron-profiler-common/pom.xml | 2 +-
metron-analytics/metron-profiler/README.md | 15 +
metron-analytics/metron-profiler/pom.xml | 2 +-
.../src/main/flux/profiler/remote.yaml | 3 +-
metron-analytics/metron-statistics/pom.xml | 2 +-
.../StellarStatisticsFunctionsTest.java | 5 +-
.../sampling/SamplerFunctionsTest.java | 3 +-
metron-analytics/pom.xml | 2 +-
metron-contrib/metron-docker/pom.xml | 2 +-
metron-contrib/metron-performance/pom.xml | 2 +-
metron-contrib/pom.xml | 2 +-
metron-deployment/Kerberos-manual-setup.md | 8 +-
metron-deployment/README.md | 22 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 2 +-
.../ansible/playbooks/docker_probe_install.yml | 2 +-
.../roles/ambari_master/defaults/main.yml | 2 +-
.../roles/metron_pcapservice/defaults/main.yml | 2 +-
metron-deployment/development/centos6/README.md | 8 +-
.../centos6/ansible/inventory/group_vars/all | 2 +-
.../development/ubuntu14/README.md | 6 +-
.../ubuntu14/ansible/inventory/group_vars/all | 2 +-
.../ambari/elasticsearch-mpack/pom.xml | 4 +-
.../packaging/ambari/metron-mpack/pom.xml | 4 +-
.../CURRENT/package/files/bro_index.template | 2 +-
.../CURRENT/package/files/error_index.template | 2 +-
.../package/files/metaalert_index.template | 2 +-
.../CURRENT/package/files/snort_index.template | 2 +-
.../CURRENT/package/files/yaf_index.template | 2 +-
.../CURRENT/package/scripts/metron_service.py | 5 +
.../metron-mpack/src/main/resources/mpack.json | 2 +-
.../packaging/docker/deb-docker/pom.xml | 4 +-
.../packaging/docker/rpm-docker/pom.xml | 4 +-
metron-deployment/pom.xml | 2 +-
.../e2e/mock-data/alerts_ui_e2e_index.template | 2 +-
metron-interface/metron-alerts/package.json | 2 +-
metron-interface/metron-alerts/pom.xml | 2 +-
.../alert-details/alert-details.component.ts | 2 +-
.../alerts/alerts-list/alerts-list.component.ts | 2 +-
.../table-view/table-view.component.html | 10 +-
.../table-view/table-view.component.ts | 2 +-
.../metron-alerts/src/app/model/alert-source.ts | 2 +-
.../src/app/service/global-config.service.ts | 10 +-
metron-interface/metron-config/package.json | 2 +-
metron-interface/metron-config/pom.xml | 2 +-
.../metron-config/scripts/package.json | 2 +-
metron-interface/metron-rest-client/pom.xml | 2 +-
metron-interface/metron-rest/pom.xml | 2 +-
.../rest/service/impl/SearchServiceImpl.java | 10 +-
.../src/main/resources/application.yml | 2 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 5 +-
.../MetaAlertControllerIntegrationTest.java | 4 +-
.../service/impl/SearchServiceImplTest.java | 3 +-
metron-interface/pom.xml | 2 +-
metron-platform/Performance-tuning-guide.md | 4 +-
metron-platform/README.md | 2 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +-
metron-platform/metron-api/pom.xml | 2 +-
metron-platform/metron-common/README.md | 5 +
metron-platform/metron-common/pom.xml | 2 +-
.../org/apache/metron/common/Constants.java | 1 +
.../metron/common/bolt/ConfiguredBolt.java | 14 +-
.../common/bolt/ConfiguredEnrichmentBolt.java | 11 +-
.../common/bolt/ConfiguredIndexingBolt.java | 13 +-
.../common/bolt/ConfiguredParserBolt.java | 13 +-
.../common/bolt/ConfiguredProfilerBolt.java | 9 +-
.../common/configuration/Configurations.java | 7 +-
.../configuration/ConfigurationsUtils.java | 7 +
.../configuration/EnrichmentConfigurations.java | 26 +
.../configuration/IndexingConfigurations.java | 28 +-
.../configuration/ParserConfigurations.java | 1 +
.../enrichment/handler/StellarConfig.java | 11 +-
.../profiler/ProfilerConfigurations.java | 25 +
.../writer/ConfigurationStrategy.java | 44 +
.../writer/ConfigurationsStrategies.java | 144 +
.../writer/EnrichmentWriterConfiguration.java | 110 +
.../writer/IndexingWriterConfiguration.java | 5 +
.../writer/ParserWriterConfiguration.java | 12 +-
.../writer/ProfilerWriterConfiguration.java | 109 +
.../writer/SingleBatchConfigurationFacade.java | 6 +
.../writer/WriterConfiguration.java | 69 +
.../common/field/DeDotFieldNameConverter.java | 46 +
.../metron/common/field/FieldNameConverter.java | 32 +
.../common/field/FieldNameConverters.java | 116 +
.../common/field/NoopFieldNameConverter.java | 32 +
.../common/interfaces/FieldNameConverter.java | 24 -
.../src/main/scripts/cluster_info.py | 2 +-
.../configuration/ParserConfigurationsTest.java | 120 +
.../profiler/ProfilerConfigTest.java | 13 +-
.../writer/ConfigurationsStrategiesTest.java | 79 +
.../EnrichmentWriterConfigurationTest.java | 54 +
.../writer/IndexingWriterConfigurationTest.java | 70 +
.../writer/ParserWriterConfigurationTest.java | 72 +
.../writer/ProfilerWriterConfigurationTest.java | 54 +
.../field/DeDotFieldNameConverterTest.java | 38 +
.../common/field/FieldNameConvertersTest.java | 211 +
.../StellarTransformationTest.java | 30 +-
.../writer/IndexingWriterConfigurationTest.java | 70 -
.../writer/ParserWriterConfigurationTest.java | 38 -
metron-platform/metron-data-management/pom.xml | 2 +-
metron-platform/metron-elasticsearch/README.md | 8 +-
metron-platform/metron-elasticsearch/pom.xml | 8 +-
.../dao/ElasticsearchMetaAlertDao.java | 33 +-
.../writer/ElasticsearchFieldNameConverter.java | 32 -
.../writer/ElasticsearchWriter.java | 70 +-
.../ElasticsearchIndexingIntegrationTest.java | 7 +-
.../ElasticsearchMetaAlertIntegrationTest.java | 16 +-
.../ElasticsearchFieldNameConverterTest.java | 32 -
.../metron-enrichment/Performance.md | 2 +-
metron-platform/metron-enrichment/README.md | 15 +-
metron-platform/metron-enrichment/pom.xml | 2 +-
.../main/flux/enrichment/remote-splitjoin.yaml | 9 +-
.../main/flux/enrichment/remote-unified.yaml | 9 +-
.../adapters/stellar/StellarAdapter.java | 14 +-
.../adapters/stellar/StellarAdapterTest.java | 26 +
.../bolt/BulkMessageWriterBoltTest.java | 112 +-
.../integration/EnrichmentIntegrationTest.java | 2 +
metron-platform/metron-hbase-client/pom.xml | 2 +-
metron-platform/metron-hbase/pom.xml | 2 +-
metron-platform/metron-indexing/README.md | 20 +-
metron-platform/metron-indexing/pom.xml | 2 +-
.../src/main/flux/indexing/batch/remote.yaml | 4 +-
.../flux/indexing/random_access/remote.yaml | 4 +-
.../apache/metron/indexing/dao/HBaseDao.java | 2 +-
.../dao/metaalert/MetaAlertConstants.java | 2 +-
.../indexing/dao/InMemoryMetaAlertDao.java | 2 +-
.../integration/HBaseDaoIntegrationTest.java | 17 +
.../HDFSIndexingIntegrationTest.java | 4 +-
.../integration/IndexingIntegrationTest.java | 3 +-
metron-platform/metron-integration-test/pom.xml | 2 +-
.../main/config/zookeeper/enrichments/test.json | 1 +
metron-platform/metron-management/pom.xml | 2 +-
.../metron/management/KafkaFunctions.java | 573 +-
.../management/IndexingConfigFunctionsTest.java | 7 +-
.../KafkaFunctionsIntegrationTest.java | 353 +-
.../management/ThreatTriageFunctionsTest.java | 7 +-
.../metron-parsers/3rdPartyParser.md | 439 +-
metron-platform/metron-parsers/README.md | 6 +-
metron-platform/metron-parsers/pom.xml | 2 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 108 +-
.../metron/parsers/bolt/WriterHandler.java | 63 +-
.../src/main/scripts/start_parser_topology.sh | 27 +-
.../metron/parsers/bolt/ParserBoltTest.java | 176 +-
.../SimpleHBaseEnrichmentWriterTest.java | 6 +-
.../integration/WriterBoltIntegrationTest.java | 6 +-
metron-platform/metron-pcap-backend/pom.xml | 2 +-
.../src/main/scripts/pcap_zeppelin_run.sh | 2 +-
metron-platform/metron-pcap/pom.xml | 2 +-
.../metron/pcap/filter/PcapFieldResolver.java | 5 +
metron-platform/metron-solr/pom.xml | 2 +-
.../SolrIndexingIntegrationTest.java | 3 +-
.../schema/SchemaValidationIntegrationTest.java | 5 +
.../metron-storm-kafka-override/pom.xml | 2 +-
metron-platform/metron-storm-kafka/pom.xml | 2 +-
metron-platform/metron-test-utilities/pom.xml | 2 +-
metron-platform/metron-writer/pom.xml | 2 +-
.../metron/writer/BulkWriterComponent.java | 36 +-
.../metron/writer/bolt/BatchTimeoutHelper.java | 6 +-
.../writer/bolt/BulkMessageWriterBolt.java | 138 +-
.../apache/metron/writer/kafka/KafkaWriter.java | 92 +-
metron-platform/metron-zookeeper/pom.xml | 2 +-
metron-platform/pom.xml | 2 +-
metron-stellar/pom.xml | 2 +-
.../stellar-3rd-party-example/pom.xml | 2 +-
metron-stellar/stellar-common/README.md | 5 +-
metron-stellar/stellar-common/pom.xml | 2 +-
.../stellar/common/BaseStellarProcessor.java | 28 +-
.../stellar/common/CachingStellarProcessor.java | 141 +-
.../common/StellarPredicateProcessor.java | 6 +
.../metron/stellar/common/utils/ConcatMap.java | 256 +
.../common/utils/StellarProcessorUtils.java | 51 +-
.../metron/stellar/dsl/MapVariableResolver.java | 8 +
.../metron/stellar/dsl/VariableResolver.java | 1 +
.../dsl/functions/DataStructureFunctions.java | 4 +
.../common/CachingStellarProcessorTest.java | 195 +-
.../shell/specials/AssignmentCommandTest.java | 7 +-
.../stellar/common/utils/ConcatMapTest.java | 83 +
.../stellar/dsl/functions/BasicStellarTest.java | 18 +
.../dsl/functions/OrdinalFunctionsTest.java | 9 +-
metron-stellar/stellar-zeppelin/README.md | 6 +-
metron-stellar/stellar-zeppelin/pom.xml | 2 +-
pom.xml | 5 +-
site-book/pom.xml | 2 +-
site/current-book/CONTRIBUTING.html | 157 +
site/current-book/Upgrading.html | 354 +-
.../css/apache-maven-fluido-1.3.0.min.css | 9 -
.../css/apache-maven-fluido-1.7.min.css | 17 +
site/current-book/css/maven-base.css | 168 +
site/current-book/css/maven-theme.css | 161 +
site/current-book/css/print.css | 11 +-
.../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 35283 bytes
.../fonts/glyphicons-halflings-regular.svg | 229 +
.../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 55016 bytes
.../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 41793 bytes
.../images/apache-maven-project-2.png | Bin 33442 -> 43073 bytes
site/current-book/images/collapsed.gif | Bin 0 -> 53 bytes
site/current-book/images/expanded.gif | Bin 0 -> 52 bytes
site/current-book/images/external.png | Bin 0 -> 230 bytes
site/current-book/images/icon_error_sml.gif | Bin 633 -> 1010 bytes
site/current-book/images/icon_info_sml.gif | Bin 638 -> 606 bytes
site/current-book/images/icon_success_sml.gif | Bin 604 -> 990 bytes
site/current-book/images/icon_warning_sml.gif | Bin 625 -> 576 bytes
site/current-book/images/newwindow.png | Bin 0 -> 220 bytes
site/current-book/index.html | 347 +-
.../js/apache-maven-fluido-1.3.0.min.js | 21 -
.../js/apache-maven-fluido-1.7.min.js | 25 +
site/current-book/metron-analytics/index.html | 303 +-
.../metron-maas-service/index.html | 451 +-
.../metron-profiler-client/index.html | 888 +-
.../metron-analytics/metron-profiler/index.html | 1338 +-
.../metron-statistics/HLLP.html | 10962 ++++++-----------
.../metron-statistics/index.html | 870 +-
.../metron-contrib/metron-docker/index.html | 540 +-
.../metron-performance/index.html | 317 +
.../Kerberos-ambari-setup.html | 372 +-
.../Kerberos-manual-setup.html | 1218 +-
.../metron-deployment/amazon-ec2/index.html | 755 +-
.../metron-deployment/ansible/index.html | 128 +
.../metron-deployment/ansible/roles/index.html | 140 +
.../ansible/roles/opentaxii/index.html | 301 +
.../ansible/roles/pcap_replay/index.html | 172 +
.../ansible/roles/sensor-stubs/index.html | 221 +
.../ansible/roles/sensor-test-mode/index.html | 156 +
.../development/centos6/index.html | 233 +
.../development/fastcapa/index.html | 249 +
.../metron-deployment/development/index.html | 137 +
.../development/ubuntu14/index.html | 233 +
site/current-book/metron-deployment/index.html | 691 +-
.../metron-deployment/other-examples/index.html | 363 +-
.../manual-install/Manual_Install_CentOS6.html | 1352 +-
.../ambari/elasticsearch-mpack/index.html | 190 +
.../packaging/ambari/index.html | 1150 +-
.../packaging/ambari/metron-mpack/index.html | 337 +
.../packaging/docker/ansible-docker/index.html | 432 +-
.../packaging/docker/deb-docker/index.html | 194 +
.../packaging/docker/rpm-docker/index.html | 430 +-
.../packaging/packer-build/index.html | 382 +-
.../metron-deployment/roles/index.html | 332 -
.../metron-deployment/roles/monit/index.html | 388 -
.../roles/opentaxii/index.html | 480 -
.../roles/pcap_replay/index.html | 355 -
.../roles/sensor-stubs/index.html | 401 -
.../roles/sensor-test-mode/index.html | 345 -
.../vagrant/codelab-platform/index.html | 383 -
.../vagrant/fastcapa-test-platform/index.html | 416 -
.../vagrant/full-dev-platform/index.html | 420 -
.../metron-deployment/vagrant/index.html | 305 -
.../vagrant/quick-dev-platform/index.html | 406 -
.../metron-interface/metron-alerts/index.html | 434 +-
.../metron-interface/metron-config/index.html | 415 +-
.../metron-interface/metron-rest/index.html | 2661 ++--
.../Performance-tuning-guide.html | 1123 +-
site/current-book/metron-platform/index.html | 346 +-
.../metron-platform/metron-api/index.html | 387 +-
.../metron-platform/metron-common/index.html | 877 +-
.../metron-data-management/index.html | 1348 +-
.../metron-elasticsearch/index.html | 706 +-
.../metron-enrichment/Performance.html | 802 ++
.../metron-enrichment/index.html | 760 +-
.../metron-platform/metron-indexing/index.html | 547 +-
.../metron-management/index.html | 1178 +-
.../metron-parsers/3rdPartyParser.html | 467 +
.../metron-platform/metron-parsers/index.html | 870 +-
.../metron-parsers/parser-testing.html | 434 +-
.../metron-pcap-backend/index.html | 503 +-
.../metron-platform/metron-writer/index.html | 377 +-
.../metron-sensors/bro-plugin-kafka/index.html | 519 -
.../metron-sensors/fastcapa/index.html | 1239 +-
site/current-book/metron-sensors/index.html | 299 +-
.../metron-sensors/pycapa/index.html | 605 +-
.../stellar-3rd-party-example/index.html | 263 +-
.../stellar-common/3rdPartyStellar.html | 330 +-
.../metron-stellar/stellar-common/index.html | 3603 ++----
.../metron-stellar/stellar-zeppelin/index.html | 239 +
.../use-cases/forensic_clustering/index.html | 517 +-
.../geographic_login_outliers/index.html | 442 +-
site/current-book/use-cases/index.html | 283 +-
.../use-cases/typosquat_detection/index.html | 545 +
site/documentation/index.md | 14 +-
287 files changed, 26742 insertions(+), 33226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
----------------------------------------------------------------------
diff --cc metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
index 82b9c11,1c92fcb..54759e4
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java
@@@ -18,7 -18,8 +18,8 @@@
package org.apache.metron.rest.service.impl;
import static org.apache.metron.common.Constants.ERROR_TYPE;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
+ import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
-import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME;
import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY;
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index ab6c40c,ee3ca89..f73a640
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@@ -18,21 -18,17 +18,23 @@@
package org.apache.metron.elasticsearch.dao;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+ import org.apache.lucene.search.join.ScoreMode;
import org.apache.metron.common.Constants;
+ import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.MetaAlertDao;
import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.RetrieveLatestDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
-import org.apache.metron.indexing.dao.metaalert.MetaScores;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@@ -247,4 -714,12 +277,5 @@@ public class ElasticsearchMetaAlertDao
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
+
- private String getFieldName(String globalConfigKey, String defaultFieldName) {
- if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) {
- return defaultFieldName;
- }
- Map<String, Object> globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get();
- return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName);
- }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 6fa6956,adc1a27..99f0490
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@@ -18,14 -18,18 +18,13 @@@
package org.apache.metron.elasticsearch.integration;
-import static org.apache.metron.common.Constants.SENSOR_TYPE;
-import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY;
-import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD;
-import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX;
-import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD;
-import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE;
-import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD;
-import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY;
+import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.METAALERTS_INDEX;
- import static org.apache.metron.elasticsearch.dao.ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.ALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_DOC;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_FIELD;
+import static org.apache.metron.indexing.dao.metaalert.MetaAlertConstants.METAALERT_TYPE;
import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
@@@ -46,12 -52,23 +45,13 @@@ import org.apache.metron.elasticsearch.
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.MetaAlertDao;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateRequest;
-import org.apache.metron.indexing.dao.metaalert.MetaAlertCreateResponse;
++import org.apache.metron.indexing.dao.metaalert.MetaAlertDao;
+import org.apache.metron.indexing.dao.metaalert.MetaAlertIntegrationTest;
import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.search.Group;
-import org.apache.metron.indexing.dao.search.GroupRequest;
-import org.apache.metron.indexing.dao.search.GroupResponse;
-import org.apache.metron.indexing.dao.search.GroupResult;
-import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
-import org.apache.metron.indexing.dao.update.Document;
-import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
-import org.apache.metron.indexing.dao.update.PatchRequest;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@@ -119,6 -196,12 +119,12 @@@ public class ElasticsearchMetaAlertInte
.withIndexDir(new File(INDEX_DIR))
.build();
es.start();
+ }
+
+ @Before
+ public void setup() throws IOException {
- es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert"));
++ es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert"));
+ es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index"));
AccessConfig accessConfig = new AccessConfig();
Map<String, Object> globalConfig = new HashMap<String, Object>() {
@@@ -135,19 -218,9 +141,11 @@@
esDao = new ElasticsearchDao();
esDao.init(accessConfig);
- metaDao = new ElasticsearchMetaAlertDao(esDao);
+ ElasticsearchMetaAlertDao elasticsearchMetaDao = new ElasticsearchMetaAlertDao(esDao);
+ elasticsearchMetaDao.setPageSize(5);
+ metaDao = elasticsearchMetaDao;
}
- @Before
- public void setup() throws IOException {
- es.createIndexWithMapping(METAALERTS_INDEX, METAALERT_DOC,
- template.replace("%MAPPING_NAME%", "metaalert"));
- es.createIndexWithMapping(
- INDEX_WITH_SEPARATOR, "index_doc", template.replace("%MAPPING_NAME%", "index"));
- }
-
@AfterClass
public static void teardown() {
if (es != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
index a055db5,0000000..daa5424
mode 100644,000000..100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/MetaAlertConstants.java
@@@ -1,30 -1,0 +1,30 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.metaalert;
+
+public class MetaAlertConstants {
+ public static String METAALERT_TYPE = "metaalert";
+ public static String METAALERT_FIELD = "metaalerts";
+ public static String METAALERT_DOC = METAALERT_TYPE + "_doc";
+ public static String THREAT_FIELD_DEFAULT = "threat:triage:score";
+ public static String THREAT_SORT_DEFAULT = "sum";
- public static String ALERT_FIELD = "alert";
++ public static String ALERT_FIELD = "metron_alert";
+ public static String STATUS_FIELD = "status";
+ public static String GROUPS_FIELD = "groups";
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
index 918a4ae,256f23b..2f9b285
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrIndexingIntegrationTest.java
@@@ -17,16 -17,10 +17,16 @@@
*/
package org.apache.metron.solr.integration;
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+
import com.google.common.base.Function;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import javax.annotation.Nullable;
import org.apache.metron.common.configuration.Configurations;
import org.apache.metron.common.configuration.ConfigurationsUtils;
- import org.apache.metron.common.interfaces.FieldNameConverter;
+ import org.apache.metron.common.field.FieldNameConverter;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.enrichment.integration.utils.SampleUtil;
import org.apache.metron.indexing.integration.IndexingIntegrationTest;
@@@ -39,10 -33,14 +39,11 @@@ import org.apache.metron.integration.co
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.solr.integration.components.SolrComponent;
-import javax.annotation.Nullable;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+
public class SolrIndexingIntegrationTest extends IndexingIntegrationTest {
- private String collection = "metron";
+ private String collection = "yaf";
+
private FieldNameConverter fieldNameConverter = fieldName -> fieldName;
@Override
public FieldNameConverter getFieldNameConverter() {
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
index 2b523d8,0000000..1a8e290
mode 100644,000000..100644
--- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
+++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/schema/SchemaValidationIntegrationTest.java
@@@ -1,196 -1,0 +1,201 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.solr.integration.schema;
+
+import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.writer.BulkWriterResponse;
+import org.apache.metron.solr.integration.components.SolrComponent;
+import org.apache.metron.solr.writer.SolrWriter;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.storm.tuple.Tuple;
+import org.apache.zookeeper.KeeperException;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.*;
+
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+import static org.mockito.Mockito.mock;
+
+public class SchemaValidationIntegrationTest {
+ public static Iterable<String> getData(String sensor) throws IOException {
+ return Iterables.filter(
+ Files.readLines(new File("src/test/resources/example_data/" + sensor), Charset.defaultCharset()),
+ s -> !s.startsWith("#") && s.length() > 0
+ );
+ }
+
+ public static Map<String, Object> getGlobalConfig(String sensorType, SolrComponent component) {
+ Map<String, Object> globalConfig = new HashMap<>();
+ globalConfig.put(SOLR_ZOOKEEPER, component.getZookeeperUrl());
+ return globalConfig;
+ }
+
+ public static SolrComponent createSolrComponent(String sensor) throws Exception {
+ return new SolrComponent.Builder().build();
+ }
+
+ @Test
+ public void testError() throws Exception {
+ test("error");
+ }
+
+ @Test
+ public void testBro() throws Exception {
+ test("bro");
+ }
+
+ @Test
+ public void testSnort() throws Exception {
+ test("snort");
+ }
+
+ @Test
+ public void testYaf() throws Exception {
+ test("yaf");
+ }
+
+ public String getGuid(Map<String, Object> m) {
+ if(m.containsKey("guid")) {
+ return (String)m.get("guid");
+ }
+ else {
+ return (String) m.get("original_string");
+ }
+ }
+
+ public void test(String sensorType) throws Exception {
+ SolrComponent component = null;
+ try {
+ component = createSolrComponent(sensorType);
+ component.start();
+ component.addCollection(String.format("%s", sensorType), String.format("src/main/config/schema/%s", sensorType));
+ Map<String, Object> globalConfig = getGlobalConfig(sensorType, component);
+
+ List<JSONObject> inputs = new ArrayList<>();
+ List<Tuple> tuples = new ArrayList<>();
+ Map<String, Map<String, Object>> index = new HashMap<>();
+ for (String message : getData(sensorType)) {
+ if (message.trim().length() > 0) {
+ Tuple t = mock(Tuple.class);
+ tuples.add(t);
+ Map<String, Object> m = JSONUtils.INSTANCE.load(message.trim(), JSONUtils.MAP_SUPPLIER);
+ String guid = getGuid(m);
+ index.put(guid, m);
+ inputs.add(new JSONObject(m));
+ }
+ }
+ Assert.assertTrue(inputs.size() > 0);
+
+ SolrWriter solrWriter = new SolrWriter();
+
+ WriterConfiguration writerConfig = new WriterConfiguration() {
+ @Override
+ public int getBatchSize(String sensorName) {
+ return inputs.size();
+ }
+
+ @Override
+ public int getBatchTimeout(String sensorName) {
+ return 0;
+ }
+
+ @Override
+ public List<Integer> getAllConfiguredTimeouts() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public String getIndex(String sensorName) {
+ return sensorType;
+ }
+
+ @Override
+ public boolean isEnabled(String sensorName) {
+ return true;
+ }
+
+ @Override
+ public Map<String, Object> getSensorConfig(String sensorName) {
+ return new HashMap<String, Object>() {{
+ put("index", sensorType);
+ put("batchSize", inputs.size());
+ put("enabled", true);
+ }};
+ }
+
+ @Override
+ public Map<String, Object> getGlobalConfig() {
+ return globalConfig;
+ }
+
+ @Override
+ public boolean isDefault(String sensorName) {
+ return false;
+ }
++
++ @Override
++ public String getFieldNameConverter(String sensorName) {
++ return null;
++ }
+ };
+
+ solrWriter.init(null, null, writerConfig);
+
+ BulkWriterResponse response = solrWriter.write(sensorType, writerConfig, tuples, inputs);
+ Assert.assertTrue(response.getErrors().isEmpty());
+ for (Map<String, Object> m : component.getAllIndexedDocs(sensorType)) {
+ Map<String, Object> expected = index.get(getGuid(m));
+ for (Map.Entry<String, Object> field : expected.entrySet()) {
+ if (field.getValue() instanceof Collection && ((Collection) field.getValue()).size() == 0) {
+ continue;
+ }
+ if(m.get(field.getKey()) instanceof Number) {
+ Number n1 = ConversionUtils.convert(field.getValue(), Double.class);
+ Number n2 = (Number)m.get(field.getKey());
+ boolean isSame = Math.abs(n1.doubleValue() - n2.doubleValue()) < 1e-3;
+ if(!isSame) {
+ String s1 = "" + n1.doubleValue();
+ String s2 = "" + n2.doubleValue();
+ isSame = s1.startsWith(s2) || s2.startsWith(s1);
+ }
+ Assert.assertTrue("Unable to validate " + field.getKey() + ": " + n1 + " != " + n2, isSame);
+ }
+ else {
+ Assert.assertEquals("Unable to find " + field.getKey(), "" + field.getValue(), "" + m.get(field.getKey()));
+ }
+ }
+ }
+ }
+ finally {
+ if(component != null) {
+ component.stop();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/metron-platform/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/41708876/pom.xml
----------------------------------------------------------------------