You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:33:06 UTC
[50/50] [abbrv] metron git commit: Resolve merge conflicts with
master.
Resolve merge conflicts with master.
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f30e9cc1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f30e9cc1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f30e9cc1
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: f30e9cc1fb9b6985e7ad7582c764d035b5cb0e1e
Parents: 9cee51e cbdaee1
Author: Michael Miklavcic <mi...@gmail.com>
Authored: Tue Jul 10 19:31:56 2018 -0600
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Jul 10 19:31:56 2018 -0600
----------------------------------------------------------------------
dependencies_with_url.csv | 3 +
.../metron/maas/service/runner/Runner.java | 2 +-
metron-analytics/metron-profiler/.gitignore | 1 +
.../profiler/bolt/ProfileBuilderBolt.java | 4 +-
metron-deployment/Kerberos-manual-setup.md | 441 ++---
.../ansible/playbooks/metron_full_install.yml | 4 +
.../ansible/playbooks/solr_install.yml | 40 +
.../roles/ambari_config/vars/single_node_vm.yml | 1 +
.../roles/sensor-stubs/templates/start-bro-stub | 25 +-
.../sensor-stubs/templates/start-snort-stub | 25 +-
.../roles/sensor-stubs/templates/start-yaf-stub | 26 +-
.../ansible/roles/solr/defaults/main.yml | 23 +
.../ansible/roles/solr/meta/main.yml | 19 +
.../ansible/roles/solr/tasks/main.yml | 36 +
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 8 +-
.../METRON/CURRENT/configuration/metron-env.xml | 38 +-
.../configuration/metron-indexing-env.xml | 32 +-
.../common-services/METRON/CURRENT/metainfo.xml | 4 +
.../package/scripts/indexing_commands.py | 76 +-
.../CURRENT/package/scripts/indexing_master.py | 48 +-
.../CURRENT/package/scripts/metron_service.py | 102 +-
.../package/scripts/params/params_linux.py | 21 +-
.../package/scripts/params/status_params.py | 3 +
.../CURRENT/package/scripts/rest_commands.py | 1 +
.../METRON/CURRENT/themes/metron_theme.json | 18 +-
.../docker/rpm-docker/SPECS/metron.spec | 17 +
metron-interface/metron-alerts/.gitignore | 1 +
metron-interface/metron-alerts/README.md | 3 +-
.../alert-filters/alert-filters.e2e-spec.ts | 2 +-
.../e2e/alerts-list/alerts-list.po.ts | 6 +-
.../meta-alerts/meta-alert.e2e-spec.ts | 2 +-
.../table-view/table-view.e2e-spec.ts | 90 +
.../e2e/alerts-list/table-view/table-view.po.ts | 26 +
.../alerts-list/tree-view/tree-view.e2e-spec.ts | 4 +-
.../e2e/mock-data/alerts_ui_e2e_index.data | 16 +-
.../metron-alerts/e2e/utils/e2e_util.ts | 62 +-
metron-interface/metron-alerts/karma.conf.js | 2 +-
.../metron-alerts/protractor.conf.js | 1 +
.../alert-details/alert-details.component.ts | 135 +-
.../alert-filters.component.spec.ts | 14 +-
.../alerts-list/alerts-list.component.html | 5 +-
.../alerts/alerts-list/alerts-list.component.ts | 7 +-
.../table-view/table-view.component.html | 12 +-
.../table-view/table-view.component.spec.ts | 49 +-
.../table-view/table-view.component.ts | 16 +-
.../tree-view/tree-view.component.spec.ts | 32 +-
.../tree-view/tree-view.component.ts | 14 +-
.../configure-rows.component.spec.ts | 18 +-
.../save-search/save-search.component.spec.ts | 16 +
.../saved-searches.component.spec.ts | 27 +-
.../metron-alerts/src/app/app.component.spec.ts | 26 +-
.../src/app/login/login.component.spec.ts | 40 +-
.../src/app/model/comment-add-remove-request.ts | 25 +
.../src/app/service/update.service.ts | 31 +-
.../shared/collapse/collapse.component.spec.ts | 12 +-
.../app/shared/collapse/collapse.component.ts | 2 +-
.../date-picker/date-picker.component.spec.ts | 4 +
.../directives/alert-search.directive.spec.ts | 21 -
.../alert-severity-hexagon.directive.spec.ts | 5 +-
.../directives/nav-content.directive.spec.ts | 6 +-
.../list-group/list-group.component.spec.ts | 2 +-
.../metron-sorter.component.spec.ts | 14 +-
.../metron-table-pagination.component.spec.ts | 2 +-
.../shared/pipes/center-ellipses.pipe.spec.ts | 2 +-
.../pipes/column-name-translate.pipe.spec.ts | 2 +-
.../src/app/shared/pipes/map-keys.pipe.spec.ts | 2 +-
.../app/shared/switch/switch.component.spec.ts | 2 +-
.../time-range/time-range.component.spec.ts | 12 +-
.../metron-alerts/src/app/utils/constants.ts | 1 -
metron-interface/metron-rest/README.md | 1 +
.../apache/metron/rest/config/IndexConfig.java | 14 +-
.../rest/controller/UpdateController.java | 23 +
.../metron/rest/service/UpdateService.java | 3 +
.../rest/service/impl/MetaAlertServiceImpl.java | 5 +-
.../rest/service/impl/SearchServiceImpl.java | 2 +-
.../rest/service/impl/UpdateServiceImpl.java | 21 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 32 +-
.../MetaAlertControllerIntegrationTest.java | 6 +-
.../UpdateControllerIntegrationTest.java | 96 +-
.../org/apache/metron/common/Constants.java | 2 +-
.../configuration/SensorParserConfig.java | 69 +-
.../apache/metron/common/error/MetronError.java | 6 +-
.../transformation/FieldTransformations.java | 1 +
.../RegexSelectTransformation.java | 97 ++
.../metadata/DefaultRawMessageStrategy.java | 86 +
.../metadata/EnvelopedRawMessageStrategy.java | 146 ++
.../common/message/metadata/MetadataUtil.java | 124 ++
.../common/message/metadata/RawMessage.java | 85 +
.../message/metadata/RawMessageStrategies.java | 81 +
.../message/metadata/RawMessageStrategy.java | 77 +
.../common/message/metadata/RawMessageUtil.java | 59 +
.../apache/metron/common/utils/JSONUtils.java | 4 +-
.../common/bolt/ConfiguredParserBoltTest.java | 31 +-
.../RegexSelectTransformationTest.java | 169 ++
.../message/metadata/RawMessageUtilTest.java | 282 +++
.../src/main/config/elasticsearch.properties.j2 | 2 +-
.../elasticsearch/dao/ColumnMetadataDao.java | 67 -
.../dao/ElasticsearchColumnMetadataDao.java | 31 +-
.../elasticsearch/dao/ElasticsearchDao.java | 670 +-------
.../dao/ElasticsearchMetaAlertDao.java | 669 ++------
...ElasticsearchMetaAlertRetrieveLatestDao.java | 44 +
.../dao/ElasticsearchMetaAlertSearchDao.java | 110 ++
.../dao/ElasticsearchMetaAlertUpdateDao.java | 254 +++
.../dao/ElasticsearchRetrieveLatestDao.java | 151 ++
.../dao/ElasticsearchSearchDao.java | 467 +++++
.../dao/ElasticsearchUpdateDao.java | 200 +++
.../elasticsearch/utils/ElasticsearchUtils.java | 64 +
.../elasticsearch/dao/ElasticsearchDaoTest.java | 68 +-
.../dao/ElasticsearchMetaAlertDaoTest.java | 238 +--
.../ElasticsearchMetaAlertIntegrationTest.java | 1224 ++-----------
.../ElasticsearchSearchIntegrationTest.java | 175 +-
.../ElasticsearchUpdateIntegrationTest.java | 239 +--
.../components/ElasticSearchComponent.java | 26 +-
metron-platform/metron-indexing/README.md | 16 +-
metron-platform/metron-indexing/pom.xml | 8 +-
.../metron/indexing/dao/AccessConfig.java | 22 +
.../metron/indexing/dao/ColumnMetadataDao.java | 39 +
.../apache/metron/indexing/dao/HBaseDao.java | 93 +-
.../apache/metron/indexing/dao/IndexDao.java | 141 +-
.../metron/indexing/dao/MetaAlertDao.java | 155 --
.../metron/indexing/dao/MultiIndexDao.java | 47 +
.../metron/indexing/dao/RetrieveLatestDao.java | 67 +
.../metaalert/DeferredMetaAlertIndexDao.java | 42 +
.../metaalert/MetaAlertAddRemoveRequest.java | 1 -
.../indexing/dao/metaalert/MetaAlertConfig.java | 83 +
.../dao/metaalert/MetaAlertConstants.java | 30 +
.../indexing/dao/metaalert/MetaAlertDao.java | 77 +
.../metaalert/MetaAlertRetrieveLatestDao.java | 25 +
.../dao/metaalert/MetaAlertSearchDao.java | 35 +
.../dao/metaalert/MetaAlertUpdateDao.java | 146 ++
.../indexing/dao/metaalert/MetaScores.java | 52 +-
.../AbstractLuceneMetaAlertUpdateDao.java | 337 ++++
.../indexing/dao/search/AlertComment.java | 130 ++
.../metron/indexing/dao/search/SearchDao.java | 38 +
.../indexing/dao/search/SearchResponse.java | 33 +-
.../dao/update/CommentAddRemoveRequest.java | 78 +
.../metron/indexing/dao/update/Document.java | 14 +-
.../metron/indexing/dao/update/UpdateDao.java | 108 ++
.../metron/indexing/util/IndexingCacheUtil.java | 43 +
.../apache/metron/indexing/dao/InMemoryDao.java | 17 +
.../indexing/dao/InMemoryMetaAlertDao.java | 86 +-
.../indexing/dao/SearchIntegrationTest.java | 321 ++--
.../indexing/dao/UpdateIntegrationTest.java | 306 ++++
.../dao/metaalert/MetaAlertIntegrationTest.java | 1012 +++++++++++
.../indexing/dao/metaalert/MetaScoresTest.java | 101 ++
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 885 ++++++++++
.../integration/HBaseDaoIntegrationTest.java | 79 +-
.../integration/IndexingIntegrationTest.java | 4 +-
.../indexing/util/IndexingCacheUtilTest.java | 66 +
.../components/FluxTopologyComponent.java | 9 +
.../metron-parsers/ParserChaining.md | 179 ++
metron-platform/metron-parsers/README.md | 63 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 60 +-
.../EnvelopedParserIntegrationTest.java | 209 +++
.../parsers/integration/ParserDriver.java | 2 +-
.../integration/WriterBoltIntegrationTest.java | 75 +-
.../src/test/resources/patterns/cisco_patterns | 6 +
metron-platform/metron-pcap-backend/.gitignore | 1 +
metron-platform/metron-solr/README.md | 170 ++
metron-platform/metron-solr/pom.xml | 41 +-
.../src/main/config/schema/bro/schema.xml | 700 ++++++++
.../src/main/config/schema/bro/solrconfig.xml | 1601 ++++++++++++++++++
.../src/main/config/schema/error/schema.xml | 60 +
.../src/main/config/schema/error/solrconfig.xml | 1601 ++++++++++++++++++
.../src/main/config/schema/metaalert/schema.xml | 58 +
.../main/config/schema/metaalert/solrconfig.xml | 1601 ++++++++++++++++++
.../src/main/config/schema/snort/schema.xml | 93 +
.../src/main/config/schema/snort/solrconfig.xml | 1601 ++++++++++++++++++
.../src/main/config/schema/yaf/schema.xml | 99 ++
.../src/main/config/schema/yaf/solrconfig.xml | 1601 ++++++++++++++++++
.../src/main/config/solr.properties.j2 | 2 +-
.../org/apache/metron/solr/SolrConstants.java | 2 +
.../metron/solr/dao/SolrColumnMetadataDao.java | 149 ++
.../org/apache/metron/solr/dao/SolrDao.java | 207 +++
.../metron/solr/dao/SolrMetaAlertDao.java | 257 +++
.../dao/SolrMetaAlertRetrieveLatestDao.java | 77 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 215 +++
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 239 +++
.../metron/solr/dao/SolrRetrieveLatestDao.java | 103 ++
.../apache/metron/solr/dao/SolrSearchDao.java | 278 +++
.../apache/metron/solr/dao/SolrUpdateDao.java | 204 +++
.../apache/metron/solr/dao/SolrUtilities.java | 143 ++
.../apache/metron/solr/schema/FieldType.java | 98 ++
.../metron/solr/schema/SchemaTranslator.java | 194 +++
.../metron/solr/writer/MetronSolrClient.java | 34 +
.../apache/metron/solr/writer/SolrWriter.java | 250 ++-
.../src/main/scripts/create_collection.sh | 36 +
.../src/main/scripts/delete_collection.sh | 33 +
.../src/main/scripts/install_solr.sh | 62 +
.../metron-solr/src/main/scripts/start_solr.sh | 27 +
.../src/main/scripts/start_solr_topology.sh | 2 +-
.../metron-solr/src/main/scripts/stop_solr.sh | 27 +
.../metron/solr/dao/SolrColumnMetadataTest.java | 149 ++
.../org/apache/metron/solr/dao/SolrDaoTest.java | 185 ++
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 155 ++
.../metron/solr/dao/SolrSearchDaoTest.java | 478 ++++++
.../metron/solr/dao/SolrUpdateDaoTest.java | 241 +++
.../metron/solr/dao/SolrUtilitiesTest.java | 48 +
.../SolrIndexingIntegrationTest.java | 19 +-
.../SolrMetaAlertIntegrationTest.java | 412 +++++
.../SolrRetrieveLatestIntegrationTest.java | 207 +++
.../integration/SolrSearchIntegrationTest.java | 242 +++
.../integration/SolrUpdateIntegrationTest.java | 189 +++
.../integration/components/SolrComponent.java | 105 +-
.../schema/SchemaValidationIntegrationTest.java | 201 +++
.../matcher/ModifiableSolrParamsMatcher.java | 55 +
.../matcher/SolrInputDocumentListMatcher.java | 60 +
.../solr/matcher/SolrInputDocumentMatcher.java | 50 +
.../metron/solr/matcher/SolrQueryMatcher.java | 56 +
.../metron/solr/writer/SolrWriterTest.java | 228 ++-
.../resources/config/test/conf/managed-schema | 77 +
.../resources/config/test/conf/solrconfig.xml | 1601 ++++++++++++++++++
.../src/test/resources/example_data/bro | 29 +
.../src/test/resources/example_data/error | 17 +
.../src/test/resources/example_data/snort | 21 +
.../src/test/resources/example_data/yaf | 21 +
.../test/error/MetronErrorJSONMatcher.java | 2 +
metron-platform/metron-writer/README.md | 24 +
.../apache/metron/writer/kafka/KafkaWriter.java | 43 +-
.../metron/writer/kafka/KafkaWriterTest.java | 63 +
metron-platform/pom.xml | 4 +-
metron-stellar/stellar-common/README.md | 18 +-
.../metron/stellar/common/StellarCompiler.java | 82 +-
.../org/apache/metron/stellar/dsl/Token.java | 1 +
.../stellar/dsl/functions/RegExFunctions.java | 27 +-
.../stellar/dsl/functions/BasicStellarTest.java | 25 +
.../metron/stellar/dsl/functions/MatchTest.java | 35 +
.../dsl/functions/RegExFunctionsTest.java | 7 +
pom.xml | 4 +-
use-cases/forensic_clustering/README.md | 2 +-
use-cases/geographic_login_outliers/README.md | 5 +-
use-cases/parser_chaining/README.md | 235 +++
.../message_routing_high_level.svg | 14 +
.../message_routing_high_level.xml | 14 +
use-cases/typosquat_detection/README.md | 2 +-
236 files changed, 26712 insertions(+), 3958 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/dependencies_with_url.csv
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 73bafbb,0ff6fb1..2fd3f0d
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@@ -161,7 -125,7 +161,8 @@@ class RestCommands
"export METRON_INDEX_CP={metron_indexing_classpath};"
"export METRON_LOG_DIR={metron_log_dir};"
"export METRON_PID_FILE={pid_file};"
+ "export HDP_VERSION={hdp_version};"
+ "export METRON_RA_INDEXING_WRITER={ra_indexing_writer};"
"{metron_home}/bin/metron-rest.sh;"
"unset METRON_JDBC_PASSWORD;"
))
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------
diff --cc metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index cf7bf58,c0c9fac..7c89ae5
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@@ -109,8 -112,21 +112,21 @@@ METRON_REST_CLASSPATH+=":${indexing_fil
echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
+ #Use Solr daos if ra indexing writer set to Solr
+ if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then
+ METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao"
+ METRON_METAALERT_DAO=" --meta.dao.impl=org.apache.metron.solr.dao.SolrMetaAlertDao"
+ METRON_WRITER_NAME=" --index.writer.name=solr"
+ echo "METRON_INDEX_DAO=${METRON_INDEX_DAO}"
+ echo "METRON_METAALERT_DAO=${METRON_METAALERT_DAO}"
+ echo "METRON_WRITER_NAME=${METRON_WRITER_NAME}"
+ METRON_SPRING_OPTIONS+=${METRON_INDEX_DAO}
+ METRON_SPRING_OPTIONS+=${METRON_METAALERT_DAO}
+ METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME}
+ fi
+
echo "Starting application"
-${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \
+${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \
-cp ${METRON_REST_CLASSPATH} \
org.apache.metron.rest.MetronRestApplication \
${METRON_SPRING_OPTIONS} >> ${METRON_LOG_DIR}/metron-rest.log 2>&1 & echo $! > ${METRON_PID_FILE};
http://git-wip-us.apache.org/repos/asf/metron/blob/f30e9cc1/metron-platform/pom.xml
----------------------------------------------------------------------