You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2019/09/06 14:50:31 UTC
[metron] 01/02: Merge remote-tracking branch 'apache/master' into
MERGE-MASTER-SEPT
This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch feature/METRON-2088-support-hdp-3.1
in repository https://gitbox.apache.org/repos/asf/metron.git
commit e6359655733395c4c2a45b126611a823909075f7
Merge: f77946d c402e64
Author: Nick Allen <ni...@nickallen.org>
AuthorDate: Wed Sep 4 17:15:12 2019 -0400
Merge remote-tracking branch 'apache/master' into MERGE-MASTER-SEPT
Upgrading.md | 9 +
dependencies_with_url.csv | 2 +-
metron-analytics/metron-maas-common/pom.xml | 1 +
.../java/org/apache/metron/maas/util/RESTUtil.java | 4 +-
metron-analytics/metron-maas-service/pom.xml | 6 +
.../apache/metron/maas/service/runner/Runner.java | 3 +-
.../metron/maas/service/MaasIntegrationTest.java | 7 +-
metron-analytics/metron-profiler-client/pom.xml | 8 +
.../src/main/assembly/assembly.xml | 2 +-
.../profiler/client/HBaseProfilerClient.java | 34 +-
.../metron/profiler/client/stellar/GetProfile.java | 61 +-
.../profiler/client/stellar/VerboseProfile.java | 57 +-
.../profiler/client/HBaseProfilerClientTest.java | 30 +-
.../metron/profiler/client/ProfileWriter.java | 25 +-
.../profiler/client/stellar/GetProfileTest.java | 35 +-
.../client/stellar/VerboseProfileTest.java | 50 +-
metron-analytics/metron-profiler-common/pom.xml | 6 +
.../profiler/DefaultMessageDistributorTest.java | 6 +-
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 33 +-
metron-analytics/metron-profiler-repl/pom.xml | 8 +
.../src/main/assembly/assembly.xml | 2 +-
.../profiler/repl/StandAloneProfilerTest.java | 3 +-
metron-analytics/metron-profiler-spark/pom.xml | 3 +
.../src/main/assembly/assembly.xml | 2 +-
.../metron/profiler/spark/BatchProfilerConfig.java | 11 +-
.../spark/function/MessageRouterFunction.java | 9 +-
.../src/main/scripts/start_batch_profiler.sh | 4 +-
metron-analytics/metron-profiler-storm/pom.xml | 8 +
.../metron/profiler/storm/ProfileSplitterBolt.java | 11 +-
.../src/main/scripts/start_profiler_topology.sh | 3 +-
.../profiler/storm/ProfileSplitterBoltTest.java | 6 +-
.../storm/integration/ConfigUploadComponent.java | 3 +-
.../storm/integration/ProfilerIntegrationTest.java | 4 +-
metron-analytics/metron-statistics/pom.xml | 3 +
metron-contrib/metron-performance/pom.xml | 8 +
.../src/main/assembly/assembly.xml | 2 +-
.../metron/performance/load/LoadOptions.java | 8 +-
.../load/monitor/writers/CSVWriter.java | 5 +-
.../src/main/scripts/load_tool.sh | 2 +-
.../metron/performance/load/LoadOptionsTest.java | 5 +-
metron-deployment/development/centos6/README.md | 5 +-
.../5.6.14/package/scripts/elastic_commands.py | 14 +-
.../5.6.14/package/scripts/elastic_master.py | 7 +-
.../5.6.14/package/scripts/elastic_slave.py | 3 +-
.../5.6.14/package/scripts/service_check.py | 4 +-
.../KIBANA/5.6.14/package/scripts/common.py | 8 +-
.../KIBANA/5.6.14/package/scripts/kibana_master.py | 2 +-
.../CURRENT/configuration/metron-parsers-env.xml | 2 +-
.../common-services/METRON/CURRENT/metainfo.xml | 5 +
.../CURRENT/package/scripts/alerts_ui_commands.py | 4 +-
.../CURRENT/package/scripts/alerts_ui_master.py | 10 +-
.../package/scripts/dashboard/dashboardindex.py | 10 +-
.../CURRENT/package/scripts/enrichment_commands.py | 6 +-
.../CURRENT/package/scripts/enrichment_master.py | 10 +-
.../CURRENT/package/scripts/indexing_commands.py | 79 +-
.../CURRENT/package/scripts/indexing_master.py | 32 +-
.../package/scripts/management_ui_commands.py | 6 +-
.../package/scripts/management_ui_master.py | 7 +-
.../CURRENT/package/scripts/metron_client.py | 11 +-
.../CURRENT/package/scripts/metron_security.py | 7 +-
.../CURRENT/package/scripts/metron_service.py | 12 +-
.../CURRENT/package/scripts/params/params.py | 2 +-
.../CURRENT/package/scripts/params/params_linux.py | 9 +-
.../package/scripts/params/status_params.py | 4 +-
.../CURRENT/package/scripts/parser_commands.py | 9 +-
.../CURRENT/package/scripts/parser_master.py | 7 +-
.../CURRENT/package/scripts/pcap_commands.py | 11 +-
.../METRON/CURRENT/package/scripts/pcap_master.py | 13 +-
.../CURRENT/package/scripts/profiler_commands.py | 6 +-
.../CURRENT/package/scripts/profiler_master.py | 13 +-
.../CURRENT/package/scripts/rest_commands.py | 6 +-
.../METRON/CURRENT/package/scripts/rest_master.py | 8 +-
.../CURRENT/package/scripts/service_check.py | 14 +-
.../package/templates/alerts-ui-app-config.json.j2 | 4 +-
.../templates/management-ui-app-config.json.j2 | 4 +-
.../METRON/CURRENT/service_advisor.py | 6 +-
.../packaging/docker/rpm-docker/SPECS/metron.spec | 35 +-
.../packaging/docker/rpm-docker/pom.xml | 6 +
metron-interface/metron-alerts/package-lock.json | 31972 ++++++++-----------
metron-interface/metron-alerts/package.json | 40 +-
.../alert-details/alert-details.component.spec.ts | 4 +-
.../alerts/alerts-list/alerts-list.component.html | 43 +-
.../alerts/alerts-list/alerts-list.component.scss | 24 +
.../alerts-list/alerts-list.component.spec.ts | 148 +-
.../alerts/alerts-list/alerts-list.component.ts | 139 +-
.../app/alerts/alerts-list/alerts-list.module.ts | 89 +-
.../app/alerts/alerts-list/query-builder.spec.ts | 165 +
.../src/app/alerts/alerts-list/query-builder.ts | 42 +-
.../table-view/table-view.component.spec.ts | 7 +-
.../alerts-list/table-view/table-view.component.ts | 29 +-
.../alerts-list/tree-view/tree-view.component.html | 2 +-
.../tree-view/tree-view.component.spec.ts | 13 +-
.../alerts-list/tree-view/tree-view.component.ts | 63 +-
.../configure-rows/configure-rows.component.html | 7 +-
.../configure-rows/configure-rows.component.scss | 2 +-
.../configure-rows.component.spec.ts | 14 +-
.../configure-rows/configure-rows.component.ts | 5 +-
.../alerts/configure-rows/configure-rows.module.ts | 22 +-
.../show-hide-alert-entries.component.spec.ts | 132 +
.../show-hide/show-hide-alert-entries.component.ts | 51 +
.../show-hide/show-hide.service.spec.ts | 125 +
.../configure-rows/show-hide/show-hide.service.ts | 70 +
.../metron-alerts/src/app/app.component.html | 69 +-
.../metron-alerts/src/app/app.component.scss | 31 +-
.../metron-alerts/src/app/app.component.spec.ts | 37 +-
.../metron-alerts/src/app/app.component.ts | 37 +-
.../metron-alerts/src/app/app.module.ts | 30 +-
.../metron-alerts/src/app/model/filter.spec.ts | 42 +-
.../metron-alerts/src/app/model/filter.ts | 69 +-
.../src/app/service/app-config.service.ts | 14 +
.../app/service/elasticsearch-localstorage-impl.ts | 2 +-
.../src/app/service/search.service.ts | 5 +-
.../shared/context-menu/context-menu.component.ts | 18 +-
.../src/app/shared/switch/switch.component.html | 2 +-
.../src/app/shared/switch/switch.component.ts | 12 +-
.../src/app/shared/switch/switch.module.ts | 15 +-
.../shared/time-range/time-range.component.scss | 4 +-
.../metron-alerts/src/app/utils/constants.ts | 18 +-
.../metron-alerts/src/app/utils/httpUtil.ts | 4 +-
.../metron-alerts/src/app/utils/utils.ts | 16 +-
.../metron-alerts/src/assets/app-config.json | 4 +-
.../metron-alerts/src/assets/images/logo-name.png | Bin 0 -> 3760 bytes
.../src/assets/images/logo-symbol.png | Bin 0 -> 19063 bytes
metron-interface/metron-alerts/src/slider.scss | 3 +-
metron-interface/metron-alerts/src/styles.scss | 64 +
metron-interface/metron-config/angular.json | 1 -
metron-interface/metron-config/package-lock.json | 5931 ++--
metron-interface/metron-config/package.json | 43 +-
.../metron-config/src/app/app.component.html | 69 +-
.../metron-config/src/app/app.component.scss | 23 +
.../metron-config/src/app/app.component.ts | 37 +-
.../metron-config/src/app/app.module.ts | 21 +-
.../src/app/navbar/navbar.component.scss | 10 +
.../metron-config/src/app/navbar/navbar.html | 8 +-
.../sensor-rule-editor.component.spec.ts | 3 +
.../src/app/service/app-config.service.ts | 14 +
.../src/app/service/mock.app-config.service.ts | 9 +
.../sensor-enrichment-config.service.spec.ts | 4 +-
.../service/sensor-enrichment-config.service.ts | 2 +-
.../service/sensor-indexing-config.service.spec.ts | 10 +-
.../app/service/sensor-indexing-config.service.ts | 2 +-
.../metron-config/src/app/shared/metron-alerts.ts | 2 +-
.../number-spinner/number-spinner.component.html | 2 +-
.../number-spinner/number-spinner.component.scss | 9 +-
.../metron-config/src/app/util/httpUtil.ts | 2 +-
.../metron-config/src/assets/app-config.json | 4 +-
.../metron-config/src/assets/images/logo-name.png | Bin 0 -> 3760 bytes
.../src/assets/images/logo-symbol.png | Bin 0 -> 19063 bytes
metron-interface/metron-config/src/styles.scss | 66 +
metron-interface/metron-config/tsconfig.json | 1 +
metron-interface/metron-rest/pom.xml | 22 +
.../metron-rest/src/main/assembly/assembly.xml | 2 +-
.../org/apache/metron/rest/config/GrokConfig.java | 3 +-
.../org/apache/metron/rest/config/IndexConfig.java | 2 +-
.../rest/service/impl/DockerStormCLIWrapper.java | 4 +-
.../metron/rest/service/impl/GrokServiceImpl.java | 5 +-
.../impl/SensorEnrichmentConfigServiceImpl.java | 15 +-
.../impl/SensorIndexingConfigServiceImpl.java | 4 +-
.../impl/SensorParserConfigServiceImpl.java | 6 +-
.../metron/rest/service/impl/StormCLIWrapper.java | 4 +-
.../metron/rest/user/UserSettingsClient.java | 14 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 2 +
.../apache/metron/rest/config/HBaseConfigTest.java | 8 -
.../org/apache/metron/rest/config/TestConfig.java | 33 +-
.../controller/GrokControllerIntegrationTest.java | 3 +-
.../controller/HdfsControllerIntegrationTest.java | 3 +-
.../controller/PcapControllerIntegrationTest.java | 5 +-
...rEnrichmentConfigControllerIntegrationTest.java | 24 +-
...sorIndexingConfigControllerIntegrationTest.java | 9 +-
.../UpdateControllerIntegrationTest.java | 3 +-
.../metron/rest/generator/SampleDataGenerator.java | 2 +-
.../rest/mock/MockPcapToPdmlScriptWrapper.java | 3 +-
.../service/impl/DockerStormCLIWrapperTest.java | 4 +-
.../service/impl/GlobalConfigServiceImplTest.java | 8 +-
.../rest/service/impl/GrokServiceImplTest.java | 3 +-
.../rest/service/impl/PcapServiceImplTest.java | 5 +-
.../SensorEnrichmentConfigServiceImplTest.java | 30 +-
.../impl/SensorIndexingConfigServiceImplTest.java | 8 +-
.../impl/SensorParserConfigServiceImplTest.java | 12 +-
.../metron/rest/user/UserSettingsClientTest.java | 22 +-
metron-platform/elasticsearch-shaded/pom.xml | 2 +
.../metron-common-storm/pom.xml | 6 +
.../common/message/metadata/RawMessageUtil.java | 3 +-
.../message/metadata/RawMessageUtilTest.java | 84 +-
metron-platform/metron-common/pom.xml | 51 +-
.../metron-common/src/main/assembly/assembly.xml | 2 +-
.../common/configuration/ConfigurationsUtils.java | 30 +-
.../common/configuration/SensorParserConfig.java | 3 +-
.../enrichment/SensorEnrichmentConfig.java | 3 +-
.../enrichment/SensorEnrichmentUpdateConfig.java | 12 +-
.../configuration/profiler/ProfileConfig.java | 3 +-
.../configuration/profiler/ProfilerConfig.java | 3 +-
.../apache/metron/common/error/MetronError.java | 2 +-
.../common/field/validation/SimpleValidation.java | 3 +
.../field/validation/network/IPValidation.java | 2 +-
.../field/validation/primitive/DateValidation.java | 3 +
.../metron/common/hadoop/SequenceFileIterable.java | 10 +-
.../metadata/EnvelopedRawMessageStrategy.java | 8 +-
.../org/apache/metron/common/utils/KafkaUtils.java | 3 +-
.../org/apache/metron/common/utils/LazyLogger.java | 94 +
.../metron/common/utils/LazyLoggerFactory.java | 71 +
.../apache/metron/common/utils/LazyLoggerImpl.java | 572 +
.../configurations/ConfigurationsUpdater.java | 13 +-
.../metron-common/src/main/scripts/stellar | 5 +-
.../src/main/scripts/zk_load_configs.sh | 3 +-
.../cli/ConfigurationManagerIntegrationTest.java | 9 +-
.../common/configuration/ConfigurationTest.java | 3 +-
.../configuration/ParserConfigurationsTest.java | 4 +-
.../configuration/SensorEnrichmentConfigTest.java | 10 +-
.../configuration/SensorParserConfigTest.java | 8 +-
.../writer/EnrichmentWriterConfigurationTest.java | 3 +-
.../writer/ParserWriterConfigurationTest.java | 4 +-
.../writer/ProfilerWriterConfigurationTest.java | 3 +-
.../metron/common/error/MetronErrorTest.java | 3 +-
.../common/field/FieldNameConvertersTest.java | 3 +-
.../validation/network/DomainValidationTest.java | 8 +
.../validation/network/EmailValidationTest.java | 6 +
.../validation/network/URLValidationTest.java | 8 +-
.../validation/primitive/DateValidationTest.java | 4 +
.../primitive/IntegerValidationTest.java | 6 +
.../typosquat/TyposquattingStrategiesTest.java | 4 +-
.../apache/metron/common/utils/JSONUtilsTest.java | 7 +-
.../apache/metron/common/utils/KafkaUtilsTest.java | 9 +-
.../metron/common/utils/LazyLoggerImplTest.java | 759 +
.../common/utils/LazzyLoggerImplPerfTest.java | 118 +
.../common/utils/file/ReaderSpliteratorTest.java | 5 +-
.../ZKConfigurationsCacheIntegrationTest.java | 15 +-
.../components/ConfigUploadComponent.java | 18 +-
metron-platform/metron-data-management/pom.xml | 89 +-
.../src/main/assembly/assembly.xml | 2 +-
.../metron/dataloads/hbase/mr/PrunerMapper.java | 3 +-
.../nonbulk/flatfile/HBaseExtractorState.java | 8 +-
.../nonbulk/flatfile/importer/LocalImporter.java | 12 +-
.../nonbulk/flatfile/location/RawLocation.java | 11 +-
.../dataloads/nonbulk/taxii/TaxiiHandler.java | 16 +-
.../src/main/scripts/flatfile_loader.sh | 7 +-
.../src/main/scripts/flatfile_summarizer.sh | 7 +-
.../src/main/scripts/maxmind_enrichment_load.sh | 10 +-
.../src/main/scripts/prune_hdfs_files.sh | 2 +-
.../src/main/scripts/threatintel_bulk_prune.sh | 2 +-
.../src/main/scripts/threatintel_taxii_load.sh | 2 +-
.../extractor/stix/StixExtractorTest.java | 6 +-
.../metron/dataloads/hbase/mr/HBaseUtil.java | 22 +-
.../mr/LeastRecentlyUsedPrunerIntegrationTest.java | 22 +-
...pleEnrichmentFlatFileLoaderIntegrationTest.java | 63 +-
.../flatfile/SimpleFlatFileSummarizerTest.java | 3 +-
.../nonbulk/taxii/TaxiiIntegrationTest.java | 4 +-
.../metron-elasticsearch-common/pom.xml | 26 +
.../bulk/ElasticsearchImportExport.java | 10 +-
.../ElasticsearchUpdateIntegrationTest.java | 5 -
.../metron-elasticsearch-storm/pom.xml | 19 +
.../main/scripts/start_elasticsearch_topology.sh | 5 +-
.../metron-enrichment-common/pom.xml | 1 +
.../enrichment/adapters/cif/CIFHbaseAdapter.java | 18 +-
.../adapters/simplehbase/SimpleHBaseAdapter.java | 10 +-
.../adapters/stellar/StellarAdapter.java | 2 +-
.../adapters/threatintel/ThreatIntelAdapter.java | 2 +
.../metron/enrichment/cli/LatencySummarizer.java | 4 +-
.../enrichment/converter/EnrichmentHelper.java | 7 +-
.../metron/enrichment/lookup/EnrichmentLookup.java | 18 +-
.../lookup/accesstracker/AccessTrackerUtil.java | 4 +-
.../accesstracker/PersistentAccessTracker.java | 8 +-
.../PersistentBloomTrackerCreator.java | 9 +-
.../stellar/SimpleHBaseEnrichmentFunctions.java | 20 +-
.../metron/enrichment/utils/EnrichmentUtils.java | 6 +-
.../threatintel/ThreatIntelAdapterTest.java | 4 +-
.../metron-enrichment-storm/pom.xml | 6 +
.../src/main/scripts/start_enrichment_topology.sh | 3 +-
.../integration/EnrichmentIntegrationTest.java | 21 +-
metron-platform/metron-hbase-client/pom.xml | 2 +
metron-platform/metron-hbase-server/README.md | 27 +
metron-platform/metron-hbase-server/pom.xml | 6 +
.../EnrichmentCoprocessorIntegrationTest.java | 11 +-
.../coprocessor/EnrichmentCoprocessorTest.java | 8 +-
.../metron/hbase/HBaseProjectionCriteria.java | 7 +-
.../org/apache/metron/hbase/HTableProvider.java | 46 +-
.../org/apache/metron/hbase/TableProvider.java | 8 +-
.../apache/metron/hbase/client/HBaseClient.java | 11 +-
.../metron/hbase/client/HBaseClientTest.java | 37 +-
.../metron/hbase/mock/MockHBaseTableProvider.java | 18 +-
.../org/apache/metron/hbase/mock/MockHTable.java | 38 +-
.../metron-indexing/metron-indexing-common/pom.xml | 11 +-
.../org/apache/metron/indexing/dao/HBaseDao.java | 12 +-
.../metron/indexing/dao/UpdateIntegrationTest.java | 84 +-
.../integration/HBaseDaoIntegrationTest.java | 5 -
.../metron-indexing/metron-indexing-storm/pom.xml | 6 +
.../src/main/scripts/start_hdfs_topology.sh | 3 +-
.../integration/HDFSIndexingIntegrationTest.java | 3 +-
.../integration/IndexingIntegrationTest.java | 25 +-
metron-platform/metron-integration-test/pom.xml | 1 +
.../apache/metron/integration/ProcessorResult.java | 3 +-
.../components/FluxTopologyComponent.java | 8 +-
.../apache/metron/integration/utils/TestUtils.java | 6 +-
.../sample/data/syslog5424/parsed/Syslog5424Parsed | 4 +-
.../sample/data/syslog5424/raw/Syslog5424Output | 4 +-
metron-platform/metron-management/pom.xml | 3 +
.../src/main/assembly/assembly.xml | 2 +-
.../metron/management/ConfigurationFunctions.java | 11 +-
.../apache/metron/management/GrokFunctions.java | 3 +-
.../metron/management/StellarParserRunner.java | 8 +-
.../management/ConfigurationFunctionsTest.java | 31 +-
.../metron/management/ParserFunctionsTest.java | 14 +-
metron-platform/metron-parsing/README.md | 7 +-
.../metron-parsers-common/3rdPartyParser.md | 17 +
.../metron-parsing/metron-parsers-common/pom.xml | 6 +
.../org/apache/metron/parsers/BasicParser.java | 39 +-
.../java/org/apache/metron/parsers/GrokParser.java | 35 +-
.../apache/metron/parsers/ParserRunnerImpl.java | 3 +-
.../org/apache/metron/parsers/csv/CSVParser.java | 8 +-
.../metron/parsers/interfaces/MessageParser.java | 21 +-
.../apache/metron/parsers/json/JSONMapParser.java | 5 +-
.../parsers/regex/RegularExpressionsParser.java | 25 +-
.../metron/parsers/syslog/BaseSyslogParser.java | 44 +-
.../metron/parsers/AbstractParserConfigTest.java | 9 +-
.../org/apache/metron/parsers/BasicParserTest.java | 167 +
.../org/apache/metron/parsers/GrokParserTest.java | 3 +-
.../apache/metron/parsers/MessageParserTest.java | 13 +-
.../metron/parsers/MultiLineGrokParserTest.java | 5 +-
.../parsers/MultiLineWithErrorsGrokParserTest.java | 5 +-
.../metron/parsers/ParserRunnerImplTest.java | 24 +-
.../apache/metron/parsers/csv/CSVParserTest.java | 29 +-
.../EnvelopedParserIntegrationTest.java | 16 +-
.../parsers/integration/ParserIntegrationTest.java | 9 +-
.../integration/validation/ParserDriver.java | 6 +-
.../validation/SampleDataValidation.java | 5 +-
.../parsers/json/JSONMapParserQueryTest.java | 17 +-
.../metron/parsers/json/JSONMapParserTest.java | 49 +-
.../json/JSONMapParserWrappedQueryTest.java | 15 +-
.../regex/RegularExpressionsParserTest.java | 32 +-
.../parsers/syslog/Syslog3164ParserTest.java | 9 +-
.../parsers/syslog/Syslog5424ParserTest.java | 22 +-
.../metron-parsing/metron-parsers/pom.xml | 6 +
.../apache/metron/parsers/asa/BasicAsaParser.java | 26 +-
.../apache/metron/parsers/bro/BasicBroParser.java | 12 +-
.../org/apache/metron/parsers/cef/CEFParser.java | 5 +-
.../metron/parsers/fireeye/BasicFireEyeParser.java | 8 +-
.../apache/metron/parsers/ise/BasicIseParser.java | 13 +-
.../apache/metron/parsers/ise/JavaCharStream.java | 10 +-
.../metron/parsers/lancope/BasicLancopeParser.java | 4 +-
.../org/apache/metron/parsers/leef/LEEFParser.java | 7 +-
.../parsers/logstash/BasicLogstashParser.java | 3 +-
.../paloalto/BasicPaloAltoFirewallParser.java | 13 +-
.../metron/parsers/snort/BasicSnortParser.java | 7 +-
.../parsers/sourcefire/BasicSourcefireParser.java | 4 +-
.../org/apache/metron/parsers/SnortParserTest.java | 43 +-
.../metron/parsers/asa/BasicAsaParserTest.java | 67 +-
.../metron/parsers/bro/BasicBroParserTest.java | 98 +-
.../apache/metron/parsers/cef/CEFParserTest.java | 62 +-
.../parsers/fireeye/BasicFireEyeParserTest.java | 32 +-
.../metron/parsers/ise/BasicIseParserTest.java | 23 +-
.../parsers/lancope/BasicLancopeParserTest.java | 23 +-
.../apache/metron/parsers/leef/LEEFParserTest.java | 53 +-
.../paloalto/BasicPaloAltoFirewallParserTest.java | 57 +-
.../sourcefire/BasicSourcefireParserTest.java | 25 +-
.../parsers/websphere/GrokWebSphereParserTest.java | 85 +-
.../metron-parsing/metron-parsing-storm/pom.xml | 11 +
.../src/main/scripts/start_parser_topology.sh | 8 +-
.../integration/validation/StormParserDriver.java | 27 +-
.../integration/WriterBoltIntegrationTest.java | 5 +-
metron-platform/metron-pcap-backend/pom.xml | 40 +-
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/scripts/pcap_inspector.sh | 2 +-
.../src/main/scripts/pcap_query.sh | 2 +-
.../src/main/scripts/start_pcap_topology.sh | 6 +-
.../org/apache/metron/pcap/query/PcapCliTest.java | 8 +-
metron-platform/metron-pcap/pom.xml | 10 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 9 +-
.../apache/metron/pcap/utils/FileFilterUtil.java | 12 +-
.../metron-solr/metron-solr-common/pom.xml | 6 +
.../org/apache/metron/solr/dao/SolrSearchDao.java | 50 +-
.../org/apache/metron/solr/dao/SolrUtilities.java | 5 +-
.../metron/solr/schema/SchemaTranslator.java | 3 +-
.../apache/metron/solr/dao/SolrSearchDaoTest.java | 22 +-
.../apache/metron/solr/dao/SolrUtilitiesTest.java | 5 +-
.../SolrRetrieveLatestIntegrationTest.java | 22 +-
.../integration/SolrSearchIntegrationTest.java | 1 -
.../integration/SolrUpdateIntegrationTest.java | 54 +-
.../src/main/scripts/start_solr_topology.sh | 3 +-
.../metron-storm-kafka-override/pom.xml | 2 +
metron-platform/metron-storm-kafka/pom.xml | 1 +
.../metron/test/converters/BinaryConverters.java | 4 +-
.../apache/metron/test/filereaders/FileReader.java | 3 +-
.../test/spouts/GenericInternalTestSpout.java | 3 +-
.../org/apache/metron/test/utils/KafkaLoader.java | 5 +-
.../metron-writer/metron-writer-common/pom.xml | 45 +-
.../writer/hbase/SimpleHbaseEnrichmentWriter.java | 31 +-
.../metron-writer/metron-writer-storm/pom.xml | 6 +
.../org/apache/metron/writer/hdfs/HdfsWriter.java | 13 +-
.../apache/metron/writer/hdfs/SourceHandler.java | 11 +-
.../writer/bolt/BulkMessageWriterBoltTest.java | 3 +-
.../metron/zookeeper/SimpleEventListener.java | 4 +-
metron-stellar/stellar-common/README.md | 100 +-
metron-stellar/stellar-common/pom.xml | 10 +-
.../stellar-common/src/main/assembly/assembly.xml | 12 -
.../metron/stellar/common/utils/JSONUtils.java | 6 +-
.../metron/stellar/dsl/functions/RestConfig.java | 23 +-
.../stellar/dsl/functions/RestFunctions.java | 657 +-
.../src/main/scripts/deployed/stellar | 4 +-
.../stellar/common/shell/cli/StellarShellTest.java | 5 +-
.../stellar/dsl/functions/DateFunctionsTest.java | 29 +-
.../functions/RestFunctionsIntegrationTest.java | 490 +
.../stellar/dsl/functions/RestFunctionsTest.java | 458 +-
metron-stellar/stellar-zeppelin/pom.xml | 1 +
pom.xml | 7 +-
404 files changed, 24488 insertions(+), 23197 deletions(-)
diff --cc metron-contrib/metron-performance/pom.xml
index 4726608,40992ab..610e804
--- a/metron-contrib/metron-performance/pom.xml
+++ b/metron-contrib/metron-performance/pom.xml
@@@ -43,8 -43,14 +43,14 @@@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka-clients</artifactId>
<version>${global_kafka_version}</version>
<scope>provided</scope>
</dependency>
diff --cc metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 694c7fa,aec0afe..80f4b87
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@@ -18,22 -18,14 +18,22 @@@ limitations under the License
"""
- from resource_management.libraries.script import Script
- from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions import default, format
+ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.version import format_stack_version
+ from resource_management.libraries.script import Script
+def get_hostname(config):
+ if 'agentLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['agentLevelParams']['hostname']
+ else:
+ # Ambari 2.6.x
+ return config['hostname']
+
config = Script.get_config()
-hostname = config['hostname']
+hostname = get_hostname(config)
metron_user = config['configurations']['metron-env']['metron_user']
metron_home = config['configurations']['metron-env']['metron_home']
metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
diff --cc metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index 1509ae2,d363e91..d0fc444
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@@ -39,14 -27,10 +27,14 @@@ import org.apache.curator.framework.Cur
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.hadoop.hbase.HBaseConfiguration;
- import org.apache.hadoop.hbase.client.HTableInterface;
+ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
diff --cc metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
index e915d99,15df035..8edf5ab
--- a/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
+++ b/metron-platform/metron-elasticsearch/metron-elasticsearch-common/pom.xml
@@@ -62,11 -68,18 +68,23 @@@
</exclusions>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${global_guava_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${global_hbase_version}</version>
diff --cc metron-platform/metron-parsing/metron-parsers/pom.xml
index 5c323d8,cbf9c2f..0f4bc31
--- a/metron-platform/metron-parsing/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsing/metron-parsers/pom.xml
@@@ -46,9 -41,19 +46,15 @@@
<artifactId>metron-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- 3rd party dependencies -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${global_guava_version}</version>
- </dependency>
+
<dependency>
<groupId>io.thekraken</groupId>
<artifactId>grok</artifactId>