You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2017/08/30 15:05:04 UTC
[39/39] metron git commit: METRON-1136 Metron Extensions System and
Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720
METRON-1136 Metron Extensions System and Parser Extensions Feature Branch (ottobackwards) closes apache/metron#720
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ffcb91ed
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ffcb91ed
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ffcb91ed
Branch: refs/heads/feature/METRON-1136-extensions-parsers
Commit: ffcb91ed09c36d25008cfe57f06a9568d278f2f4
Parents: c8e84fa
Author: ottobackwards <ot...@gmail.com>
Authored: Wed Aug 30 11:03:32 2017 -0400
Committer: otto <ot...@apache.org>
Committed: Wed Aug 30 11:03:32 2017 -0400
----------------------------------------------------------------------
.gitignore | 1 +
.travis.yml | 1 +
README.md | 11 +-
bundles-lib/README.md | 171 +++
bundles-lib/pom.xml | 185 +++
.../metron/bundles/BundleClassLoaders.java | 353 ++++++
.../apache/metron/bundles/BundleCloseable.java | 86 ++
.../metron/bundles/BundleManifestEntry.java | 48 +
.../org/apache/metron/bundles/BundleMapper.java | 191 +++
.../org/apache/metron/bundles/BundleSystem.java | 202 +++
.../bundles/BundleThreadContextClassLoader.java | 211 ++++
.../apache/metron/bundles/ExtensionManager.java | 534 ++++++++
.../apache/metron/bundles/ExtensionMapping.java | 156 +++
.../metron/bundles/InstanceClassLoader.java | 161 +++
.../metron/bundles/NotInitializedException.java | 38 +
.../metron/bundles/VFSBundleClassLoader.java | 520 ++++++++
.../bundles/VFSBundleClassLoaderResource.java | 110 ++
.../behavior/RequiresInstanceClassLoading.java | 37 +
.../apache/metron/bundles/bundle/Bundle.java | 48 +
.../bundles/bundle/BundleCoordinates.java | 96 ++
.../metron/bundles/bundle/BundleDetails.java | 205 +++
.../metron/bundles/util/BundleProperties.java | 247 ++++
.../metron/bundles/util/BundleSelector.java | 48 +
.../apache/metron/bundles/util/BundleUtil.java | 145 +++
.../metron/bundles/util/DummyFileObject.java | 230 ++++
.../bundles/util/FileSystemManagerFactory.java | 87 ++
.../apache/metron/bundles/util/FileUtils.java | 35 +
.../bundles/util/ImmutableCollectionUtils.java | 65 +
.../apache/metron/bundles/util/StringUtils.java | 118 ++
.../org/apache/metron/bundles/AbstractFoo.java | 24 +
.../metron/bundles/BundleClassLoadersTest.java | 110 ++
.../apache/metron/bundles/BundleMapperTest.java | 180 +++
.../apache/metron/bundles/BundleSystemTest.java | 56 +
.../BundleThreadContextClassLoaderTest.java | 142 +++
.../apache/metron/bundles/BundleUtilTest.java | 125 ++
.../bundles/ExtensionClassInitializerTest.java | 33 +
.../metron/bundles/ExtensionManagerTest.java | 82 ++
.../BundleMapperIntegrationTest.java | 146 +++
.../util/ImmutableCollectionUtilsTest.java | 85 ++
.../metron/bundles/util/ResourceCopier.java | 60 +
.../apache/metron/bundles/util/TestUtil.java | 55 +
.../parsers/interfaces/MessageParser.java | 22 +
.../BundleMapper/conf/bundle.properties | 21 +
.../lib/metron-parser-bar-bundle-0.4.1.bundle | Bin 0 -> 21972 bytes
.../lib2/metron-parser-foo-bundle-0.4.1.bundle | Bin 0 -> 21983 bytes
.../org.apache.metron.bundles.AbstractFoo | 16 +
.../src/test/resources/bundle.properties | 22 +
.../bundle-with-versioning/META-INF/MANIFEST.MF | 15 +
.../META-INF/MANIFEST.MF | 12 +
.../META-INF/MANIFEST.MF | 7 +
bundles-maven-plugin/LICENSE | 202 +++
bundles-maven-plugin/NOTICE | 5 +
bundles-maven-plugin/README.md | 89 ++
bundles-maven-plugin/pom.xml | 328 +++++
.../maven/plugins/bundles/BundleMojo.java | 743 +++++++++++
.../bundles/BundleProvidedDependenciesMojo.java | 328 +++++
.../resources/META-INF/plexus/components.xml | 52 +
metron-analytics/metron-maas-service/README.md | 2 +-
.../METRON/CURRENT/configuration/metron-env.xml | 42 +
.../common-services/METRON/CURRENT/metainfo.xml | 39 +
.../CURRENT/package/scripts/metron_service.py | 43 +-
.../package/scripts/params/params_linux.py | 8 +
.../package/scripts/params/status_params.py | 9 +
.../CURRENT/package/scripts/parser_commands.py | 62 +-
.../package/templates/bundle.properties.j2 | 21 +
.../CURRENT/package/templates/global.json.j2 | 3 +-
.../METRON/CURRENT/package/templates/metron.j2 | 1 +
.../docker/rpm-docker/SPECS/metron.spec | 498 +++++++-
.../packaging/docker/rpm-docker/pom.xml | 79 ++
.../sensor-config-single-parser.e2e-spec.ts | 2 +-
.../sensor-field-schema.component.spec.ts | 4 +-
...sensor-parser-config-readonly.component.html | 2 +-
...sor-parser-config-readonly.component.spec.ts | 4 +-
.../sensor-parser-config.component.spec.ts | 51 +-
.../sensor-parser-config.component.ts | 12 +-
.../sensor-parser-list.component.spec.ts | 16 +-
.../src/app/service/grok-validation.service.ts | 9 +-
.../sensor-parser-config.service.spec.ts | 2 +-
metron-interface/metron-rest/README.md | 33 +
metron-interface/metron-rest/pom.xml | 34 +
.../src/main/config/rest_application.yml | 4 +
.../apache/metron/rest/MetronRestConstants.java | 4 +-
.../metron/rest/config/BundleSystemConfig.java | 64 +
.../apache/metron/rest/config/GrokConfig.java | 8 +-
.../metron/rest/controller/GrokController.java | 13 +-
.../controller/ParserExtensionController.java | 114 ++
.../metron/rest/service/ExtensionService.java | 58 +
.../apache/metron/rest/service/GrokService.java | 4 +-
.../apache/metron/rest/service/HdfsService.java | 4 +
.../rest/service/SensorParserConfigService.java | 4 +-
.../rest/service/impl/ExtensionServiceImpl.java | 562 +++++++++
.../rest/service/impl/GrokServiceImpl.java | 72 +-
.../rest/service/impl/HdfsServiceImpl.java | 15 +
.../impl/SensorParserConfigServiceImpl.java | 110 +-
.../src/main/resources/application-test.yml | 4 +
.../src/main/resources/application-vagrant.yml | 3 +-
.../apache/metron/rest/config/TestConfig.java | 27 +-
.../GrokControllerIntegrationTest.java | 4 +-
.../KafkaControllerIntegrationTest.java | 4 +-
...arserExtensionControllerIntegrationTest.java | 165 +++
...richmentConfigControllerIntegrationTest.java | 3 +-
...IndexingConfigControllerIntegrationTest.java | 3 +-
...orParserConfigControllerIntegrationTest.java | 74 +-
.../service/impl/GrokServiceImplNoMockTest.java | 122 ++
.../rest/service/impl/GrokServiceImplTest.java | 31 +-
.../impl/SensorParserConfigServiceImplTest.java | 95 +-
...on-parser-test-assembly-0.4.0-archive.tar.gz | Bin 0 -> 24772 bytes
.../test/resources/zookeeper/bundle.properties | 21 +
.../src/test/resources/zookeeper/global.json | 25 +
metron-maven-archetypes/README.md | 57 +
.../metron-parser-extension-archetype/README.md | 83 ++
.../metron-parser-extension-archetype/pom.xml | 46 +
.../META-INF/maven/archetype-metadata.xml | 81 ++
.../pom.xml | 49 +
.../src/main/assembly/assembly.xml | 56 +
.../metron-parser-__parserName__-bundle/pom.xml | 41 +
.../metron-parser-__parserName__/README.md | 5 +
.../metron-parser-__parserName__/pom.xml | 83 ++
.../elasticsearch/__parserName___index.template | 124 ++
.../zookeeper/enrichments/__parserName__.json | 20 +
.../zookeeper/indexing/__parserName__.json | 18 +
.../zookeeper/parsers/__parserName__.json | 7 +
.../__parserClassName__Parser.java | 79 ++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../__parserClassName__ParserTest.java | 51 +
.../__parserClassName__IntegrationTest.java | 52 +
.../config/zookeeper/bundle.properties | 23 +
.../test/resources/config/zookeeper/global.json | 27 +
.../src/test/resources/data/parsed/test.parsed | 2 +
.../src/test/resources/data/raw/test.raw | 2 +
.../src/test/resources/log4j.properties | 37 +
.../main/resources/archetype-resources/pom.xml | 36 +
.../projects/basic/archetype.properties | 8 +
.../src/test/resources/projects/basic/goal.txt | 0
metron-maven-archetypes/pom.xml | 34 +
metron-maven-archetypes/project.png | Bin 0 -> 127469 bytes
.../org/apache/metron/common/Constants.java | 1 +
.../metron/common/bolt/ConfiguredBolt.java | 29 +-
.../common/configuration/ConfigurationType.java | 15 +-
.../configuration/ConfigurationsUtils.java | 89 +-
.../extensions/ParserExtensionConfig.java | 192 +++
.../metron/common/utils/ResourceLoader.java | 174 +++
.../common/cli/ConfigurationsUtilsTest.java | 18 +-
.../SensorEnrichmentConfigTest.java | 7 +-
.../extensions/ParserExtensionConfigTest.java | 39 +
.../metron/common/utils/ResourceLoaderTest.java | 90 ++
.../main/config/zookeeper/enrichments/asa.json | 8 -
.../main/config/zookeeper/enrichments/bro.json | 18 -
.../config/zookeeper/enrichments/snort.json | 29 -
.../config/zookeeper/enrichments/websphere.json | 18 -
.../main/config/zookeeper/enrichments/yaf.json | 20 -
.../components/ConfigUploadComponent.java | 11 +
metron-platform/metron-extensions/README.md | 46 +
.../metron-extensions/extension_deployment.md | 20 +
.../metron-extensions/extension_packaging.md | 16 +
.../metron-extensions/extension_terms.md | 25 +
.../metron-parser-extensions/README.md | 9 +
.../adding_system_parsers.md | 345 +++++
.../metron-parser-asa-assembly/pom.xml | 50 +
.../src/main/assembly/assembly.xml | 65 +
.../metron-parser-asa-bundle/pom.xml | 42 +
.../metron-parser-asa/README.md | 3 +
.../metron-parser-asa/pom.xml | 86 ++
.../main/config/zookeeper/enrichments/asa.json | 8 +
.../src/main/config/zookeeper/indexing/asa.json | 18 +
.../src/main/config/zookeeper/parsers/asa.json | 7 +
.../metron/parsers/asa/BasicAsaParser.java | 240 ++++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/asa | 179 +++
.../src/main/resources/patterns/common | 96 ++
.../metron/parsers/asa/BasicAsaParserTest.java | 187 +++
.../asa/integration/AsaIntegrationTest.java | 49 +
.../src/test/resources/data/parsed/test.parsed | 128 ++
.../src/test/resources/data/raw/test.raw | 128 ++
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-asa-extension/pom.xml | 36 +
.../metron-parser-bro-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-bro-bundle/pom.xml | 41 +
.../metron-parser-bro/README.md | 3 +
.../metron-parser-bro/pom.xml | 83 ++
.../config/elasticsearch/bro_index.template | 972 ++++++++++++++
.../main/config/zookeeper/enrichments/bro.json | 17 +
.../src/main/config/zookeeper/indexing/bro.json | 18 +
.../src/main/config/zookeeper/parsers/bro.json | 6 +
.../metron/parsers/bro/BasicBroParser.java | 180 +++
.../apache/metron/parsers/bro/JSONCleaner.java | 91 ++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../metron/parsers/bro/BasicBroParserTest.java | 1189 ++++++++++++++++++
.../parsers/integration/BroIntegrationTest.java | 48 +
.../test/resources/TestSchemas/BroSchema.json | 28 +
.../test/resources/config/BroParserTest.config | 20 +
.../src/test/resources/data/parsed/test.parsed | 27 +
.../src/test/resources/data/raw/test.raw | 27 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-bro-extension/pom.xml | 36 +
.../metron-parser-bundle-tests/pom.xml | 51 +
.../parsers/ASABundleHDFSIntegrationTest.java | 265 ++++
...etron-parser-test-bundle-1.0-SNAPSHOT.bundle | Bin 0 -> 22129 bytes
.../metron-parser-asa-bundle-0.4.0.bundle | Bin 0 -> 45283 bytes
.../test/resources/zookeeper/bundle.properties | 23 +
.../resources/zookeeper/enrichments/test.json | 75 ++
.../src/test/resources/zookeeper/global.json | 27 +
.../test/resources/zookeeper/indexing/test.json | 18 +
.../metron-parser-cef-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-cef-bundle/pom.xml | 41 +
.../metron-parser-cef/README.md | 3 +
.../metron-parser-cef/pom.xml | 89 ++
.../main/config/zookeeper/enrichments/cef.json | 19 +
.../src/main/config/zookeeper/indexing/cef.json | 18 +
.../src/main/config/zookeeper/parsers/cef.json | 7 +
.../apache/metron/parsers/cef/CEFParser.java | 274 ++++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../metron/parsers/cef/CEFParserTest.java | 291 +++++
.../src/test/resources/log4j.properties | 34 +
.../org/apache/metron/parsers/cef/adallom.cef | 1 +
.../apache/metron/parsers/cef/adallom.schema | 37 +
.../org/apache/metron/parsers/cef/cyberark.cef | 1 +
.../org/apache/metron/parsers/cef/cyberark.json | 21 +
.../apache/metron/parsers/cef/cyberark.schema | 38 +
.../org/apache/metron/parsers/cef/palo.cef | 1 +
.../org/apache/metron/parsers/cef/palo.schema | 38 +
.../org/apache/metron/parsers/cef/waf.cef | 1 +
.../org/apache/metron/parsers/cef/waf.schema | 67 +
.../metron-parser-cef-extension/pom.xml | 36 +
.../metron-parser-extensions-testing/pom.xml | 88 ++
.../metron-parser-fireeye-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 53 +
.../metron-parser-fireeye-bundle/pom.xml | 41 +
.../metron-parser-fireeye/README.md | 3 +
.../metron-parser-fireeye/pom.xml | 89 ++
.../config/zookeeper/enrichments/fireeye.json | 20 +
.../main/config/zookeeper/indexing/fireeye.json | 18 +
.../main/config/zookeeper/parsers/fireeye.json | 7 +
.../parsers/fireeye/BasicFireEyeParser.java | 222 ++++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../src/main/resources/patterns/fireeye | 9 +
.../parsers/fireeye/BasicFireEyeParserTest.java | 76 ++
.../config/BasicFireEyeParserTest.config | 20 +
.../src/test/resources/data/parsed/test.parsed | 2 +
.../src/test/resources/data/raw/test.raw | 2 +
.../src/test/resources/log4j.properties | 34 +
.../resources/logData/FireEyeParserTest.txt | 8 +
.../metron-parser-fireeye-extension/pom.xml | 36 +
.../metron-parser-ise-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-ise-bundle/pom.xml | 41 +
.../metron-parser-ise/README.md | 3 +
.../metron-parser-ise/pom.xml | 89 ++
.../main/config/zookeeper/enrichments/ise.json | 20 +
.../src/main/config/zookeeper/indexing/ise.json | 18 +
.../src/main/config/zookeeper/parsers/ise.json | 7 +
.../metron/parsers/ise/BasicIseParser.java | 101 ++
.../apache/metron/parsers/ise/ISEParser.java | 665 ++++++++++
.../org/apache/metron/parsers/ise/ISEParser.jj | 12 +
.../metron/parsers/ise/ISEParserConstants.java | 69 +
.../parsers/ise/ISEParserTokenManager.java | 676 ++++++++++
.../metron/parsers/ise/JavaCharStream.java | 633 ++++++++++
.../metron/parsers/ise/ParseException.java | 204 +++
.../org/apache/metron/parsers/ise/Token.java | 148 +++
.../metron/parsers/ise/TokenMgrError.java | 164 +++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../metron/parsers/ise/BasicIseParserTest.java | 56 +
.../test/resources/TestSchemas/IseSchema.json | 21 +
.../resources/config/BasicIseParserTest.config | 20 +
.../src/test/resources/log4j.properties | 34 +
.../test/resources/logData/IseParserTest.txt | 308 +++++
.../metron-parser-ise-extension/pom.xml | 36 +
.../metron-parser-lancope-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-lancope-bundle/pom.xml | 41 +
.../metron-parser-lancope/README.md | 3 +
.../metron-parser-lancope/pom.xml | 89 ++
.../config/zookeeper/enrichments/lancope.json | 20 +
.../main/config/zookeeper/indexing/lancope.json | 18 +
.../main/config/zookeeper/parsers/lancope.json | 7 +
.../parsers/lancope/BasicLancopeParser.java | 95 ++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../parsers/lancope/BasicLancopeParserTest.java | 57 +
.../resources/TestSchemas/LancopeSchema.json | 28 +
.../config/BasicLancopeParserTest.config | 20 +
.../src/test/resources/log4j.properties | 34 +
.../resources/logData/LancopeParserTest.txt | 1 +
.../metron-parser-lancope-extension/pom.xml | 36 +
.../metron-parser-logstash-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-logstash-bundle/pom.xml | 41 +
.../metron-parser-logstash/README.md | 3 +
.../metron-parser-logstash/pom.xml | 83 ++
.../config/zookeeper/enrichments/logstash.json | 20 +
.../config/zookeeper/indexing/logstash.json | 18 +
.../main/config/zookeeper/parsers/logstash.json | 7 +
.../parsers/logstash/BasicLogstashParser.java | 95 ++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../logstash/BasicLogstashParserTest.java | 46 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-logstash-extension/pom.xml | 36 +
.../metron-parser-paloalto-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 42 +
.../metron-parser-paloalto-bundle/pom.xml | 41 +
.../metron-parser-paloalto/README.md | 3 +
.../metron-parser-paloalto/pom.xml | 89 ++
.../config/zookeeper/enrichments/paloalto.json | 20 +
.../config/zookeeper/indexing/paloalto.json | 18 +
.../main/config/zookeeper/parsers/paloalto.json | 7 +
.../paloalto/BasicPaloAltoFirewallParser.java | 215 ++++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../BasicPaloAltoFirewallParserTest.java | 57 +
.../BasicPaloAltoFirewallParserTest.config | 20 +
.../src/test/resources/log4j.properties | 34 +
.../logData/PaloAltoFirewallParserTest.txt | 2 +
.../metron-parser-paloalto-extension/pom.xml | 36 +
.../metron-parser-snort-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 53 +
.../metron-parser-snort-bundle/pom.xml | 41 +
.../metron-parser-snort/README.md | 3 +
.../metron-parser-snort/pom.xml | 83 ++
.../config/elasticsearch/snort_index.template | 183 +++
.../config/zookeeper/enrichments/snort.json | 29 +
.../main/config/zookeeper/indexing/snort.json | 18 +
.../main/config/zookeeper/parsers/snort.json | 6 +
.../metron/parsers/snort/BasicSnortParser.java | 214 ++++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../integration/SnortIntegrationTest.java | 49 +
.../metron/parsers/snort/SnortParserTest.java | 149 +++
.../src/test/resources/data/parsed/test.parsed | 3 +
.../src/test/resources/data/raw/test.raw | 3 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-snort-extension/pom.xml | 36 +
.../metron-parser-sourcefire-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 53 +
.../metron-parser-sourcefire-bundle/pom.xml | 41 +
.../metron-parser-sourcefire/README.md | 3 +
.../metron-parser-sourcefire/pom.xml | 89 ++
.../zookeeper/enrichments/sourcefire.json | 20 +
.../config/zookeeper/indexing/sourcefire.json | 18 +
.../config/zookeeper/parsers/sourcefire.json | 7 +
.../sourcefire/BasicSourcefireParser.java | 129 ++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../src/main/resources/patterns/sourcefire | 30 +
.../sourcefire/BasicSourcefireParserTest.java | 58 +
.../resources/TestSchemas/SourcefireSchema.json | 34 +
.../config/BasicSourcefireParserTest.config | 20 +
.../src/test/resources/global.json | 27 +
.../src/test/resources/log4j.properties | 34 +
.../resources/logData/SourcefireParserTest.txt | 3 +
.../metron-parser-sourcefire-extension/pom.xml | 36 +
.../metron-parser-squid-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 55 +
.../metron-parser-squid-bundle/pom.xml | 41 +
.../metron-parser-squid/README.md | 3 +
.../metron-parser-squid/pom.xml | 83 ++
.../config/zookeeper/enrichments/squid.json | 20 +
.../main/config/zookeeper/indexing/squid.json | 18 +
.../main/config/zookeeper/parsers/squid.json | 19 +
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../src/main/resources/patterns/squid | 2 +
.../integration/SquidIntegrationTest.java | 48 +
.../metron/parsers/squid/SquidParserTest.java | 102 ++
.../src/test/resources/data/parsed/test.parsed | 2 +
.../src/test/resources/data/raw/test.raw | 2 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-squid-extension/pom.xml | 38 +
.../metron-parser-websphere-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 53 +
.../metron-parser-websphere-bundle/pom.xml | 41 +
.../metron-parser-websphere/README.md | 3 +
.../metron-parser-websphere/pom.xml | 83 ++
.../config/zookeeper/enrichments/websphere.json | 18 +
.../config/zookeeper/indexing/websphere.json | 18 +
.../config/zookeeper/parsers/websphere.json | 11 +
.../parsers/websphere/GrokWebSphereParser.java | 143 +++
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../src/main/resources/patterns/websphere | 37 +
.../integration/WebSphereIntegrationTest.java | 52 +
.../websphere/GrokWebSphereParserTest.java | 269 ++++
.../src/test/resources/data/parsed/test.parsed | 5 +
.../src/test/resources/data/raw/test.raw | 5 +
.../src/test/resources/global.json | 27 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-websphere-extension/pom.xml | 36 +
.../metron-parser-yaf-assembly/pom.xml | 49 +
.../src/main/assembly/assembly.xml | 55 +
.../metron-parser-yaf-bundle/pom.xml | 41 +
.../metron-parser-yaf/README.md | 3 +
.../metron-parser-yaf/pom.xml | 83 ++
.../config/elasticsearch/yaf_index.template | 205 +++
.../main/config/zookeeper/enrichments/yaf.json | 20 +
.../src/main/config/zookeeper/indexing/yaf.json | 18 +
.../src/main/config/zookeeper/parsers/yaf.json | 18 +
.../src/main/resources/META-INF/LICENSE | 602 +++++++++
.../src/main/resources/META-INF/NOTICE | 40 +
.../src/main/resources/patterns/common | 96 ++
.../src/main/resources/patterns/yaf | 2 +
.../parsers/integration/YafIntegrationTest.java | 48 +
.../metron/parsers/yaf/YafParserTest.java | 101 ++
.../test/resources/data/indexed/test.indexed | 10 +
.../src/test/resources/data/parsed/test.parsed | 10 +
.../src/test/resources/data/raw/test.raw | 10 +
.../src/test/resources/log4j.properties | 34 +
.../metron-parser-yaf-extension/pom.xml | 38 +
.../parser_extension_deployment.md | 240 ++++
.../parser_extension_packaging.md | 84 ++
.../metron-parser-extensions/pom.xml | 97 ++
metron-platform/metron-extensions/pom.xml | 60 +
.../src/main/config/zookeeper/indexing/asa.json | 18 -
.../src/main/config/zookeeper/indexing/bro.json | 17 -
.../main/config/zookeeper/indexing/snort.json | 17 -
.../config/zookeeper/indexing/websphere.json | 18 -
.../src/main/config/zookeeper/indexing/yaf.json | 17 -
.../src/main/config/zookeeper/bundle.properties | 20 +
.../extensions/parsers/metron-test-parsers.json | 7 +
.../src/main/config/zookeeper/global.json | 3 +-
.../src/main/sample/data/asa/parsed/asa_parsed | 128 --
.../src/main/sample/data/asa/raw/asa_raw | 128 --
.../sample/data/bro/parsed/BroExampleParsed | 27 -
.../main/sample/data/bro/raw/BroExampleOutput | 27 -
.../main/sample/data/snort/parsed/SnortParsed | 3 -
.../src/main/sample/data/snort/raw/SnortOutput | 3 -
.../sample/data/squid/parsed/SquidExampleParsed | 2 -
.../sample/data/squid/raw/SquidExampleOutput | 2 -
.../data/websphere/parsed/WebsphereParsed | 5 -
.../data/websphere/raw/WebsphereOutput.txt | 5 -
.../src/main/sample/data/yaf/indexed/YafIndexed | 10 -
.../sample/data/yaf/parsed/YafExampleParsed | 10 -
.../main/sample/data/yaf/raw/YafExampleOutput | 10 -
metron-platform/metron-management/README.md | 8 +-
.../management/ConfigurationFunctions.java | 52 +-
.../management/ConfigurationFunctionsTest.java | 53 +-
.../management/ParserConfigFunctionsTest.java | 6 +-
metron-platform/metron-parsers/README.md | 6 +-
metron-platform/metron-parsers/pom.xml | 10 +
.../src/main/config/zookeeper/parsers/asa.json | 7 -
.../src/main/config/zookeeper/parsers/bro.json | 5 -
.../main/config/zookeeper/parsers/snort.json | 5 -
.../main/config/zookeeper/parsers/squid.json | 19 -
.../config/zookeeper/parsers/websphere.json | 11 -
.../src/main/config/zookeeper/parsers/yaf.json | 18 -
.../org/apache/metron/parsers/BasicParser.java | 1 +
.../org/apache/metron/parsers/GrokParser.java | 211 ----
.../metron/parsers/asa/BasicAsaParser.java | 240 ----
.../apache/metron/parsers/bolt/ParserBolt.java | 25 +-
.../metron/parsers/bolt/ParserLoader.java | 95 ++
.../metron/parsers/bro/BasicBroParser.java | 180 ---
.../apache/metron/parsers/bro/JSONCleaner.java | 91 --
.../apache/metron/parsers/cef/CEFParser.java | 274 ----
.../parsers/fireeye/BasicFireEyeParser.java | 222 ----
.../apache/metron/parsers/grok/GrokBuilder.java | 231 ++++
.../apache/metron/parsers/grok/GrokParser.java | 175 +++
.../parsers/interfaces/MessageParser.java | 3 +
.../metron/parsers/ise/BasicIseParser.java | 101 --
.../apache/metron/parsers/ise/ISEParser.java | 660 ----------
.../org/apache/metron/parsers/ise/ISEParser.jj | 12 -
.../metron/parsers/ise/ISEParserConstants.java | 69 -
.../parsers/ise/ISEParserTokenManager.java | 676 ----------
.../metron/parsers/ise/JavaCharStream.java | 633 ----------
.../metron/parsers/ise/ParseException.java | 204 ---
.../org/apache/metron/parsers/ise/Token.java | 148 ---
.../metron/parsers/ise/TokenMgrError.java | 164 ---
.../parsers/lancope/BasicLancopeParser.java | 95 --
.../parsers/logstash/BasicLogstashParser.java | 95 --
.../paloalto/BasicPaloAltoFirewallParser.java | 215 ----
.../metron/parsers/snort/BasicSnortParser.java | 214 ----
.../sourcefire/BasicSourcefireParser.java | 129 --
.../parsers/topology/ParserTopologyBuilder.java | 10 +-
.../parsers/websphere/GrokWebSphereParser.java | 143 ---
.../src/main/resources/patterns/asa | 179 ---
.../src/main/resources/patterns/fireeye | 9 -
.../src/main/resources/patterns/sourcefire | 30 -
.../src/main/resources/patterns/squid | 2 -
.../src/main/resources/patterns/websphere | 37 -
.../src/main/resources/patterns/yaf | 2 -
.../apache/metron/parsers/GrokParserTest.java | 95 --
.../metron/parsers/SampleGrokParserTest.java | 97 --
.../apache/metron/parsers/SnortParserTest.java | 150 ---
.../apache/metron/parsers/SquidParserTest.java | 101 --
.../apache/metron/parsers/YafParserTest.java | 100 --
.../metron/parsers/asa/BasicAsaParserTest.java | 188 ---
.../metron/parsers/bro/BasicBroParserTest.java | 1189 ------------------
.../metron/parsers/cef/CEFParserTest.java | 291 -----
.../parsers/fireeye/BasicFireEyeParserTest.java | 76 --
.../metron/parsers/grok/GrokBuilderTest.java | 83 ++
.../metron/parsers/grok/GrokParserTest.java | 106 ++
.../parsers/grok/SampleGrokParserTest.java | 97 ++
.../parsers/integration/AsaIntegrationTest.java | 37 -
.../parsers/integration/BroIntegrationTest.java | 37 -
.../integration/JSONMapIntegrationTest.java | 4 +-
.../integration/ParserIntegrationTest.java | 62 +-
.../integration/SnortIntegrationTest.java | 37 -
.../integration/SquidIntegrationTest.java | 37 -
.../integration/WebSphereIntegrationTest.java | 40 -
.../parsers/integration/YafIntegrationTest.java | 37 -
.../validation/PathedSampleDataValidation.java | 61 +
.../validation/SampleDataValidation.java | 22 +-
.../metron/parsers/ise/BasicIseParserTest.java | 56 -
.../parsers/lancope/BasicLancopeParserTest.java | 57 -
.../BasicPaloAltoFirewallParserTest.java | 57 -
.../sourcefire/BasicSourcefireParserTest.java | 58 -
.../websphere/GrokWebSphereParserTest.java | 260 ----
...pleHbaseEnrichmentWriterIntegrationTest.java | 32 +-
.../integration/WriterBoltIntegrationTest.java | 47 +-
.../test/resources/TestSchemas/BroSchema.json | 28 -
.../test/resources/TestSchemas/IseSchema.json | 21 -
.../resources/TestSchemas/LancopeSchema.json | 28 -
.../test/resources/TestSchemas/PcapSchema.json | 22 -
.../resources/TestSchemas/SourcefireSchema.json | 34 -
.../config/BasicFireEyeParserTest.config | 20 -
.../resources/config/BasicIseParserTest.config | 20 -
.../config/BasicLancopeParserTest.config | 20 -
.../BasicPaloAltoFirewallParserTest.config | 20 -
.../config/BasicSourcefireParserTest.config | 20 -
.../test/resources/config/BroParserTest.config | 20 -
.../resources/logData/FireEyeParserTest.txt | 8 -
.../test/resources/logData/IseParserTest.txt | 308 -----
.../resources/logData/LancopeParserTest.txt | 1 -
.../logData/PaloAltoFirewallParserTest.txt | 2 -
.../resources/logData/SourcefireParserTest.txt | 3 -
.../org/apache/metron/parsers/cef/adallom.cef | 1 -
.../apache/metron/parsers/cef/adallom.schema | 37 -
.../org/apache/metron/parsers/cef/cyberark.cef | 1 -
.../org/apache/metron/parsers/cef/cyberark.json | 21 -
.../apache/metron/parsers/cef/cyberark.schema | 38 -
.../org/apache/metron/parsers/cef/palo.cef | 1 -
.../org/apache/metron/parsers/cef/palo.schema | 38 -
.../org/apache/metron/parsers/cef/waf.cef | 1 -
.../org/apache/metron/parsers/cef/waf.schema | 67 -
.../src/test/resources/otherPatterns/common | 96 ++
.../src/test/resources/patterns/test | 2 +
.../java/org/apache/metron/TestConstants.java | 4 +
.../metron/test/utils/ResourceCopier.java | 96 ++
.../metron/test/utils/SampleDataUtils.java | 17 +-
metron-platform/pom.xml | 1 +
pom.xml | 17 +-
.../metron-maas-service/index.html | 2 +-
.../metron-management/index.html | 6 +-
.../metron-platform/metron-parsers/index.html | 6 +-
555 files changed, 38637 insertions(+), 9757 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d505c9c..2d048d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
target/
*/target/
*dependency-reduced-pom.xml
+*.versionsBackup
.idea
*.iml
*.iws
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 917b184..d85a7ca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,6 +34,7 @@ before_install:
- time build_utils/verify_licenses.sh
install:
+ - cd bundles-maven-plugin && mvn -q install && cd ..
- time mvn -q -T 2C -DskipTests clean install
script:
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 8599f0f..308c82a 100644
--- a/README.md
+++ b/README.md
@@ -70,6 +70,12 @@ To start exploring the capabilities of Apache Metron [follow these instructions
# Building Metron
+Build the maven plugin required for Metron ( this should only have to be one once )
+
+```
+$cd bundles-maven-plugin && mvn -q install && cd ..
+```
+
Build the full project and run tests:
```
$ mvn clean install
@@ -109,10 +115,13 @@ component and Apache Kafka as the unified data bus.
Some high level links to the relevant subparts of the architecture, for
more information:
-* [Parsers](metron-platform/metron-parsers) : Parsing data from kafka into the Metron data model and passing it downstream to Enrichment.
+* [Parsers](metron-platform/metron-parsers) : Base classes for parsing data from kafka into the Metron data model and passing it downstream to Enrichment.
* [Enrichment](metron-platform/metron-enrichment) : Enriching data post-parsing and providing the ability to tag a message as an alert and assign a risk triage level via a custom rule language.
* [Indexing](metron-platform/metron-indexing) : Indexing the data post-enrichment into HDFS, Elasticsearch or Solr.
+Metron Extensions
+* [Metron Parser Extensions](metron-platform/metron-extensions/metron-parser-extensions) : Parser Extensions parsing data from supported sources
+
Some useful utilities that cross all of these parts of the architecture:
* [Stellar](metron-platform/metron-common) : A custom data transformation language that is used throughout metron from simple field transformation to expressing triage rules.
* [Model as a Service](metron-analytics/metron-maas-service) : A Yarn application which can deploy machine learning and statistical models onto the cluster along with the associated Stellar functions to be able to call out to them in a scalable manner.
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/README.md
----------------------------------------------------------------------
diff --git a/bundles-lib/README.md b/bundles-lib/README.md
new file mode 100644
index 0000000..d2b41a4
--- /dev/null
+++ b/bundles-lib/README.md
@@ -0,0 +1,171 @@
+# Apache Metron Bundles
+
+Apache Metron Bundles and this documentation are a derivitave of the [Apache Nifi](http://www.nifi.apache.org) [NARs](http://nifi.apache.org/developer-guide.html).
+
+When software from many different organizations is all hosted within
+the same environment, Java ClassLoaders quickly
+become a concern. If multiple components have a dependency on the same
+library but each depends on a different
+version, many problems arise, typically resulting in unexpected
+behavior or `NoClassDefFoundError` errors occurring.
+In order to prevent these issues from becoming problematic, Apache NiFi
+introduces the notion of a NiFi Archive, or NAR. The Apache Metron project has adapted and extended the NAR system as
+Apache Metron Bundles.
+
+A BUNDLE allows several components and their dependencies to be packaged
+together into a single package.
+The BUNDLE package is then provided ClassLoader isolation from other BUNDLES
+packages. Developers should always deploy their Apache Metron Extensions as BUNDLE packages.
+
+To achieve this, a developer creates a new Maven Artifact, which we
+refer to as the BUNDLE artifact. The packaging is
+set to `bundle`. The `dependencies` section of the POM is then created so
+that the BUNDLE has a dependency on all Extension Components that are to be included within the BUNDLE.
+
+In order to use a packaging of `bundle`, we must use the `bundles-maven-plugin` module.
+This is included by adding the following snippet to the bundle's pom.xml:>
+> 0.4.1 below should be the current metron version
+```xml
+<build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>bundles-maven-plugin</artifactId>
+ <version>0.4.1</version>
+ <extensions>true</extensions>
+ </plugin>
+ </plugins>
+</build>
+```
+
+The bundles-maven-plugin is included in the projects created by Apache Metron Extension maven archetypes.
+
+
+The BUNDLE is able to have one dependency that is of type `bundle`. If more
+than one dependency is specified that is of type
+`bundle`, then the bundles-maven-plugin will error. If BUNDLE A adds a
+dependency on BUNDLE B, this will *not* result in
+BUNDLE A packaging all of the components of BUNDLE B. Rather, this will add
+a `Bundle-Dependency-Id` element to the `MANIFEST.MF`
+file of BUNDLE A. This will result in setting the ClassLoader of BUNDLE B as
+the Parent ClassLoader of BUNDLE A. In this case,
+we refer to BUNDLE B as the _Parent_ of BUNDLE A.
+
+## Per-Instance ClassLoading
+
+The bundles-lib provides the `@RequiresInstanceClassLoading` annotation to further expand and isolate the libraries
+available on a component’s classpath. You can annotate a extension class with `@RequiresInstanceClassLoading`
+to indicate that the instance ClassLoader for the component requires a copy of all the resources in the
+component's BUNDLE ClassLoader. When `@RequiresInstanceClassLoading` is not present, the
+instance ClassLoader simply has it's parent ClassLoader set to the BUNDLE ClassLoader, rather than
+copying resources.
+
+Because @RequiresInstanceClassLoading copies resources from the BUNDLE ClassLoader for each instance of the
+extension, use this capability judiciously in an environment where many extensions may be present. If ten instances of one extension are created, all classes
+from the component's BUNDLE ClassLoader are loaded into memory ten times. This could eventually increase the
+memory footprint significantly when enough instances of the component are created.
+
+
+## Apache VFS
+
+The bundles-lib utilizes the Apache VFS library for loading bundles. Bundles, which are zip files of a known structure, containing
+jars of dependencies may be loaded by VFS as File Systems *themselves*, and in tern each jar in the bundle can be loaded by VFS as a File System.
+
+The VFSBundleClassloader therefore loads the bundle as a Jar File System, and also each dependency jar as a filesystem. This allows
+for the complete loading of the bundle without having to unpack the bundle into a working directory.
+
+This is significantly different from the original Nifi implementation.
+
+## BundleSystem
+
+The BundleSystem class provides a useful and simple interface for using Bundles and instantiated class instances.
+While the raw classes may be used, in many cases the BundleSystem will be sufficient and reduce complexity.
+
+Without BundleSystem, the minimum required to instantiate a class (simplified):
+
+```java
+public static Optional<MessageParser<JSONObject>> loadParser(Map stormConfig, CuratorFramework client, SensorParserConfig parserConfig){
+ MessageParser<JSONObject> parser = null;
+ try {
+ // fetch the BundleProperties from zookeeper
+ Optional<BundleProperties> bundleProperties = getBundleProperties(client);
+
+ // create the FileSystemManager
+ FileSystemManager fileSystemManager = FileSystemManagerFactory.createFileSystemManager(new String[] {props.getArchiveExtension()});
+
+ // ADD in the classes we are going to support for plugins
+ ArrayList<Class> classes = new ArrayList<>();
+ for( Map.Entry<String,String> entry : props.getBundleExtensionTypes().entrySet()){
+ classes.add(Class.forName(entry.getValue()));
+ }
+
+ // create the SystemBundle ( the bundle for the types that may be in the system classloader already )
+ Bundle systemBundle = ExtensionManager.createSystemBundle(fileSystemManager, props);
+
+ // get the correct bundle library directories from properties
+ List<URI> libDirs = props.getBundleLibraryDirectories();
+ List<FileObject> libFileObjects = new ArrayList<>();
+ for(URI libUri : libDirs){
+ FileObject fileObject = fileSystemManager.resolveFile(libUri);
+ if(fileObject.exists()){
+ libFileObjects.add(fileObject);
+ }
+ }
+
+ // Initialize everything
+ BundleClassLoaders.getInstance().init(fileSystemManager, libFileObjects, props);
+ ExtensionManager.getInstance().init(classes, systemBundle, BundleClassLoaders.getInstance().getBundles());
+
+ // everything is ready, create our instance
+ parser = BundleThreadContextClassLoader.createInstance(parserConfig.getParserClassName(),MessageParser.class,props);
+
+ }catch(Exception e){
+ LOG.error("Failed to load parser " + parserConfig.getParserClassName(),e);
+ return Optional.empty();
+ }
+ return Optional.of(parser);
+ }
+
+```
+
+This is a lot to do, and a lot of unnecessary boilerplate code
+
+Using the BundleSystem class however:
+
+```java
+public static Optional<MessageParser<JSONObject>> loadParser(Map stormConfig,
+ CuratorFramework client, SensorParserConfig parserConfig) {
+ MessageParser<JSONObject> parser = null;
+ try {
+ // fetch the BundleProperties from zookeeper
+ Optional<BundleProperties> bundleProperties = getBundleProperties(client);
+ BundleProperties props = bundleProperties.get();
+
+ // create the BundleSystem
+ // we only need to pass in the properties
+ BundleSystem bundleSystem = new BundleSystem.Builder().withBundleProperties(props).build();
+
+ // create our instance
+ parser = bundleSystem
+ .createInstance(parserConfig.getParserClassName(), MessageParser.class);
+ } else {
+ LOG.error("BundleProperties are missing!");
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to load parser " + parserConfig.getParserClassName(), e);
+ return Optional.empty();
+ }
+ return Optional.of(parser);
+}
+
+```
+
+As we can see, this is much easier.
+
+With the BundleSystem, you may the defaults by calling the builder with :
+
+- withSystemBundle
+- withFileSystemManager
+- withExtensionClasses
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/pom.xml
----------------------------------------------------------------------
diff --git a/bundles-lib/pom.xml b/bundles-lib/pom.xml
new file mode 100644
index 0000000..0aecde9
--- /dev/null
+++ b/bundles-lib/pom.xml
@@ -0,0 +1,185 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>Metron</artifactId>
+ <version>0.4.1</version>
+ </parent>
+
+ <artifactId>bundles-lib</artifactId>
+ <packaging>jar</packaging>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <version>${global_accumulo_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.atteo.classindex</groupId>
+ <artifactId>classindex</artifactId>
+ <version>${global_classindex_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- from accumulo -->
+ <!-- <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-vfs2-project</artifactId>
+ <version>2.1</version>
+ </dependency> -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${global_slf4j_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${global_slf4j_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${global_junit_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.6.6</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${global_hadoop_version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${global_hadoop_version}</version>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${global_hadoop_version}</version>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-integration-test</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${global_jar_version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java
----------------------------------------------------------------------
diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java
new file mode 100644
index 0000000..946c71a
--- /dev/null
+++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleClassLoaders.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bundles;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.lang.invoke.MethodHandles;
+import java.net.URISyntaxException;
+import java.util.*;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.metron.bundles.bundle.Bundle;
+import org.apache.metron.bundles.bundle.BundleCoordinates;
+import org.apache.metron.bundles.bundle.BundleDetails;
+import org.apache.metron.bundles.util.BundleProperties;
+import org.apache.metron.bundles.util.BundleSelector;
+import org.apache.metron.bundles.util.FileUtils;
+import org.apache.metron.bundles.util.BundleUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A singleton class used to initialize the extension and framework classloaders.
+ */
+public final class BundleClassLoaders {
+
+ private static volatile BundleClassLoaders bundleClassLoaders;
+ private static volatile InitContext initContext;
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Holds the context from {@code BundleClassLoaders} initialization,
+ * being the coordinate to bundle mapping.
+ *
+ * After initialization these are not changed, and as such they
+ * are immutable.
+ *
+ */
+ private final static class InitContext {
+
+ private final List<FileObject> extensionDirs;
+ private final Map<String, Bundle> bundles;
+ private final BundleProperties properties;
+
+ private InitContext(
+ final List<FileObject> extensionDirs,
+ final Map<String, Bundle> bundles,
+ final BundleProperties properties) {
+ this.extensionDirs = ImmutableList.copyOf(extensionDirs);
+ this.bundles = ImmutableMap.copyOf(bundles);
+ this.properties = properties;
+ }
+ }
+
+ private BundleClassLoaders() {
+ }
+
+ /**
+ * @return The singleton instance of the BundleClassLoaders
+ * @throws NotInitializedException if BundleClassLoaders has not been init'd
+ */
+ public static BundleClassLoaders getInstance() throws NotInitializedException {
+ BundleClassLoaders result = bundleClassLoaders;
+ if (result == null) {
+ throw new NotInitializedException("BundleClassLoaders not initialized");
+ }
+ return result;
+ }
+
+ /**
+ * Uninitializes the BundleClassloaders. After calling this <code>init</code> must be called
+ * before the rest of the methods are called afterwards.
+ * This is for TESTING ONLY at this time. Reset does not unload or clear any loaded classloaders.
+ */
+ @VisibleForTesting
+ public static void reset() {
+ synchronized (BundleClassLoaders.class) {
+ initContext = null;
+ bundleClassLoaders = null;
+ }
+ }
+
+ /**
+ * Initializes and loads the BundleClassLoaders. This method must be called before the rest of the
+ * methods to access the classloaders are called. Multiple calls to this method will have no effect,
+ * unless a different set of extension directories is passed, which will result in an <code>IllegaStateException</code>
+ *
+ * @param fileSystemManager the FileSystemManager
+ * @param extensionsDirs where to find extension artifacts
+ * @param props BundleProperties
+ * @throws FileSystemException if any issue occurs while working with the bundle files.
+ * @throws java.lang.ClassNotFoundException if unable to load class definition
+ * @throws IllegalStateException when already initialized with a given set of extension directories
+ * and extensionDirs does not match
+ */
+ public static void init(final FileSystemManager fileSystemManager, final List<FileObject> extensionsDirs,
+ BundleProperties props)
+ throws FileSystemException, ClassNotFoundException, URISyntaxException {
+ if (extensionsDirs == null || fileSystemManager == null) {
+ throw new NullPointerException("cannot have empty arguments");
+ }
+ synchronized (BundleClassLoaders.class) {
+ if (bundleClassLoaders != null) {
+ throw new IllegalStateException("BundleClassloader already exists");
+ }
+ BundleClassLoaders b = new BundleClassLoaders();
+ InitContext ic = b.load(fileSystemManager, extensionsDirs, props);
+ initContext = ic;
+ bundleClassLoaders = b;
+ }
+ }
+
+ private InitContext load(final FileSystemManager fileSystemManager,
+ final List<FileObject> extensionsDirs, BundleProperties props)
+ throws FileSystemException, ClassNotFoundException, URISyntaxException {
+ // get the system classloader
+ final ClassLoader systemClassLoader = ClassLoader.getSystemClassLoader();
+
+ // find all bundle files and create class loaders for them.
+ final Map<String, Bundle> directoryBundleLookup = new LinkedHashMap<>();
+ final Map<String, ClassLoader> coordinateClassLoaderLookup = new HashMap<>();
+ final Map<String, Set<BundleCoordinates>> idBundleLookup = new HashMap<>();
+
+ for (FileObject extensionsDir : extensionsDirs) {
+ // make sure the bundle directory is there and accessible
+ FileUtils.ensureDirectoryExistAndCanRead(extensionsDir);
+
+ final List<FileObject> bundleDirContents = new ArrayList<>();
+ FileObject[] dirFiles = extensionsDir.findFiles(new BundleSelector(props.getArchiveExtension()));
+ if (dirFiles != null) {
+ List<FileObject> fileList = Arrays.asList(dirFiles);
+ bundleDirContents.addAll(fileList);
+ }
+
+ if (!bundleDirContents.isEmpty()) {
+ final List<BundleDetails> bundleDetails = new ArrayList<>();
+ final Map<String, String> bundleCoordinatesToBundleFile = new HashMap<>();
+
+ // load the bundle details which includes bundle dependencies
+ for (final FileObject bundleFile : bundleDirContents) {
+ if(!bundleFile.exists() || !bundleFile.isFile()) {
+ continue;
+ }
+ BundleDetails bundleDetail = null;
+ try {
+ bundleDetail = getBundleDetails(bundleFile, props);
+ } catch (IllegalStateException e) {
+ logger.warn("Unable to load BUNDLE {} due to {}, skipping...",
+ new Object[]{bundleFile.getURL(), e.getMessage()});
+ }
+
+ // prevent the application from starting when there are two BUNDLEs with same group, id, and version
+ final String bundleCoordinate = bundleDetail.getCoordinates().getCoordinates();
+ if (bundleCoordinatesToBundleFile.containsKey(bundleCoordinate)) {
+ final String existingBundleWorkingDir = bundleCoordinatesToBundleFile
+ .get(bundleCoordinate);
+ throw new IllegalStateException(
+ "Unable to load BUNDLE with coordinates " + bundleCoordinate
+ + " and bundle file " + bundleDetail.getBundleFile()
+ + " because another BUNDLE with the same coordinates already exists at "
+ + existingBundleWorkingDir);
+ }
+
+ bundleDetails.add(bundleDetail);
+ bundleCoordinatesToBundleFile.put(bundleCoordinate,
+ bundleDetail.getBundleFile().getURL().toURI().toString());
+ }
+
+ for (final Iterator<BundleDetails> bundleDetailsIter = bundleDetails.iterator();
+ bundleDetailsIter.hasNext(); ) {
+ final BundleDetails bundleDetail = bundleDetailsIter.next();
+ // populate bundle lookup
+ idBundleLookup.computeIfAbsent(bundleDetail.getCoordinates().getId(),
+ id -> new HashSet<>()).add(bundleDetail.getCoordinates());
+ }
+
+ int bundleCount;
+ do {
+ // record the number of bundles to be loaded
+ bundleCount = bundleDetails.size();
+
+ // attempt to create each bundle class loader
+ for (final Iterator<BundleDetails> bundleDetailsIter = bundleDetails.iterator();
+ bundleDetailsIter.hasNext(); ) {
+ final BundleDetails bundleDetail = bundleDetailsIter.next();
+ final BundleCoordinates bundleDependencyCoordinate = bundleDetail
+ .getDependencyCoordinates();
+
+ // see if this class loader is eligible for loading
+ ClassLoader potentialBundleClassLoader = null;
+ if (bundleDependencyCoordinate == null) {
+ potentialBundleClassLoader = createBundleClassLoader(fileSystemManager,
+ bundleDetail.getBundleFile(), ClassLoader.getSystemClassLoader());
+ } else {
+ final String dependencyCoordinateStr = bundleDependencyCoordinate
+ .getCoordinates();
+
+ // if the declared dependency has already been loaded
+ if (coordinateClassLoaderLookup.containsKey(dependencyCoordinateStr)) {
+ final ClassLoader bundleDependencyClassLoader = coordinateClassLoaderLookup
+ .get(dependencyCoordinateStr);
+ potentialBundleClassLoader = createBundleClassLoader(
+ fileSystemManager, bundleDetail.getBundleFile(),
+ bundleDependencyClassLoader);
+ } else {
+ // get all bundles that match the declared dependency id
+ final Set<BundleCoordinates> coordinates = idBundleLookup
+ .get(bundleDependencyCoordinate.getId());
+
+ // ensure there are known bundles that match the declared dependency id
+ if (coordinates != null && !coordinates
+ .contains(bundleDependencyCoordinate)) {
+ // ensure the declared dependency only has one possible bundle
+ if (coordinates.size() == 1) {
+ // get the bundle with the matching id
+ final BundleCoordinates coordinate = coordinates.stream()
+ .findFirst().get();
+
+ // if that bundle is loaded, use it
+ if (coordinateClassLoaderLookup
+ .containsKey(coordinate.getCoordinates())) {
+ logger.warn(String.format(
+ "While loading '%s' unable to locate exact BUNDLE dependency '%s'. Only found one possible match '%s'. Continuing...",
+ bundleDetail.getCoordinates().getCoordinates(),
+ dependencyCoordinateStr,
+ coordinate.getCoordinates()));
+
+ final ClassLoader bundleDependencyClassLoader = coordinateClassLoaderLookup
+ .get(coordinate.getCoordinates());
+ potentialBundleClassLoader = createBundleClassLoader(
+ fileSystemManager, bundleDetail.getBundleFile(),
+ bundleDependencyClassLoader);
+ }
+ }
+ }
+ }
+ }
+
+ // if we were able to create the bundle class loader, store it and remove the details
+ final ClassLoader bundleClassLoader = potentialBundleClassLoader;
+ if (bundleClassLoader != null) {
+ directoryBundleLookup
+ .put(bundleDetail.getBundleFile().getURL().toURI().toString(),
+ new Bundle(bundleDetail, bundleClassLoader));
+ coordinateClassLoaderLookup
+ .put(bundleDetail.getCoordinates().getCoordinates(),
+ bundleClassLoader);
+ bundleDetailsIter.remove();
+ }
+ }
+
+ // attempt to load more if some were successfully loaded this iteration
+ } while (bundleCount != bundleDetails.size());
+
+ // see if any bundle couldn't be loaded
+ for (final BundleDetails bundleDetail : bundleDetails) {
+ logger.warn(String
+ .format("Unable to resolve required dependency '%s'. Skipping BUNDLE '%s'",
+ bundleDetail.getDependencyCoordinates().getId(),
+ bundleDetail.getBundleFile().getURL().toURI().toString()));
+ }
+ }
+ }
+ return new InitContext(extensionsDirs, new LinkedHashMap<>(directoryBundleLookup), props);
+ }
+
+ /**
+ * Creates a new BundleClassLoader. The parentClassLoader may be null.
+ *
+ * @param bundleFile the Bundle File
+ * @param parentClassLoader parent classloader of bundle
+ * @return the bundle classloader
+ * @throws FileSystemException ioe
+ * @throws ClassNotFoundException cfne
+ */
+ private static ClassLoader createBundleClassLoader(final FileSystemManager fileSystemManager,
+ final FileObject bundleFile, final ClassLoader parentClassLoader)
+ throws FileSystemException, ClassNotFoundException {
+ logger.debug("Loading Bundle file: " + bundleFile.getURL());
+ final ClassLoader bundleClassLoader = new VFSBundleClassLoader.Builder()
+ .withFileSystemManager(fileSystemManager)
+ .withBundleFile(bundleFile)
+ .withParentClassloader(parentClassLoader).build();
+ logger.info(
+ "Loaded Bundle file: " + bundleFile.getURL() + " as class loader " + bundleClassLoader);
+ return bundleClassLoader;
+ }
+
+ /**
+ * Loads the details for the specified BUNDLE. The details will be extracted from the manifest
+ * file.
+ *
+ * @param bundleFile the bundle file
+ * @return details about the Bundle
+ * @throws FileSystemException ioe
+ */
+ private static BundleDetails getBundleDetails(final FileObject bundleFile, BundleProperties props)
+ throws FileSystemException {
+ return BundleUtil.fromBundleFile(bundleFile, props);
+ }
+
+ /**
+ * @param extensionFile the bundle file
+ * @return the bundle for the specified bundle file. Returns null when no bundle exists
+ * @throws IllegalStateException if the bundles have not been loaded
+ */
+ public Bundle getBundle(final FileObject extensionFile) {
+ if (initContext == null) {
+ throw new IllegalStateException("Extensions class loaders have not been loaded.");
+ }
+
+ try {
+ return initContext.bundles.get(extensionFile.getURL().toURI().toString());
+ } catch (URISyntaxException | FileSystemException e) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Unable to get extension classloader for bundle '{}'",
+ extensionFile.getName().toString());
+ }
+ return null;
+ }
+ }
+
+ /**
+ * @return the extensions that have been loaded
+ * @throws IllegalStateException if the extensions have not been loaded
+ */
+ public Set<Bundle> getBundles() {
+ if (initContext == null) {
+ throw new IllegalStateException("Bundles have not been loaded.");
+ }
+
+ return new LinkedHashSet<>(initContext.bundles.values());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java
----------------------------------------------------------------------
diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java
new file mode 100644
index 0000000..1f25c0e
--- /dev/null
+++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleCloseable.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bundles;
+
+import java.io.Closeable;
+
+import java.lang.invoke.MethodHandles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class BundleCloseable implements Closeable {
+
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static BundleCloseable withBundleLoader() throws NotInitializedException{
+ final ClassLoader current = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(BundleThreadContextClassLoader.getInstance());
+ return new BundleCloseable(current);
+ }
+
+ /**
+ * Sets the current thread context class loader to the specific appropriate class loader for the given
+ * component. If the component requires per-instance class loading then the class loader will be the
+ * specific class loader for instance with the given identifier, otherwise the class loader will be
+ * the BundleClassLoader.
+ *
+ * @param componentClass the component class
+ * @param componentIdentifier the identifier of the component
+ * @return BundleCloseable with the current thread context classloader jailed to the Bundle
+ * or instance class loader of the component
+ */
+ public static BundleCloseable withComponentBundleLoader(final Class componentClass, final String componentIdentifier) throws NotInitializedException{
+ final ClassLoader current = Thread.currentThread().getContextClassLoader();
+
+ ClassLoader componentClassLoader = ExtensionManager.getInstance().getInstanceClassLoader(componentIdentifier);
+ if (componentClassLoader == null) {
+ componentClassLoader = componentClass.getClassLoader();
+ }
+
+ Thread.currentThread().setContextClassLoader(componentClassLoader);
+ return new BundleCloseable(current);
+ }
+
+ /**
+ * Sets the current thread context class loader to the provided class loader, and returns a BundleCloseable that will
+ * return the current thread context class loader to it's previous state.
+ *
+ * @param componentBundleLoader the class loader to set as the current thread context class loader
+ *
+ * @return BundleCloseable that will return the current thread context class loader to its previous state
+ */
+ public static BundleCloseable withComponentBundleLoader(final ClassLoader componentBundleLoader) {
+ final ClassLoader current = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(componentBundleLoader);
+ return new BundleCloseable(current);
+ }
+ private final ClassLoader toSet;
+
+ private BundleCloseable(final ClassLoader toSet) {
+ this.toSet = toSet;
+ }
+
+ @Override
+ public void close() {
+ if (toSet != null) {
+ Thread.currentThread().setContextClassLoader(toSet);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java
----------------------------------------------------------------------
diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java
new file mode 100644
index 0000000..2df4172
--- /dev/null
+++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleManifestEntry.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.bundles;
+
+/**
+ * Enumeration of entries that will be in a BUNDLE MANIFEST file.
+ */
+public enum BundleManifestEntry {
+
+ PRE_GROUP("-Group"),
+ PRE_ID("-Id"),
+ PRE_VERSION("-Version"),
+ PRE_DEPENDENCY_GROUP("-Dependency-Group"),
+ PRE_DEPENDENCY_ID("-Dependency-Id"),
+ PRE_DEPENDENCY_VERSION("-Dependency-Version"),
+ BUILD_TAG("Build-Tag"),
+ BUILD_REVISION("Build-Revision"),
+ BUILD_BRANCH("Build-Branch"),
+ BUILD_TIMESTAMP("Build-Timestamp"),
+ BUILD_JDK("Build-Jdk"),
+ BUILT_BY("Built-By")
+ ;
+
+ final String manifestName;
+
+ BundleManifestEntry(String manifestName) {
+ this.manifestName = manifestName;
+ }
+
+ public String getManifestName() {
+ return manifestName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ffcb91ed/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java
----------------------------------------------------------------------
diff --git a/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java
new file mode 100644
index 0000000..6da5762
--- /dev/null
+++ b/bundles-lib/src/main/java/org/apache/metron/bundles/BundleMapper.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.bundles;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.metron.bundles.bundle.BundleCoordinates;
+import org.apache.metron.bundles.util.BundleProperties;
+import org.apache.metron.bundles.util.BundleSelector;
+import org.apache.metron.bundles.util.BundleUtil;
+import org.apache.metron.bundles.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The BundleMapper loads all the Bundles available to the system and maps their extensions
+ *
+ */
+public final class BundleMapper {
+
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static String archiveExtension = BundleProperties.DEFAULT_ARCHIVE_EXTENSION;
+ private final static String META_FMT = "META-INF/services/%s";
+
+ public static ExtensionMapping mapBundles(final FileSystemManager fileSystemManager,
+ BundleProperties props) {
+ try {
+ final List<URI> bundleLibraryDirs = props.getBundleLibraryDirectories();
+ final Map<FileObject, BundleCoordinates> bundlesToCoordinates = new HashMap<>();
+ archiveExtension = props.getArchiveExtension();
+
+ final List<FileObject> bundleFiles = new ArrayList<>();
+
+ for (URI bundleLibraryDir : bundleLibraryDirs) {
+
+ FileObject bundleDir = fileSystemManager.resolveFile(bundleLibraryDir);
+
+ if(bundleDir.exists() == false) {
+ continue;
+ }
+ // Test if the source BUNDLEs can be read
+ FileUtils.ensureDirectoryExistAndCanRead(bundleDir);
+
+ FileObject[] dirFiles = bundleDir.findFiles(new BundleSelector(archiveExtension));
+ if (dirFiles != null) {
+ List<FileObject> fileList = Arrays.asList(dirFiles);
+ bundleFiles.addAll(fileList);
+ }
+ }
+
+ if (!bundleFiles.isEmpty()) {
+ for (FileObject bundleFile : bundleFiles) {
+ bundlesToCoordinates
+ .put(bundleFile, BundleUtil.coordinateFromBundleFile(bundleFile, props));
+ }
+ }
+
+ final ExtensionMapping extensionMapping = new ExtensionMapping();
+ mapExtensions(bundlesToCoordinates, extensionMapping, props);
+ return extensionMapping;
+ } catch (IOException | URISyntaxException e) {
+ logger.warn("Unable to load BUNDLE library bundles due to " + e
+ + " Will proceed without loading any further bundles");
+ if (logger.isDebugEnabled()) {
+ logger.warn("", e);
+ }
+ }
+
+ return null;
+ }
+
+ private static void mapExtensions(final Map<FileObject, BundleCoordinates> bundlesToCoordinates,
+ final ExtensionMapping mapping, BundleProperties props) throws IOException {
+ for (final Map.Entry<FileObject, BundleCoordinates> entry : bundlesToCoordinates.entrySet()) {
+ final FileObject bundle = entry.getKey();
+ final BundleCoordinates bundleCoordinates = entry.getValue();
+
+ mapExtentionsForCoordinate(mapping, bundleCoordinates, bundle, props);
+ }
+ }
+
+ private static void mapExtentionsForCoordinate(final ExtensionMapping mapping,
+ final BundleCoordinates bundleCoordinates, final FileObject bundle, BundleProperties props)
+ throws IOException {
+ final FileObject bundleFileSystem = bundle.getFileSystem().getFileSystemManager()
+ .createFileSystem(bundle);
+ final FileObject deps = bundleFileSystem.resolveFile(VFSBundleClassLoader.DEPENDENCY_PATH);
+ final FileObject[] directoryContents = deps.getChildren();
+ if (directoryContents != null) {
+ for (final FileObject file : directoryContents) {
+ if (file.getName().getExtension().equals("jar")) {
+ mapExtensionsForJarFileObject(bundleCoordinates, file, mapping, props);
+ }
+ }
+ }
+ }
+
+ private static void mapExtensionsForJarFileObject(final BundleCoordinates coordinate, final FileObject jar,
+ final ExtensionMapping extensionMapping, final BundleProperties props) throws IOException {
+ final ExtensionMapping jarExtensionMapping = buildExtensionMappingForJar(coordinate, jar,
+ props);
+
+ // skip if there are not components to document
+ if (jarExtensionMapping.isEmpty()) {
+ return;
+ }
+
+ // merge the extension mapping found in this jar
+ extensionMapping.merge(jarExtensionMapping);
+ }
+
+
+ private static ExtensionMapping buildExtensionMappingForJar(final BundleCoordinates coordinate,
+ final FileObject jar, final BundleProperties props) throws IOException {
+ final ExtensionMapping mapping = new ExtensionMapping();
+
+ // The BundleProperties has configuration for the extension names and classnames
+ final Map<String, String> extensions = props.getBundleExtensionTypes();
+ if (extensions.isEmpty()) {
+ logger.info("No Extensions configured in properties");
+ return mapping;
+ }
+ JarEntry jarEntry;
+ try (final JarInputStream jarFile = new JarInputStream(jar.getContent().getInputStream())) {
+
+ while ((jarEntry = jarFile.getNextJarEntry()) != null) {
+ for (Map.Entry<String, String> extensionEntry : extensions.entrySet()) {
+ if (jarEntry.getName().equals(String.format(META_FMT, extensionEntry.getValue()))) {
+ mapping.addAllExtensions(extensionEntry.getKey(), coordinate,
+ buildExtensionMappingForJar(jarFile, jarEntry));
+ }
+ }
+ }
+ }
+ return mapping;
+
+ }
+
+ private static List<String> buildExtensionMappingForJar(final JarInputStream jarFile,
+ final JarEntry jarEntry) throws IOException {
+ final List<String> componentNames = new ArrayList<>();
+
+ if (jarEntry == null) {
+ return componentNames;
+ }
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(
+ jarFile));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ final String trimmedLine = line.trim();
+ if (!trimmedLine.isEmpty() && !trimmedLine.startsWith("#")) {
+ final int indexOfPound = trimmedLine.indexOf("#");
+ final String effectiveLine = (indexOfPound > 0) ? trimmedLine.substring(0,
+ indexOfPound) : trimmedLine;
+ componentNames.add(effectiveLine);
+ }
+ }
+ return componentNames;
+ }
+
+ private BundleMapper() {
+ }
+}