You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/07/02 22:43:51 UTC
[25/25] metron git commit: METRON-877 Extract core implementation and
UDF support,
create metron-stellar module (mattf-horton) closes apache/metron#616
METRON-877 Extract core implementation and UDF support, create metron-stellar module (mattf-horton) closes apache/metron#616
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a5b13777
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a5b13777
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a5b13777
Branch: refs/heads/master
Commit: a5b13777a92418155da61864e98fc24cde866501
Parents: df94ed4
Author: mattf-horton <mf...@hortonworks.com>
Authored: Sun Jul 2 15:38:14 2017 -0700
Committer: mattf <ma...@apache.org>
Committed: Sun Jul 2 15:38:14 2017 -0700
----------------------------------------------------------------------
.gitignore | 1 +
dependencies_with_url.csv | 1 +
metron-analytics/metron-maas-service/pom.xml | 10 +
.../metron/maas/functions/MaaSFunctions.java | 324 +++
.../metron/maas/service/MockDGAModel.java | 83 +
.../service/StellarMaaSIntegrationTest.java | 181 ++
.../profiler/client/stellar/FixedLookback.java | 8 +-
.../profiler/client/stellar/GetProfile.java | 8 +-
.../profiler/client/stellar/ProfilerConfig.java | 2 +-
.../metron/profiler/client/stellar/Util.java | 9 +-
.../profiler/client/stellar/WindowLookback.java | 10 +-
.../profiler/client/window/WindowProcessor.java | 10 +-
.../metron/profiler/client/GetProfileTest.java | 21 +-
.../client/HBaseProfilerClientTest.java | 8 +-
.../client/stellar/WindowLookbackTest.java | 10 +-
.../client/window/WindowProcessorTest.java | 10 +-
.../apache/metron/profiler/ProfileBuilder.java | 16 +-
.../stellar/DefaultStellarExecutor.java | 163 --
.../profiler/stellar/StellarExecutor.java | 81 -
.../metron/profiler/ProfileBuilderTest.java | 3 +-
.../stellar/DefaultStellarExecutorTest.java | 192 --
metron-analytics/metron-profiler/pom.xml | 2 +-
.../profiler/bolt/ProfileBuilderBolt.java | 2 +-
.../profiler/bolt/ProfileSplitterBolt.java | 17 +-
.../profiler/bolt/ProfileBuilderBoltTest.java | 3 +-
.../profiler/bolt/ProfileHBaseMapperTest.java | 4 -
.../profiler/bolt/ProfileSplitterBoltTest.java | 11 +-
metron-analytics/metron-statistics/pom.xml | 6 +-
.../apache/metron/statistics/BinFunctions.java | 71 +
.../apache/metron/statistics/MathFunctions.java | 104 -
.../statistics/StellarStatisticsFunctions.java | 11 +-
.../approximation/HyperLogLogPlusFunctions.java | 6 +-
.../MedianAbsoluteDeviationFunctions.java | 8 +-
.../metron/statistics/BinFunctionsTest.java | 53 +
.../metron/statistics/MathFunctionsTest.java | 61 -
.../StellarStatisticsFunctionsTest.java | 7 +-
...HyperLogLogPlusFunctionsIntegrationTest.java | 2 +-
.../outlier/MedianAbsoluteDeviationTest.java | 6 +-
metron-deployment/README.md | 1 +
.../rest/service/impl/StellarServiceImpl.java | 11 +-
.../metron-common/3rdPartyStellar.md | 160 -
metron-platform/metron-common/README.md | 987 +------
metron-platform/metron-common/pom.xml | 19 +-
.../metron/common/stellar/generated/Stellar.g4 | 273 --
.../metron-common/src/main/java/Stellar.tokens | 67 -
.../src/main/java/StellarLexer.tokens | 67 -
.../metron/common/aggregator/Aggregators.java | 2 +-
.../metron/common/bolt/ConfiguredBolt.java | 1 -
.../common/bolt/ConfiguredEnrichmentBolt.java | 2 -
.../common/configuration/Configurations.java | 1 -
.../configuration/ConfigurationsUtils.java | 5 +-
.../common/configuration/FieldTransformer.java | 2 +-
.../common/configuration/FieldValidator.java | 2 +-
.../configuration/IndexingConfigurations.java | 2 +-
.../configuration/SensorParserConfig.java | 1 -
.../enrichment/handler/StellarConfig.java | 5 +-
.../threatintel/ThreatTriageConfig.java | 4 +-
.../writer/ParserWriterConfiguration.java | 2 +-
.../metron/common/dsl/BaseStellarFunction.java | 42 -
.../org/apache/metron/common/dsl/Context.java | 115 -
.../apache/metron/common/dsl/ErrorListener.java | 51 -
.../metron/common/dsl/FunctionMarker.java | 21 -
.../apache/metron/common/dsl/GrammarUtils.java | 133 -
.../metron/common/dsl/MapVariableResolver.java | 49 -
.../metron/common/dsl/ParseException.java | 28 -
.../common/dsl/Predicate2StellarFunction.java | 33 -
.../org/apache/metron/common/dsl/Stellar.java | 35 -
.../metron/common/dsl/StellarFunction.java | 26 -
.../metron/common/dsl/StellarFunctionInfo.java | 117 -
.../metron/common/dsl/StellarFunctions.java | 33 -
.../org/apache/metron/common/dsl/Token.java | 72 -
.../metron/common/dsl/VariableResolver.java | 23 -
.../dsl/functions/ConversionFunctions.java | 83 -
.../dsl/functions/DataStructureFunctions.java | 220 --
.../common/dsl/functions/DateFunctions.java | 368 ---
.../dsl/functions/FunctionalFunctions.java | 121 -
.../common/dsl/functions/MaaSFunctions.java | 325 ---
.../common/dsl/functions/MapFunctions.java | 85 -
.../common/dsl/functions/NetworkFunctions.java | 293 --
.../common/dsl/functions/StringFunctions.java | 476 ---
.../common/dsl/functions/SystemFunctions.java | 87 -
.../resolver/BaseFunctionResolver.java | 202 --
.../resolver/ClasspathFunctionResolver.java | 264 --
.../functions/resolver/FunctionResolver.java | 46 -
.../resolver/SimpleFunctionResolver.java | 59 -
.../resolver/SingletonFunctionResolver.java | 34 -
.../transformation/FieldTransformation.java | 2 +-
.../IPProtocolTransformation.java | 10 +-
.../transformation/RemoveTransformation.java | 6 +-
.../SimpleFieldTransformation.java | 2 +-
.../transformation/StellarTransformation.java | 10 +-
.../field/validation/FieldValidation.java | 2 +-
.../field/validation/QueryValidation.java | 8 +-
.../field/validation/SimpleValidation.java | 2 +-
.../validation/network/DomainValidation.java | 6 +-
.../validation/network/EmailValidation.java | 5 +-
.../field/validation/network/IPValidation.java | 6 +-
.../field/validation/network/URLValidation.java | 4 +-
.../validation/primitive/DateValidation.java | 6 +-
.../validation/primitive/IntegerValidation.java | 5 +-
.../validation/primitive/RegexValidation.java | 2 +-
.../metron/common/message/MessageGetters.java | 2 +-
.../common/stellar/BaseStellarProcessor.java | 239 --
.../apache/metron/common/stellar/BooleanOp.java | 23 -
.../metron/common/stellar/FrameContext.java | 45 -
.../metron/common/stellar/LambdaExpression.java | 68 -
.../common/stellar/StellarAssignment.java | 135 -
.../metron/common/stellar/StellarCompiler.java | 719 -----
.../stellar/StellarPredicateProcessor.java | 66 -
.../metron/common/stellar/StellarProcessor.java | 51 -
.../stellar/benchmark/Microbenchmark.java | 68 -
.../benchmark/StellarMicrobenchmark.java | 268 --
.../stellar/evaluators/ArithmeticEvaluator.java | 103 -
.../ComparisonExpressionEvaluator.java | 42 -
...mparisonExpressionWithOperatorEvaluator.java | 96 -
.../ComparisonOperatorsEvaluator.java | 185 --
.../evaluators/DoubleLiteralEvaluator.java | 34 -
.../evaluators/EqualityOperatorsEvaluator.java | 75 -
.../evaluators/FloatLiteralEvaluator.java | 34 -
.../stellar/evaluators/IntLiteralEvaluator.java | 34 -
.../evaluators/LongLiteralEvaluator.java | 42 -
.../stellar/evaluators/NumberEvaluator.java | 27 -
.../evaluators/NumberLiteralEvaluator.java | 71 -
.../stellar/generated/StellarBaseListener.java | 780 -----
.../common/stellar/generated/StellarLexer.java | 306 --
.../stellar/generated/StellarListener.java | 719 -----
.../common/stellar/generated/StellarParser.java | 2739 ------------------
.../common/stellar/shell/PausableInput.java | 372 ---
.../common/stellar/shell/StellarExecutor.java | 322 --
.../common/stellar/shell/StellarShell.java | 443 ---
.../apache/metron/common/utils/BloomFilter.java | 86 -
.../metron/common/utils/ConversionUtils.java | 58 -
.../apache/metron/common/utils/KafkaUtils.java | 2 -
.../metron/common/utils/VFSClassloaderUtil.java | 168 --
.../metron-common/src/main/scripts/stellar | 2 +-
.../dsl/functions/ConversionFunctionsTest.java | 44 -
.../functions/DataStructureFunctionsTest.java | 121 -
.../dsl/functions/FunctionalFunctionsTest.java | 291 --
.../dsl/functions/StringFunctionsTest.java | 431 ---
.../dsl/functions/SystemFunctionsTest.java | 78 -
...lasspathFunctionResolverIntegrationTest.java | 87 -
.../resolver/ClasspathFunctionResolverTest.java | 135 -
.../resolver/SimpleFunctionResolverTest.java | 122 -
.../transformation/FieldTransformationTest.java | 2 +-
.../RemoveTransformationTest.java | 2 +-
.../StellarTransformationTest.java | 2 +-
.../field/validation/BaseValidationTest.java | 2 +-
.../network/DomainValidationTest.java | 2 +-
.../validation/network/EmailValidationTest.java | 2 +-
.../validation/network/IPValidationTest.java | 2 +-
.../validation/network/URLValidationTest.java | 2 +-
.../primitive/DateValidationTest.java | 2 +-
.../primitive/IntegerValidationTest.java | 2 +-
.../stellar/BaseStellarProcessorTest.java | 116 -
.../metron/common/stellar/BloomFilterTest.java | 96 -
.../common/stellar/DateFunctionsTest.java | 231 --
.../common/stellar/StellarArithmeticTest.java | 332 ---
.../common/stellar/StellarAssignmentTest.java | 63 -
...larComparisonExpressionWithOperatorTest.java | 293 --
.../common/stellar/StellarInterpreterTest.java | 172 --
.../stellar/StellarPredicateProcessorTest.java | 36 -
.../metron/common/stellar/StellarTest.java | 735 -----
.../evaluators/ArithmeticEvaluatorTest.java | 412 ---
...isonExpressionWithOperatorEvaluatorTest.java | 126 -
.../ComparisonOperatorsEvaluatorTest.java | 446 ---
.../evaluators/DoubleLiteralEvaluatorTest.java | 70 -
.../EqualityOperatorsEvaluatorTest.java | 186 --
.../evaluators/FloatLiteralEvaluatorTest.java | 74 -
.../evaluators/IntLiteralEvaluatorTest.java | 74 -
.../evaluators/LongLiteralEvaluatorTest.java | 75 -
.../evaluators/NumberLiteralEvaluatorTest.java | 107 -
.../common/stellar/maas/MockDGAModel.java | 83 -
.../maas/StellarMaaSIntegrationTest.java | 181 --
.../stellar/network/NetworkFunctionsTest.java | 156 -
.../common/utils/ConversionUtilsTest.java | 34 -
.../metron/common/utils/SerDeUtilsTest.java | 1 +
.../common/utils/StellarProcessorUtils.java | 159 -
.../TransformFilterExtractorDecorator.java | 12 +-
.../dataloads/nonbulk/flatfile/LoadOptions.java | 2 +-
.../adapters/stellar/StellarAdapter.java | 12 +-
.../apache/metron/enrichment/bolt/CacheKey.java | 2 +-
.../enrichment/bolt/GenericEnrichmentBolt.java | 5 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 8 +-
.../accesstracker/BloomAccessTracker.java | 2 +-
.../PersistentBloomTrackerCreator.java | 2 +-
.../stellar/GeoEnrichmentFunctions.java | 8 +-
.../stellar/SimpleHBaseEnrichmentFunctions.java | 9 +-
.../writer/SimpleHbaseEnrichmentWriter.java | 2 +-
.../triage/ThreatTriageProcessor.java | 14 +-
.../adapters/geo/GeoLiteDatabaseTest.java | 2 +-
.../adapters/stellar/StellarAdapterTest.java | 9 +-
.../integration/EnrichmentIntegrationTest.java | 4 -
.../stellar/DocumentationGenerator.java | 4 +-
.../stellar/GeoEnrichmentFunctionsTest.java | 8 +-
.../SimpleHBaseEnrichmentFunctionsTest.java | 7 +-
.../threatintel/triage/ThreatTriageTest.java | 4 +-
metron-platform/metron-integration-test/pom.xml | 5 +
.../StellarClasspathFunctionResolver.java | 83 +
metron-platform/metron-management/pom.xml | 12 +-
.../management/ConfigurationFunctions.java | 13 +-
.../management/EnrichmentConfigFunctions.java | 12 +-
.../metron/management/FileSystemFunctions.java | 13 +-
.../apache/metron/management/GrokFunctions.java | 8 +-
.../management/IndexingConfigFunctions.java | 10 +-
.../metron/management/KafkaFunctions.java | 16 +-
.../management/ParserConfigFunctions.java | 11 +-
.../metron/management/ShellFunctions.java | 15 +-
.../management/ThreatTriageFunctions.java | 14 +-
.../management/ConfigurationFunctionsTest.java | 6 +-
.../EnrichmentConfigFunctionsTest.java | 11 +-
.../management/FileSystemFunctionsTest.java | 2 +-
.../metron/management/GrokFunctionsTest.java | 4 +-
.../management/IndexingConfigFunctionsTest.java | 9 +-
.../KafkaFunctionsIntegrationTest.java | 8 +-
.../management/ParserConfigFunctionsTest.java | 6 +-
.../metron/management/ShellFunctionsTest.java | 8 +-
.../management/ThreatTriageFunctionsTest.java | 9 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 7 +-
.../apache/metron/parsers/csv/CSVParser.java | 2 +-
.../parsers/filters/BroMessageFilter.java | 4 +-
.../metron/parsers/filters/StellarFilter.java | 9 +-
.../parsers/interfaces/MessageFilter.java | 5 +-
.../parsers/topology/ParserTopologyCLI.java | 2 -
.../org/apache/metron/filters/FiltersTest.java | 2 +-
.../metron/parsers/bolt/ParserBoltTest.java | 6 +-
.../integration/WriterBoltIntegrationTest.java | 3 +-
metron-platform/metron-pcap/pom.xml | 7 -
.../metron/pcap/filter/PcapFieldResolver.java | 4 +-
.../pcap/filter/fixed/FixedPcapFilter.java | 6 +-
.../pcap/filter/query/QueryPcapFilter.java | 13 +-
.../pcap/pattern/ByteArrayMatcherFunction.java | 8 +-
.../pcap/pattern/ByteArrayMatchingUtilTest.java | 5 +-
metron-platform/metron-storm-kafka/pom.xml | 5 +
.../storm/kafka/flux/SpoutConfiguration.java | 3 +-
.../org/apache/metron/writer/NoopWriter.java | 2 +-
.../apache/metron/writer/hdfs/HdfsWriter.java | 13 +-
.../apache/metron/writer/kafka/KafkaWriter.java | 2 +-
metron-stellar/pom.xml | 88 +
.../stellar-common/3rdPartyStellar.md | 160 +
metron-stellar/stellar-common/README.md | 1012 +++++++
metron-stellar/stellar-common/pom.xml | 340 +++
.../metron/stellar/common/generated/Stellar.g4 | 273 ++
.../src/main/assembly/assembly.xml | 42 +
.../stellar-common/src/main/java/Stellar.tokens | 67 +
.../src/main/java/StellarLexer.tokens | 67 +
.../stellar/common/BaseStellarProcessor.java | 236 ++
.../apache/metron/stellar/common/BooleanOp.java | 23 +
.../apache/metron/stellar/common/Constants.java | 116 +
.../common/DefaultStellarStatefulExecutor.java | 162 ++
.../metron/stellar/common/FrameContext.java | 45 +
.../metron/stellar/common/LambdaExpression.java | 68 +
.../stellar/common/StellarAssignment.java | 135 +
.../metron/stellar/common/StellarCompiler.java | 719 +++++
.../common/StellarPredicateProcessor.java | 66 +
.../metron/stellar/common/StellarProcessor.java | 49 +
.../stellar/common/StellarStatefulExecutor.java | 81 +
.../common/benchmark/Microbenchmark.java | 67 +
.../common/benchmark/StellarMicrobenchmark.java | 268 ++
.../common/configuration/ConfigurationType.java | 72 +
.../configuration/ConfigurationsUtils.java | 208 ++
.../common/evaluators/ArithmeticEvaluator.java | 103 +
.../ComparisonExpressionEvaluator.java | 42 +
...mparisonExpressionWithOperatorEvaluator.java | 96 +
.../ComparisonOperatorsEvaluator.java | 185 ++
.../evaluators/DoubleLiteralEvaluator.java | 34 +
.../evaluators/EqualityOperatorsEvaluator.java | 75 +
.../evaluators/FloatLiteralEvaluator.java | 34 +
.../common/evaluators/IntLiteralEvaluator.java | 34 +
.../common/evaluators/LongLiteralEvaluator.java | 42 +
.../common/evaluators/NumberEvaluator.java | 27 +
.../evaluators/NumberLiteralEvaluator.java | 71 +
.../common/generated/StellarBaseListener.java | 780 +++++
.../stellar/common/generated/StellarLexer.java | 306 ++
.../common/generated/StellarListener.java | 719 +++++
.../stellar/common/generated/StellarParser.java | 2739 ++++++++++++++++++
.../stellar/common/shell/PausableInput.java | 372 +++
.../stellar/common/shell/StellarExecutor.java | 322 ++
.../stellar/common/shell/StellarShell.java | 440 +++
.../metron/stellar/common/system/Clock.java | 37 +
.../stellar/common/system/Environment.java | 27 +
.../stellar/common/utils/BloomFilter.java | 86 +
.../stellar/common/utils/ConversionUtils.java | 58 +
.../metron/stellar/common/utils/JSONUtils.java | 86 +
.../metron/stellar/common/utils/SerDeUtils.java | 256 ++
.../common/utils/StellarProcessorUtils.java | 163 ++
.../common/utils/VFSClassloaderUtil.java | 168 ++
.../stellar/common/utils/cli/OptionHandler.java | 31 +
.../metron/stellar/dsl/BaseStellarFunction.java | 42 +
.../org/apache/metron/stellar/dsl/Context.java | 113 +
.../metron/stellar/dsl/ErrorListener.java | 50 +
.../metron/stellar/dsl/FunctionMarker.java | 21 +
.../apache/metron/stellar/dsl/GrammarUtils.java | 133 +
.../metron/stellar/dsl/MapVariableResolver.java | 49 +
.../metron/stellar/dsl/ParseException.java | 28 +
.../stellar/dsl/Predicate2StellarFunction.java | 33 +
.../org/apache/metron/stellar/dsl/Stellar.java | 35 +
.../metron/stellar/dsl/StellarFunction.java | 26 +
.../metron/stellar/dsl/StellarFunctionInfo.java | 116 +
.../metron/stellar/dsl/StellarFunctions.java | 33 +
.../org/apache/metron/stellar/dsl/Token.java | 72 +
.../metron/stellar/dsl/VariableResolver.java | 23 +
.../dsl/functions/ConversionFunctions.java | 83 +
.../dsl/functions/DataStructureFunctions.java | 220 ++
.../stellar/dsl/functions/DateFunctions.java | 368 +++
.../dsl/functions/FunctionalFunctions.java | 121 +
.../stellar/dsl/functions/MapFunctions.java | 84 +
.../stellar/dsl/functions/MathFunctions.java | 63 +
.../stellar/dsl/functions/NetworkFunctions.java | 293 ++
.../stellar/dsl/functions/StringFunctions.java | 476 +++
.../stellar/dsl/functions/SystemFunctions.java | 87 +
.../resolver/BaseFunctionResolver.java | 202 ++
.../resolver/ClasspathFunctionResolver.java | 262 ++
.../functions/resolver/FunctionResolver.java | 46 +
.../resolver/SimpleFunctionResolver.java | 59 +
.../resolver/SingletonFunctionResolver.java | 34 +
.../src/main/resources/META-INF/LICENSE | 605 ++++
.../src/main/resources/META-INF/NOTICE | 28 +
.../stellar-common/src/main/scripts/stellar | 34 +
.../classpath-resources/custom-1.0-SNAPSHOT.jar | Bin 0 -> 3570 bytes
.../common/BaseStellarProcessorTest.java | 116 +
.../DefaultStellarStatefulExecutorTest.java | 189 ++
.../stellar/common/StellarArithmeticTest.java | 332 +++
.../stellar/common/StellarAssignmentTest.java | 64 +
...larComparisonExpressionWithOperatorTest.java | 293 ++
.../stellar/common/StellarInterpreterTest.java | 169 ++
.../common/StellarPredicateProcessorTest.java | 36 +
.../evaluators/ArithmeticEvaluatorTest.java | 412 +++
...isonExpressionWithOperatorEvaluatorTest.java | 126 +
.../ComparisonOperatorsEvaluatorTest.java | 446 +++
.../evaluators/DoubleLiteralEvaluatorTest.java | 70 +
.../EqualityOperatorsEvaluatorTest.java | 186 ++
.../evaluators/FloatLiteralEvaluatorTest.java | 74 +
.../evaluators/IntLiteralEvaluatorTest.java | 74 +
.../evaluators/LongLiteralEvaluatorTest.java | 75 +
.../evaluators/NumberLiteralEvaluatorTest.java | 107 +
.../common/network/NetworkFunctionsTest.java | 156 +
.../metron/stellar/common/system/ClockTest.java | 54 +
.../stellar/common/utils/BloomFilterTest.java | 95 +
.../common/utils/ConversionUtilsTest.java | 35 +
.../stellar/common/utils/JSONUtilsTest.java | 102 +
.../stellar/common/utils/SerDeUtilsTest.java | 216 ++
.../stellar/common/utils/UnitTestHelper.java | 244 ++
.../stellar/dsl/functions/BasicStellarTest.java | 739 +++++
.../dsl/functions/ConversionFunctionsTest.java | 44 +
.../functions/DataStructureFunctionsTest.java | 121 +
.../dsl/functions/DateFunctionsTest.java | 232 ++
.../dsl/functions/FunctionalFunctionsTest.java | 291 ++
.../dsl/functions/MathFunctionsTest.java | 47 +
.../dsl/functions/StringFunctionsTest.java | 430 +++
.../dsl/functions/SystemFunctionsTest.java | 78 +
.../resolver/ClasspathFunctionResolverTest.java | 139 +
.../resolver/SimpleFunctionResolverTest.java | 122 +
.../src/test/resources/config/global.json | 3 +
.../src/test/resources/log4j.properties | 24 +
pom.xml | 15 +-
site-book/pom.xml | 3 +-
356 files changed, 22640 insertions(+), 20021 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 6789e7a..d505c9c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
*~
target/
+*/target/
*dependency-reduced-pom.xml
.idea
*.iml
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 3752366..7c3bf1f 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -19,6 +19,7 @@ com.esotericsoftware.minlog:minlog:jar:1.3.0:compile,New BSD License,http://code
com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD License,http://code.google.com/p/minlog/
com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD License,http://code.google.com/p/minlog/
com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf
+com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf
com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB
com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
index 32a9e5d..0f8abb1 100644
--- a/metron-analytics/metron-maas-service/pom.xml
+++ b/metron-analytics/metron-maas-service/pom.xml
@@ -38,6 +38,16 @@
<version>${project.parent.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>stellar-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<version>${hadoop.version}</version>
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
new file mode 100644
index 0000000..96d43c2
--- /dev/null
+++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.maas.functions;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.maas.config.Endpoint;
+import org.apache.metron.maas.config.MaaSConfig;
+import org.apache.metron.maas.config.ModelEndpoint;
+import org.apache.metron.maas.discovery.ServiceDiscoverer;
+import org.apache.metron.maas.util.ConfigUtil;
+import org.apache.metron.maas.util.RESTUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+public class MaaSFunctions {
+ protected static final Logger LOG = LoggerFactory.getLogger(MaaSFunctions.class);
+ private static class ModelCacheKey {
+ String name;
+ String version;
+ String method;
+ Map<String, String> args;
+ public ModelCacheKey(String name, String version, String method, Map<String, String> args) {
+ this.name = name;
+ this.version = version;
+ this.method = method;
+ this.args = args;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ModelCacheKey that = (ModelCacheKey) o;
+
+ if (name != null ? !name.equals(that.name) : that.name != null) return false;
+ if (version != null ? !version.equals(that.version) : that.version != null) return false;
+ if (method != null ? !method.equals(that.method) : that.method != null) return false;
+ return args != null ? args.equals(that.args) : that.args == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name != null ? name.hashCode() : 0;
+ result = 31 * result + (version != null ? version.hashCode() : 0);
+ result = 31 * result + (method != null ? method.hashCode() : 0);
+ result = 31 * result + (args != null ? args.hashCode() : 0);
+ return result;
+ }
+ }
+
+ @Stellar(name="MODEL_APPLY"
+ , namespace="MAAS"
+ , description = "Returns the output of a model deployed via Model as a Service. NOTE: Results are cached locally for 10 minutes."
+ , params = { "endpoint - A map containing the name, version, and url for the REST endpoint"
+ , "function - The optional endpoint path; default is 'apply'"
+ , "model_args - A Dictionary of arguments for the model (these become request params)"
+ }
+ , returns = "The output of the model deployed as a REST endpoint in Map form. Assumes REST endpoint returns a JSON Map."
+ )
+ public static class ModelApply implements StellarFunction {
+ private boolean isInitialized = false;
+ private ServiceDiscoverer discoverer;
+ private Cache<ModelCacheKey, Map<String, Object> > resultCache;
+ public ModelApply() {
+ resultCache = CacheBuilder.newBuilder()
+ .concurrencyLevel(4)
+ .weakKeys()
+ .maximumSize(100000)
+ .expireAfterWrite(10, TimeUnit.MINUTES)
+ .build();
+ }
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(args.size() < 2) {
+ throw new ParseException("Unable to execute model_apply. " +
+ "Expected arguments: endpoint_map:map, " +
+ " [endpoint method:string], model_args:map"
+ );
+ }
+ if(!isInitialized) {
+ return null;
+ }
+ int i = 0;
+ if(args.size() == 0) {
+ return null;
+ }
+ Object endpointObj = args.get(i++);
+ Map endpoint = null;
+ String modelName;
+ String modelVersion;
+ String modelUrl;
+ if(endpointObj instanceof Map) {
+ endpoint = (Map)endpointObj;
+ modelName = endpoint.get("name") + "";
+ modelVersion = endpoint.get("version") + "";
+ modelUrl = endpoint.get("url") + "";
+ }
+ else {
+ return null;
+ }
+ String modelFunction = "apply";
+ Map<String, String> modelArgs = new HashMap<>();
+ if(args.get(i) instanceof String) {
+ String func = (String)args.get(i);
+ if(endpoint.containsKey("endpoint:" + func)) {
+ modelFunction = "" + endpoint.get("endpoint:" + func);
+ }
+ else {
+ modelFunction = func;
+ }
+ i++;
+ }
+
+ if(args.get(i) instanceof Map) {
+ if(endpoint.containsKey("endpoint:apply")) {
+ modelFunction = "" + endpoint.get("endpoint:apply");
+ }
+ modelArgs = (Map)args.get(i);
+ }
+ if( modelName == null
+ || modelVersion == null
+ || modelFunction == null
+ ) {
+ return null;
+ }
+ ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs);
+ Map<String, Object> ret = resultCache.getIfPresent(cacheKey);
+ if(ret != null) {
+ return ret;
+ }
+ else {
+ String url = modelUrl;
+ if (url.endsWith("/")) {
+ url = url.substring(0, url.length() - 1);
+ }
+ if (modelFunction.startsWith("/")) {
+ modelFunction = modelFunction.substring(1);
+ }
+ try {
+ URL u = new URL(url + "/" + modelFunction);
+
+ String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs);
+ ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() {
+ });
+ resultCache.put(cacheKey, ret);
+ return ret;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ if (discoverer != null) {
+ try {
+ URL u = new URL(modelUrl);
+ discoverer.blacklist(u);
+ } catch (MalformedURLException e1) {
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public synchronized void initialize(Context context) {
+
+ try {
+ Optional<ServiceDiscoverer> discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER));
+ if (discovererOpt.isPresent()) {
+ discoverer = discovererOpt.get();
+ }
+ else {
+ Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT);
+ CuratorFramework client = null;
+ if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) {
+ client = (CuratorFramework) clientOptional.get();
+ } else {
+ throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client.");
+ }
+ discoverer = createDiscoverer(client);
+ }
+ }
+ catch(Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ }
+ finally {
+ //We always want to set initialize to true because we don't want to keep trying to initialize over and over
+ isInitialized = true;
+ }
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return isInitialized;
+ }
+ }
+
+ private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception {
+ MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
+ ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
+ discoverer.start();
+ return discoverer;
+ }
+
+ @Stellar(name="GET_ENDPOINT"
+ , namespace="MAAS"
+ , description="Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters."
+ , params = {
+ "model_name - The name of the model"
+ ,"model_version - The optional version of the model. If the model version is not specified, the most current version is used."
+ }
+ , returns = "A map containing the name, version, and url for the REST endpoint (fields named name, version and url). " +
+ "Note that the output of this function is suitable for input into the first argument of MAAS_MODEL_APPLY."
+ )
+ public static class GetEndpoint implements StellarFunction {
+ ServiceDiscoverer discoverer;
+ private boolean isInitialized = false;
+ private boolean isValidState = false;
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ if(!isValidState) {
+ LOG.error("Invalid state: Unable to find ServiceDiscoverer service.");
+ return null;
+ }
+ String modelName = null;
+ String modelVersion = null;
+ if(args.size() >= 1) {
+ modelName = args.get(0).toString();
+ }
+ if(args.size() >= 2)
+ {
+ modelVersion = args.get(1).toString();
+ }
+ if(modelName == null) {
+ return null;
+ }
+ try {
+ ModelEndpoint ep = null;
+ if (modelVersion == null) {
+ ep = discoverer.getEndpoint(modelName);
+ } else {
+ ep = discoverer.getEndpoint(modelName, modelVersion);
+ }
+ return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint());
+ }
+ catch(Exception ex) {
+ LOG.error("Unable to discover endpoint: " + ex.getMessage(), ex);
+ return null;
+ }
+ }
+
+ public static Map<String, String> endpointToMap(String name, String version, Endpoint ep) {
+ Map<String, String> ret = new HashMap<>();
+ ret.put("url", ep.getUrl());
+ ret.put("name", name);
+ ret.put("version", version);
+ for(Map.Entry<String, String> kv : ep.getFunctions().entrySet()) {
+ ret.put("endpoint:" + kv.getKey(), kv.getValue());
+ }
+ return ret;
+ }
+
+ @Override
+ public synchronized void initialize(Context context) {
+ try {
+ Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT);
+ CuratorFramework client = null;
+ if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) {
+ client = (CuratorFramework) clientOptional.get();
+ } else {
+ throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client.");
+ }
+ try {
+ discoverer = createDiscoverer(client);
+ context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer);
+ isValidState = true;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e);
+ }
+ }
+ finally {
+ isInitialized = true;
+ }
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return isInitialized;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
new file mode 100644
index 0000000..18604b4
--- /dev/null
+++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.maas.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.ImmutableMap;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.metron.common.utils.JSONUtils;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.ext.RuntimeDelegate;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.*;
+
+@Path("/")
+public class MockDGAModel {
+ private static HttpServer server;
+ private Map<String, Boolean> isMalicious = ImmutableMap.of( "badguy.com", true
+ );
+
+ @GET
+ @Path("/apply")
+ @Produces("application/json")
+ public Response apply(@QueryParam("host") String host ) throws JsonProcessingException {
+ Boolean b = isMalicious.get(host);
+ boolean isMalicious = b != null && b;
+ Map<String, Boolean> ret = new HashMap<String, Boolean>();
+ ret.put("is_malicious", isMalicious );
+ String resp = JSONUtils.INSTANCE.toJSON(ret, true);
+ return Response.ok(resp, MediaType.APPLICATION_JSON_TYPE).build();
+ }
+
+ @ApplicationPath("rs")
+ public static class ApplicationConfig extends Application {
+ private final Set<Class<?>> classes;
+ public ApplicationConfig() {
+ HashSet<Class<?>> c = new HashSet<>();
+ c.add(MockDGAModel.class);
+ classes = Collections.unmodifiableSet(c);
+ }
+ @Override
+ public Set<Class<?>> getClasses() {
+ return classes;
+ }
+ }
+
+ public static void start(int port) throws IOException {
+ // Create an HTTP server listening at port
+ URI uri = UriBuilder.fromUri("http://localhost/").port(port).build();
+ server = HttpServer.create(new InetSocketAddress(uri.getPort()), 0);
+ HttpHandler handler = RuntimeDelegate.getInstance().createEndpoint(new ApplicationConfig(), HttpHandler.class);
+ server.createContext(uri.getPath(), handler);
+ server.start();
+ }
+
+ public static void shutdown() {
+ if(server != null) {
+ server.stop(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java
new file mode 100644
index 0000000..4b95af5
--- /dev/null
+++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.maas.service;
+
+import com.google.common.collect.ImmutableMap;
+import com.sun.jersey.server.impl.application.WebApplicationImpl;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.CloseableUtils;
+import org.apache.curator.x.discovery.ServiceInstance;
+import org.apache.curator.x.discovery.ServiceInstanceBuilder;
+import org.apache.curator.x.discovery.ServiceType;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.maas.config.Endpoint;
+import org.apache.metron.maas.config.MaaSConfig;
+import org.apache.metron.maas.config.ModelEndpoint;
+import org.apache.metron.maas.discovery.ServiceDiscoverer;
+import org.apache.metron.maas.util.ConfigUtil;
+import org.apache.metron.test.utils.UnitTestHelper;
+import org.junit.*;
+
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+
+public class StellarMaaSIntegrationTest {
+ private static Context context;
+ private static TestingServer testZkServer;
+ private static String zookeeperUrl;
+ private static CuratorFramework client;
+ private static ServiceDiscoverer discoverer;
+ private static URL endpointUrl;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ UnitTestHelper.setJavaLoggingLevel(WebApplicationImpl.class, Level.WARNING);
+ MockDGAModel.start(8282);
+ testZkServer = new TestingServer(true);
+ zookeeperUrl = testZkServer.getConnectString();
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
+ client.start();
+ context = new Context.Builder()
+ .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
+ .build();
+ MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class);
+ discoverer = new ServiceDiscoverer(client, config.getServiceRoot());
+ discoverer.start();
+ endpointUrl = new URL("http://localhost:8282");
+ ModelEndpoint endpoint = new ModelEndpoint();
+ {
+ endpoint.setName("dga");
+ endpoint.setContainerId("0");
+ Endpoint ep = new Endpoint();
+ ep.setUrl(endpointUrl.toString());
+ endpoint.setEndpoint(ep);
+ endpoint.setVersion("1.0");
+ }
+ ;
+
+ ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder()
+ .address(endpointUrl.getHost())
+ .id("0")
+ .name("dga")
+ .port(endpointUrl.getPort())
+ .registrationTimeUTC(System.currentTimeMillis())
+ .serviceType(ServiceType.STATIC)
+ .payload(endpoint);
+ final ServiceInstance<ModelEndpoint> instance = builder.build();
+ discoverer.getServiceDiscovery().registerService(instance);
+ //wait til the endpoint is installed...
+ for(int i = 0;i < 10;++i) {
+ try {
+ Object o = discoverer.getEndpoint("dga");
+ if(o != null) {
+ break;
+ }
+ }
+ catch(Exception e) {
+
+ }
+ Thread.sleep(1000);
+ }
+ }
+
+ @Test
+ public void testGetEndpointWithoutVersion() throws Exception {
+ String stellar = "MAAS_GET_ENDPOINT('dga')";
+ Object result = run(stellar, new HashMap<>(), context);
+ Assert.assertTrue(result instanceof Map);
+ Map<String, String> resMap = (Map<String, String>)result;
+ Assert.assertEquals(resMap.get("url"), "http://localhost:8282");
+ Assert.assertEquals(resMap.get("name"), "dga");
+ Assert.assertEquals(resMap.get("version"), "1.0");
+ Assert.assertEquals(resMap.get("endpoint:apply"), "apply");
+
+ }
+
+ @Test
+ public void testGetEndpointWithVersion() throws Exception {
+ String stellar = "MAAS_GET_ENDPOINT('dga', '1.0')";
+ Object result = run(stellar, new HashMap<>(), context);
+ Assert.assertTrue(result instanceof Map);
+ Map<String, String> resMap = (Map<String, String>)result;
+ Assert.assertEquals(resMap.get("url"), "http://localhost:8282");
+ Assert.assertEquals(resMap.get("name"), "dga");
+ Assert.assertEquals(resMap.get("version"), "1.0");
+ Assert.assertEquals(resMap.get("endpoint:apply"), "apply");
+ }
+
+ @Test
+ public void testGetEndpointWithWrongVersion() throws Exception {
+ String stellar = "MAAS_GET_ENDPOINT('dga', '2.0')";
+ Object result = run(stellar, new HashMap<>(), context);
+ Assert.assertNull(result);
+ }
+
+ @Test
+ public void testModelApply() throws Exception {
+ {
+ String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))";
+ Object result = run(stellar, ImmutableMap.of("host", "badguy.com"), context);
+ Assert.assertTrue((Boolean) result);
+ }
+ {
+ String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))";
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Assert.assertFalse((Boolean) result);
+ }
+ {
+ String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), 'apply', {'host': host}))";
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Assert.assertFalse((Boolean) result);
+ }
+
+ }
+
+ @Test
+ public void testModelApplyNegative() {
+ {
+ String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga', '2.0'), {'host': host}))";
+ Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context);
+ Assert.assertNull( result);
+ }
+ }
+
+ @AfterClass
+ public static void teardown() {
+ MockDGAModel.shutdown();
+ if(discoverer != null) {
+ CloseableUtils.closeQuietly(discoverer);
+ }
+ if(client != null) {
+ CloseableUtils.closeQuietly(client);
+ }
+ if(testZkServer != null) {
+ CloseableUtils.closeQuietly(testZkServer);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java
index c4ed582..02b4a4c 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java
@@ -19,10 +19,10 @@
*/
package org.apache.metron.profiler.client.stellar;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
import org.apache.metron.profiler.ProfilePeriod;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
index 87232aa..04b391e 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
@@ -22,10 +22,10 @@ package org.apache.metron.profiler.client.stellar;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
import org.apache.metron.hbase.HTableProvider;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.profiler.ProfilePeriod;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java
index f409ca8..e2ec275 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java
@@ -20,7 +20,7 @@
package org.apache.metron.profiler.client.stellar;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.hbase.HTableProvider;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java
index ab22967..f30af74 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java
@@ -20,13 +20,12 @@
package org.apache.metron.profiler.client.stellar;
import org.apache.commons.lang.StringUtils;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -34,7 +33,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.lang.String.format;
-import static org.apache.metron.common.dsl.Context.Capabilities.GLOBAL_CONFIG;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
public class Util {
private static final Logger LOG = LoggerFactory.getLogger(Util.class);
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java
index c7390fa..9e420e5 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java
@@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.stellar;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Stellar;
-import org.apache.metron.common.dsl.StellarFunction;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.profiler.ProfilePeriod;
import org.apache.metron.profiler.client.window.Window;
import org.apache.metron.profiler.client.window.WindowProcessor;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
index 1ae4aaf..80dd649 100644
--- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
+++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
@@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.window;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.tree.ParseTree;
-import org.apache.metron.common.dsl.ErrorListener;
-import org.apache.metron.common.dsl.GrammarUtils;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.Token;
-import org.apache.metron.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.ErrorListener;
+import org.apache.metron.stellar.dsl.GrammarUtils;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Token;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.profiler.client.window.generated.WindowBaseListener;
import org.apache.metron.profiler.client.window.generated.WindowLexer;
import org.apache.metron.profiler.client.window.generated.WindowParser;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
index 4bc3167..917a5ca 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java
@@ -22,10 +22,9 @@ package org.apache.metron.profiler.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver;
-import org.apache.metron.common.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
import org.apache.metron.hbase.TableProvider;
import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.metron.profiler.client.stellar.FixedLookback;
@@ -34,8 +33,8 @@ import org.apache.metron.profiler.hbase.ColumnBuilder;
import org.apache.metron.profiler.hbase.RowKeyBuilder;
import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
import org.apache.metron.test.mock.MockHTable;
import org.junit.Assert;
import org.junit.Before;
@@ -62,7 +61,7 @@ public class GetProfileTest {
private static final int saltDivisor = 1000;
private static final String tableName = "profiler";
private static final String columnFamily = "P";
- private StellarExecutor executor;
+ private StellarStatefulExecutor executor;
private Map<String, Object> state;
private ProfileWriter profileWriter;
// different values of period and salt divisor, used to test config_overrides feature
@@ -119,7 +118,7 @@ public class GetProfileTest {
}};
// create the stellar execution environment
- executor = new DefaultStellarExecutor(
+ executor = new DefaultStellarStatefulExecutor(
new SimpleFunctionResolver()
.withClass(GetProfile.class)
.withClass(FixedLookback.class),
@@ -140,9 +139,9 @@ public class GetProfileTest {
* original context values in the PROFILE_GET config_overrides argument gets all expected results.
*
* @return context2 - The profiler client configuration context created by this method.
- * The context2 values are also set in the configuration of the StellarExecutor
+ * The context2 values are also set in the configuration of the StellarStatefulExecutor
* stored in the global variable 'executor'. However, there is no API for querying the
- * context values from a StellarExecutor, so we output the context2 Context object itself,
+ * context values from a StellarStatefulExecutor, so we output the context2 Context object itself,
* for validation purposes (so that its values can be validated as being significantly
* different from the setup() settings).
*/
@@ -165,7 +164,7 @@ public class GetProfileTest {
.build();
// create the stellar execution environment
- executor = new DefaultStellarExecutor(
+ executor = new DefaultStellarStatefulExecutor(
new SimpleFunctionResolver()
.withClass(GetProfile.class)
.withClass(FixedLookback.class),
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
index d3a0fe5..960e4d2 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
@@ -25,8 +25,8 @@ import org.apache.metron.profiler.hbase.ColumnBuilder;
import org.apache.metron.profiler.hbase.RowKeyBuilder;
import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
import org.apache.metron.test.mock.MockHTable;
import org.junit.After;
import org.junit.AfterClass;
@@ -58,15 +58,15 @@ public class HBaseProfilerClientTest {
private static final int periodsPerHour = 4;
private HBaseProfilerClient client;
+ private StellarStatefulExecutor executor;
private MockHTable table;
- private StellarExecutor executor;
private ProfileWriter profileWriter;
@Before
public void setup() throws Exception {
table = new MockHTable(tableName, columnFamily);
- executor = new DefaultStellarExecutor();
+ executor = new DefaultStellarStatefulExecutor();
// used to write values to be read during testing
RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder();
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
index aafbe5c..41d2074 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java
@@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.stellar;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.Range;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
-import org.apache.metron.common.stellar.StellarProcessor;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
+import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
+import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.profiler.ProfilePeriod;
import org.apache.metron.profiler.client.window.WindowProcessor;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
index 79212e2..1b8d55d 100644
--- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
+++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java
@@ -293,27 +293,27 @@ public class WindowProcessorTest {
}
}
- @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+ @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
public void testWithInvalidDaySpecifier() throws ParseException {
WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding hoolidays:us");
}
- @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+ @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
public void testWithInvalidTimeUnit() throws ParseException {
WindowProcessor.process("30 minute window every 24 months from 14 days ago");
}
- @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+ @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
public void testWithInvalidWindowUnit() throws ParseException {
WindowProcessor.process("30 minuete window every 24 hours from 14 days ago");
}
- @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+ @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
public void testWithInvalidTimeNumber() throws ParseException {
WindowProcessor.process("30p minute window every 24 hours from 14 days ago");
}
- @Test(expected=org.apache.metron.common.dsl.ParseException.class)
+ @Test(expected=org.apache.metron.stellar.dsl.ParseException.class)
public void testInvalidDaySpecifier() throws ParseException {
WindowProcessor.process("30 minute window every 14 hours from 14 days ago including date");
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
index 88d6e1b..4a9946d 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java
@@ -26,14 +26,13 @@ import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
-import org.apache.metron.common.configuration.profiler.ProfileResult;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.ParseException;
-import org.apache.metron.common.dsl.StellarFunctions;
+import org.apache.metron.stellar.common.StellarStatefulExecutor;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.metron.profiler.clock.Clock;
import org.apache.metron.profiler.clock.WallClock;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.apache.metron.profiler.stellar.StellarExecutor;
+import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -81,7 +79,7 @@ public class ProfileBuilder implements Serializable {
/**
* Executes Stellar code and maintains state across multiple invocations.
*/
- private StellarExecutor executor;
+ private StellarStatefulExecutor executor;
/**
* Has the profile been initialized?
@@ -114,7 +112,7 @@ public class ProfileBuilder implements Serializable {
this.entity = entity;
this.clock = clock;
this.periodDurationMillis = periodDurationMillis;
- this.executor = new DefaultStellarExecutor();
+ this.executor = new DefaultStellarStatefulExecutor();
Context context = new Context.Builder()
.with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client)
.with(Context.Capabilities.GLOBAL_CONFIG, () -> global)
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
deleted file mode 100644
index 62bab85..0000000
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.metron.profiler.stellar;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.commons.lang.ClassUtils;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.dsl.MapVariableResolver;
-import org.apache.metron.common.dsl.StellarFunctions;
-import org.apache.metron.common.dsl.VariableResolver;
-import org.apache.metron.common.stellar.StellarProcessor;
-import org.apache.metron.common.utils.ConversionUtils;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * The default implementation of a StellarExecutor.
- */
-public class DefaultStellarExecutor implements StellarExecutor, Serializable {
-
- /**
- * The current state of the Stellar execution environment.
- */
- private Map<String, Object> state;
-
- /**
- * Provides additional context for initializing certain Stellar functions. For
- * example, references to Zookeeper or HBase.
- */
- private Context context;
-
- /**
- * Responsible for function resolution.
- */
- private FunctionResolver functionResolver;
-
- public DefaultStellarExecutor() {
- this(StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT());
- }
-
- public DefaultStellarExecutor(FunctionResolver functionResolver, Context context) {
- clearState();
- this.context = context;
- this.functionResolver = functionResolver;
- }
-
- /**
- * @param initialState Initial state loaded into the execution environment.
- */
- public DefaultStellarExecutor(Map<String, Object> initialState) {
- this();
- this.state = new HashMap<>(initialState);
- }
-
- /**
- * The current state of the Stellar execution environment.
- */
- @Override
- public Map<String, Object> getState() {
- return ImmutableMap.copyOf(state);
- }
-
- /**
- * Execute an expression and assign the result to a variable. The variable is maintained
- * in the context of this executor and is available to all subsequent expressions.
- *
- * @param variable The name of the variable to assign to.
- * @param expression The expression to execute.
- * @param transientState Additional state available to the expression. This most often represents
- * the values available to the expression from an individual message. The state
- * maps a variable name to a variable's value.
- */
- @Override
- public void assign(String variable, String expression, Map<String, Object> transientState) {
- Object result = execute(expression, transientState);
- if(result == null || variable == null) {
- return;
- }
- state.put(variable, result);
- }
-
- @Override
- public void assign(String variable, Object value) {
- if(value == null || variable == null) {
- return;
- }
- state.put(variable, value);
- }
-
- /**
- * Execute a Stellar expression and return the result. The internal state of the executor
- * is not modified.
- *
- * @param expression The expression to execute.
- * @param state Additional state available to the expression. This most often represents
- * the values available to the expression from an individual message. The state
- * maps a variable name to a variable's value.
- * @param clazz The expected type of the expression's result.
- * @param <T> The expected type of the expression's result.
- */
- @Override
- public <T> T execute(String expression, Map<String, Object> state, Class<T> clazz) {
- Object resultObject = execute(expression, state);
-
- // perform type conversion, if necessary
- T result = ConversionUtils.convert(resultObject, clazz);
- if (result == null) {
- throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s",
- clazz.getSimpleName(), ClassUtils.getShortClassName(resultObject,"null"), expression));
- }
-
- return result;
- }
-
- @Override
- public void clearState() {
- this.state = new HashMap<>();
- }
-
- @Override
- public void setContext(Context context) {
- this.context = context;
- }
-
- public void setFunctionResolver(FunctionResolver functionResolver) {
- this.functionResolver = functionResolver;
- }
-
- /**
- * Execute a Stellar expression.
- *
- * @param expression The expression to execute.
- * @param transientState Additional state available to the expression. This most often represents
- * the values available to the expression from an individual message. The state
- * maps a variable name to a variable's value.
- */
- private Object execute(String expression, Map<String, Object> transientState) {
- VariableResolver variableResolver = new MapVariableResolver(state, transientState);
- StellarProcessor processor = new StellarProcessor();
- return processor.parse(expression, variableResolver, functionResolver, context);
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
deleted file mode 100644
index 2342eb7..0000000
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.metron.profiler.stellar;
-
-import org.apache.metron.common.dsl.Context;
-
-import java.util.Map;
-
-/**
- * Executes Stellar expressions and maintains state across multiple invocations.
- */
-public interface StellarExecutor {
-
- /**
- * Assign a variable a specific value.
- * @param variable The variable name.
- * @param value The value to assign to the variable.
- */
- void assign(String variable, Object value);
-
- /**
- * Execute an expression and assign the result to a variable. The variable is maintained
- * in the context of this executor and is available to all subsequent expressions.
- *
- * @param variable The name of the variable to assign to.
- * @param expression The expression to execute.
- * @param state Additional state available to the expression. This most often represents
- * the values available to the expression from an individual message. The state
- * maps a variable name to a variable's value.
- */
- void assign(String variable, String expression, Map<String, Object> state);
-
- /**
- * Execute a Stellar expression and return the result. The internal state of the executor
- * is not modified.
- *
- * @param expression The expression to execute.
- * @param state Additional state available to the expression. This most often represents
- * the values available to the expression from an individual message. The state
- * maps a variable name to a variable's value.
- * @param clazz The expected type of the expression's result.
- * @param <T> The expected type of the expression's result.
- */
- <T> T execute(String expression, Map<String, Object> state, Class<T> clazz);
-
- /**
- * The current state of the Stellar execution environment.
- */
- Map<String, Object> getState();
-
- /**
- * Removes all state from the execution environment.
- */
- void clearState();
-
- /**
- * Sets the Context for the Stellar execution environment. This provides global data used
- * to initialize Stellar functions.
- *
- * @param context The Stellar context.
- */
- void setContext(Context context);
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
index 794fde4..aa632e4 100644
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
+++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java
@@ -23,7 +23,6 @@ package org.apache.metron.profiler;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.profiler.clock.Clock;
import org.apache.metron.profiler.clock.FixedClock;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
@@ -32,7 +31,7 @@ import org.junit.Test;
import java.util.concurrent.TimeUnit;
-import static org.apache.metron.common.utils.ConversionUtils.convert;
+import static org.apache.metron.stellar.common.utils.ConversionUtils.convert;
import static org.junit.Assert.assertEquals;
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java
deleted file mode 100644
index d569536..0000000
--- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.metron.profiler.stellar;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.common.dsl.Context;
-import org.apache.metron.common.dsl.functions.StringFunctions;
-import org.apache.metron.common.dsl.functions.resolver.ClasspathFunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.FunctionResolver;
-import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver;
-import org.apache.metron.common.field.validation.primitive.IntegerValidation;
-import org.apache.metron.profiler.stellar.DefaultStellarExecutor;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-/**
- * Tests the DefaultStellarExecutor.
- */
-@SuppressWarnings("unchecked")
-// This test class passes raw JSONObject to the the executor, which gives unchecked cast warnings.
-// Suppressing on the class level, given that every test is a typical example of use pattern.
-public class DefaultStellarExecutorTest {
-
- /**
- * {
- * "ip_src_addr": "10.0.0.1",
- * "ip_dst_addr": "10.0.0.20"
- * }
- */
- @Multiline
- private String input;
-
- private JSONObject message;
- private DefaultStellarExecutor executor;
-
- @Before
- public void setup() throws ParseException {
-
- // parse the input message
- JSONParser parser = new JSONParser();
- message = (JSONObject) parser.parse(input);
-
- // create the executor to test
- executor = new DefaultStellarExecutor();
- executor.setContext(Context.EMPTY_CONTEXT());
-
- ClasspathFunctionResolver resolver = new ClasspathFunctionResolver();
- executor.setFunctionResolver(resolver);
- }
-
- /**
- * Ensure that a value can be assigned to a variable.
- */
- @Test
- public void testAssign() {
- executor.assign("foo", "2", message);
-
- // verify
- Object var = executor.getState().get("foo");
- assertThat(var, instanceOf(Integer.class));
- assertThat(var, equalTo(2));
- }
-
- /**
- * Ensure that a variable can be resolved from a message field.
- */
- @Test
- public void testAssignWithVariableResolution() {
- executor.assign("foo", "ip_src_addr", message);
-
- // verify
- Object var = executor.getState().get("foo");
- assertThat(var, instanceOf(String.class));
- assertThat(var, equalTo("10.0.0.1"));
- }
-
- /**
- * Ensure that state is maintained correctly in the execution environment.
- */
- @Test
- public void testState() {
- executor.assign("two", "2", message);
- executor.assign("four", "4", message);
- executor.assign("sum", "two + four", message);
-
- // verify
- Object var = executor.getState().get("sum");
- assertEquals(6, var);
- }
-
- /**
- * Ensure that state is maintained correctly in the execution environment.
- */
- @Test
- public void testClearState() {
- executor.assign("two", "2", message);
- executor.clearState();
-
- // verify
- assertThat(executor.getState().containsKey("two"), equalTo(false));
- }
-
- /**
- * Ensure that a Transformation function can be executed.
- *
- * There are two sets of functions in Stellar currently. One can be executed with
- * a PredicateProcessor and the other a TransformationProcessor. The StellarExecutor
- * abstracts away that complication.
- */
- @Test
- public void testExecuteTransformation() {
- String actual = executor.execute("TO_UPPER('lowercase')", message, String.class);
- assertThat(actual, equalTo("LOWERCASE"));
- }
-
- /**
- * Ensure that a Predicate function can be executed.
- *
- * There are two sets of functions in Stellar currently. One can be executed with
- * a PredicateProcessor and the other a TransformationProcessor. The StellarExecutor
- * abstracts away that complication.
- */
- @Test
- public void testExecutePredicate() {
- boolean actual = executor.execute("IS_INTEGER(2)", message, Boolean.class);
- assertThat(actual, equalTo(true));
- }
-
- /**
- * An exception is expected if an expression results in an unexpected type.
- */
- @Test(expected = RuntimeException.class)
- public void testExecuteWithWrongType() {
- executor.execute("2 + 2", message, Boolean.class);
- }
-
- /**
- * A best effort should be made to do sensible type conversions.
- */
- @Test
- public void testExecuteWithTypeConversion() {
- executor.execute("2", message, Double.class);
- executor.execute("2", message, Float.class);
- executor.execute("2", message, Short.class);
- executor.execute("2", message, Long.class);
- }
-
- /**
- * The executor must be serializable.
- */
- @Test
- public void testSerializable() throws Exception {
-
- // serialize
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
- new ObjectOutputStream(bytes).writeObject(executor);
-
- // deserialize - success when no exceptions
- new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray())).readObject();
- }
-}