You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/27 17:49:39 UTC
metron git commit: METRON-1708 Run the Batch Profiler in Spark
(nickwallen) closes apache/metron#1161
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1699-create-batch-profiler 3bfbf018a -> c6d0721b8
METRON-1708 Run the Batch Profiler in Spark (nickwallen) closes apache/metron#1161
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c6d0721b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c6d0721b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c6d0721b
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: c6d0721b8f02752cea9f91acb29042ca9ddf0bc9
Parents: 3bfbf01
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Aug 27 13:47:28 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Mon Aug 27 13:47:28 2018 -0400
----------------------------------------------------------------------
metron-analytics/metron-profiler-spark/pom.xml | 15 ++
.../src/main/assembly/assembly.xml | 60 +++++++
.../src/main/config/batch-profiler.properties | 20 +++
.../profiler/spark/cli/BatchProfilerCLI.java | 174 +++++++++++++++++++
.../spark/cli/BatchProfilerCLIOptions.java | 144 +++++++++++++++
.../src/main/scripts/start_batch_profiler.sh | 32 ++++
.../spark/cli/BatchProfilerCLITest.java | 61 +++++++
.../src/test/resources/globals.json | 3 +
.../src/test/resources/profiles-empty.json | 5 +
.../resources/profiles-no-timestamp-field.json | 12 ++
.../src/test/resources/profiles.json | 13 ++
11 files changed, 539 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
index 93ce08a..2d5ec98 100644
--- a/metron-analytics/metron-profiler-spark/pom.xml
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -190,6 +190,21 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id> <!-- this is used for inheritance merges -->
+ <phase>package</phase> <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..02f97eb
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/assembly/assembly.xml
@@ -0,0 +1,60 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<assembly>
+ <id>archive</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/config</directory>
+ <outputDirectory>config</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0644</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts</directory>
+ <outputDirectory>bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ <outputDirectory>lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ </fileSets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
new file mode 100644
index 0000000..c651791
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/config/batch-profiler.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+spark.master=local
+spark.app.name=Batch Profiler
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
new file mode 100644
index 0000000..bdcf231
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLI.java
@@ -0,0 +1,174 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.io.IOUtils;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.spark.BatchProfiler;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.CONFIGURATION_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.GLOBALS_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.PROFILE_DEFN_FILE;
+import static org.apache.metron.profiler.spark.cli.BatchProfilerCLIOptions.parse;
+
+/**
+ * The main entry point which launches the Batch Profiler in Spark.
+ *
+ * With this class the Batch Profiler can be submitted using the following command.
+ *
+ * <pre>{@code
+ * $SPARK_HOME/bin/spark-submit \
+ * --class org.apache.metron.profiler.spark.cli.BatchProfilerCLI \
+ * --properties-file spark.properties \
+ * metron-profiler-spark-<version>.jar \
+ * --config profiler.properties \
+ * --globals global.properties \
+ * --profiles profiles.json
+ * }</pre>
+ */
+public class BatchProfilerCLI implements Serializable {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static Properties globals;
+ public static Properties profilerProps;
+ public static ProfilerConfig profiles;
+
+ public static void main(String[] args) throws IOException, org.apache.commons.cli.ParseException {
+ // parse the command line
+ CommandLine commandLine = parseCommandLine(args);
+ profilerProps = handleProfilerProperties(commandLine);
+ globals = handleGlobals(commandLine);
+ profiles = handleProfileDefinitions(commandLine);
+
+ // the batch profiler must use 'event time'
+ if(!profiles.getTimestampField().isPresent()) {
+ throw new IllegalArgumentException("The Batch Profiler must use event time. The 'timestampField' must be defined.");
+ }
+
+ // one or more profiles must be defined
+ if(profiles.getProfiles().size() == 0) {
+ throw new IllegalArgumentException("No profile definitions found.");
+ }
+
+ SparkSession spark = SparkSession
+ .builder()
+ .config(new SparkConf())
+ .getOrCreate();
+
+ BatchProfiler profiler = new BatchProfiler();
+ long count = profiler.run(spark, profilerProps, globals, profiles);
+ LOG.info("Profiler produced {} profile measurement(s)", count);
+ }
+
+ /**
+ * Load the Stellar globals from a file.
+ *
+ * @param commandLine The command line.
+ */
+ private static Properties handleGlobals(CommandLine commandLine) throws IOException {
+ Properties globals = new Properties();
+ if(GLOBALS_FILE.has(commandLine)) {
+ String globalsPath = GLOBALS_FILE.get(commandLine);
+
+ LOG.info("Loading global properties from '{}'", globalsPath);
+ globals.load(new FileInputStream(globalsPath));
+
+ LOG.info("Globals = {}", globals);
+ }
+ return globals;
+ }
+
+ /**
+ * Load the Profiler configuration from a file.
+ *
+ * @param commandLine The command line.
+ */
+ private static Properties handleProfilerProperties(CommandLine commandLine) throws IOException {
+ Properties config = new Properties();
+ if(CONFIGURATION_FILE.has(commandLine)) {
+ String propertiesPath = CONFIGURATION_FILE.get(commandLine);
+
+ LOG.info("Loading profiler properties from '{}'", propertiesPath);
+ config.load(new FileInputStream(propertiesPath));
+
+ LOG.info("Properties = {}", config.toString());
+ }
+ return config;
+ }
+
+ /**
+ * Load the profile definitions from a file.
+ *
+ * @param commandLine The command line.
+ */
+ private static ProfilerConfig handleProfileDefinitions(CommandLine commandLine) throws IOException {
+ ProfilerConfig profiles;
+ if(PROFILE_DEFN_FILE.has(commandLine)) {
+ String profilePath = PROFILE_DEFN_FILE.get(commandLine);
+
+ LOG.info("Loading profiles from '{}'", profilePath);
+ String contents = IOUtils.toString(new FileInputStream(profilePath));
+
+ profiles = ProfilerConfig.fromJSON(contents);
+ LOG.info("Loaded {} profile(s)", profiles.getProfiles().size());
+
+ } else {
+ throw new IllegalArgumentException("No profile(s) defined");
+ }
+ return profiles;
+ }
+
+ /**
+ * Parse the command line arguments submitted by the user.
+ * @param args The command line arguments to parse.
+ * @throws org.apache.commons.cli.ParseException
+ */
+ private static CommandLine parseCommandLine(String[] args) throws ParseException {
+ CommandLineParser parser = new PosixParser();
+ return parse(parser, args);
+ }
+
+ public static Properties getGlobals() {
+ return globals;
+ }
+
+ public static Properties getProfilerProps() {
+ return profilerProps;
+ }
+
+ public static ProfilerConfig getProfiles() {
+ return profiles;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
new file mode 100644
index 0000000..f5dfe12
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLIOptions.java
@@ -0,0 +1,144 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.cli;
+
+import com.google.common.base.Joiner;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import java.util.function.Supplier;
+
+/**
+ * Defines the command line interface (CLI) options accepted by the Batch
+ * Profiler.
+ */
+public enum BatchProfilerCLIOptions {
+
+ PROFILE_DEFN_FILE(() -> {
+ Option o = new Option("p", "profiles", true, "Path to a file containing profile definitions.");
+ o.setRequired(true);
+ return o;
+ }),
+
+ CONFIGURATION_FILE(() -> {
+ Option o = new Option("c", "config", true, "Path to the profiler properties file.");
+ o.setRequired(false);
+ return o;
+ }),
+
+ GLOBALS_FILE(() -> {
+ Option o = new Option("g", "globals", true, "Path to the Stellar global config file.");
+ o.setRequired(false);
+ return o;
+ }),
+
+ HELP(() -> {
+ Option o = new Option("h", "help", false, "Usage instructions.");
+ o.setRequired(false);
+ return o;
+ });
+
+ private Option option;
+
+ BatchProfilerCLIOptions(Supplier<Option> optionSupplier) {
+ this.option = optionSupplier.get();
+ }
+
+ /**
+ * Returns true, if the command line contains this option.
+ *
+ * @param cli The command line to parse.
+ */
+ public boolean has(CommandLine cli) {
+ return cli.hasOption(option.getOpt());
+ }
+
+ /**
+ * Returns the option value from the command line.
+ *
+ * @param cli The command line to parse.
+ */
+ public String get(CommandLine cli) {
+ return cli.getOptionValue(option.getOpt());
+ }
+
+ /**
+ * Returns the option value from the command line. The default value is
+ * returned if the value is not provided on the command line.
+ *
+ * @param cli The command line to parse.
+ * @param defaultValue The default value.
+ */
+ public String get(CommandLine cli, String defaultValue) {
+ return has(cli) ? cli.getOptionValue(option.getOpt()) : defaultValue;
+ }
+
+ /**
+ * Extracts the valid command line options from a given command line.
+ *
+ * @param parser The command line parser.
+ * @param args The command line to parse.
+ * @return The command line options.
+ * @throws ParseException If the command line cannot be parsed successfully.
+ */
+ public static CommandLine parse(CommandLineParser parser, String[] args) throws ParseException {
+ try {
+ CommandLine cli = parser.parse(getOptions(), args);
+ if(HELP.has(cli)) {
+ printHelp();
+ System.exit(0);
+ }
+ return cli;
+
+ } catch (ParseException e) {
+ System.err.println("invalid arguments: " + Joiner.on(' ').join(args));
+ e.printStackTrace(System.err);
+ printHelp();
+ throw e;
+ }
+ }
+
+ /**
+ * Print a help screen.
+ */
+ public static void printHelp() {
+ HelpFormatter formatter = new HelpFormatter();
+ String header = "options:";
+ String footer = "";
+ String cmd = String.format("spark-submit --class %s --properties-file [spark.properties] [jar] [options]",
+ BatchProfilerCLI.class.getCanonicalName());
+ formatter.printHelp(cmd, header, getOptions(), footer);
+ }
+
+ /**
+ * Returns all valid CLI options.
+ */
+ public static Options getOptions() {
+ Options allOptions = new Options();
+ for(BatchProfilerCLIOptions o : BatchProfilerCLIOptions.values()) {
+ allOptions.addOption(o.option);
+ }
+ return allOptions;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
new file mode 100644
index 0000000..c489af7
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/scripts/start_batch_profiler.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/${METRON_VERSION}
+PROFILER_JAR=${METRON_HOME}/lib/${project.artifactId}-${METRON_VERSION}.jar
+MAIN_CLASS=org.apache.metron.profiler.spark.cli.BatchProfilerCLI
+PROFILER_PROPS=${PROFILER_PROPS:-"${METRON_HOME}/config/batch-profiler.properties"}
+PROFILES_FILE=${PROFILES:-"${METRON_HOME}/config/zookeeper/profiler.json"}
+SPARK_HOME=${SPARK_HOME:-"/usr/hdp/current/spark2-client"}
+
+${SPARK_HOME}/bin/spark-submit \
+ --class ${MAIN_CLASS} \
+ --properties-file ${PROFILER_PROPS} \
+ ${PROFILER_JAR} \
+ --config ${PROFILER_PROPS} \
+ --profiles ${PROFILES_FILE}
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
new file mode 100644
index 0000000..c27495e
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/cli/BatchProfilerCLITest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.cli;
+
+import org.apache.commons.cli.MissingOptionException;
+import org.junit.Test;
+
+/**
+ * Tests the {@link BatchProfilerCLI} class.
+ */
+public class BatchProfilerCLITest {
+
+ /**
+ * The user must provide a Profiler configuration that defines the 'timestampField'. The
+ * Batch Profiler only operates using event time, not processing time.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void mustDefineTimestampField() throws Exception {
+ String[] args = new String[] {
+ "--profiles", "src/test/resources/profiles-no-timestamp-field.json"
+ };
+ BatchProfilerCLI.main(args);
+ }
+
+ /**
+ * The user must define the -p, --profiles option. The Profiler cannot work without profiles.
+ */
+ @Test(expected = MissingOptionException.class)
+ public void mustDefineProfilesOption() throws Exception {
+ String[] args = new String[] {};
+ BatchProfilerCLI.main(args);
+ }
+
+ /**
+ * If the profile definition contains no valid profiles, we have a problem.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void mustDefineProfiles() throws Exception {
+ String[] args = new String[] {
+ "--profiles", "src/test/resources/profiles-empty.json"
+ };
+ BatchProfilerCLI.main(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/globals.json b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
new file mode 100644
index 0000000..2e295f1
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/globals.json
@@ -0,0 +1,3 @@
+{
+ "global-key": "global-value"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
new file mode 100644
index 0000000..ce57978
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-empty.json
@@ -0,0 +1,5 @@
+{
+ "profiles": [
+ ],
+ "timestampField":"timestamp"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
new file mode 100644
index 0000000..e47a6b5
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles-no-timestamp-field.json
@@ -0,0 +1,12 @@
+{
+ "profiles": [
+ {
+ "profile": "hello-world",
+ "onlyif": "exists(ip_src_addr)",
+ "foreach": "ip_src_addr",
+ "init": { "count": "0" },
+ "update": { "count": "count + 1" },
+ "result": "count"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/c6d0721b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
new file mode 100644
index 0000000..ac77f4b
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/profiles.json
@@ -0,0 +1,13 @@
+{
+ "profiles": [
+ {
+ "profile": "hello-world",
+ "onlyif": "exists(ip_src_addr)",
+ "foreach": "ip_src_addr",
+ "init": { "count": "0" },
+ "update": { "count": "count + 1" },
+ "result": "count"
+ }
+ ],
+ "timestampField":"timestamp"
+}
\ No newline at end of file