You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2017/09/12 16:27:44 UTC
metron git commit: METRON-1146: Add ability to parse JSON string into
JSONObject for stellar closes apache/incubator-metron#727
Repository: metron
Updated Branches:
refs/heads/master 95db8b40e -> 309d3757d
METRON-1146: Add ability to parse JSON string into JSONObject for stellar closes apache/incubator-metron#727
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/309d3757
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/309d3757
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/309d3757
Branch: refs/heads/master
Commit: 309d3757df50a2dda0ac4396750e292e76926c5d
Parents: 95db8b4
Author: Anand Subramanian <an...@gmail.com>
Authored: Tue Sep 12 09:27:09 2017 -0700
Committer: cstella <ce...@gmail.com>
Committed: Tue Sep 12 09:27:09 2017 -0700
----------------------------------------------------------------------
metron-stellar/stellar-common/README.md | 26 +-
.../stellar/dsl/functions/StringFunctions.java | 115 ++++++++
.../dsl/functions/StringFunctionsTest.java | 261 +++++++++++++++++++
3 files changed, 401 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 8552138..d464d8c 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -224,6 +224,9 @@ In the core language functions, we support basic functional programming primitiv
| [ `TO_EPOCH_TIMESTAMP`](#to_epoch_timestamp) |
| [ `TO_FLOAT`](#to_float) |
| [ `TO_INTEGER`](#to_integer) |
+| [ `TO_JSON_LIST`](#to_json_List) |
+| [ `TO_JSON_MAP`](#to_json_map) |
+| [ `TO_JSON_OBJECT`](#to_json_object) |
| [ `TO_LONG`](#to_long) |
| [ `TO_LOWER`](#to_lower) |
| [ `TO_STRING`](#to_string) |
@@ -638,6 +641,27 @@ In the core language functions, we support basic functional programming primitiv
* Description: Returns a list of the encodings that are currently supported.
* Returns: A List of String
+### `TO_JSON_LIST`
+ * Description: Accepts JSON string as an input and returns a List object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `GET_FIRST( TO_JSON_LIST( '[ "foo", 2]')` would yield `foo`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed List object
+
+### `TO_JSON_MAP`
+ * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_MAP( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed Map object
+
+### `TO_JSON_OBJECT`
+ * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed.
+ For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT( '{ "foo" : 1, "bar" : 2}' )` would yield `2`
+ * Input:
+ * string - The JSON string to be parsed
+ * Returns: A parsed JSON object
+
### `LOG2`
* Description: Returns the log (base `2`) of a number.
* Input:
@@ -698,7 +722,7 @@ In the core language functions, we support basic functional programming primitiv
* Input:
* dateTime - The datetime as a long representing the milliseconds since unix epoch
* Returns: The current month (0-based).
-
+
### `MULTISET_ADD`
* Description: Adds to a multiset, which is a map associating objects to their instance counts.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
index 7f4f396..4dc4790 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
@@ -18,15 +18,20 @@
package org.apache.metron.stellar.dsl.functions;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.stellar.common.utils.JSONUtils;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
import org.apache.metron.stellar.dsl.ParseException;
import org.apache.metron.stellar.dsl.Stellar;
import org.apache.metron.stellar.common.utils.ConversionUtils;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -502,4 +507,114 @@ public class StringFunctions {
}
}
+ @Stellar(name = "TO_JSON_OBJECT"
+ , description = "Returns a JSON object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "an Object containing the parsed JSON string"
+ )
+ public static class ToJsonObject extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT <String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return JSON Object
+ try {
+ return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class);
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return new ParseException("Unable to parse JSON string");
+ }
+ }
+
+ @Stellar(name = "TO_JSON_MAP"
+ , description = "Returns a MAP object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "a MAP object containing the parsed JSON string"
+ )
+ public static class ToJsonMap extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP <JSON String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return parsed JSON Object as a HashMap
+ try {
+ return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String, Object>>(){});
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return new ParseException("Unable to parse JSON string");
+ }
+ }
+
+ @Stellar(name = "TO_JSON_LIST"
+ , description = "Returns a List object for the specified JSON string"
+ , params = {
+ "str - the JSON String to convert, may be null"
+ }
+ , returns = "a List object containing the parsed JSON string"
+ )
+ public static class ToJsonList extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> strings) {
+
+ if (strings == null || strings.size() == 0) {
+ throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST <JSON String>");
+ }
+ String var = (strings.get(0) == null) ? null : (String) strings.get(0);
+ if (var == null) {
+ return null;
+ } else if (var.length() == 0) {
+ return var;
+ } else {
+ if (!(strings.get(0) instanceof String)) {
+ throw new ParseException("Valid JSON string not supplied");
+ }
+ // Return parsed JSON Object as a List
+ try {
+ return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){});
+ } catch (JsonProcessingException ex) {
+ throw new ParseException("Valid JSON string not supplied", ex);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new ParseException("Valid JSON string not supplied", e);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
index cb33eaa..418bf2d 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
@@ -20,15 +20,18 @@ package org.apache.metron.stellar.dsl.functions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.ParseException;
import org.junit.Assert;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
@@ -484,4 +487,262 @@ public class StringFunctionsTest {
Assert.assertTrue(thrown);
}
+
+ /**
+ * TO_JSON_OBJECT StringFunction
+ */
+
+ // Input strings to be used
+ /**
+ { "foo" : 2 }
+ */
+ @Multiline
+ private String string1;
+
+ /**
+ {
+ "foo" : "abc",
+ "bar" : "def"
+ }
+ */
+ @Multiline
+ private String string2;
+
+ /**
+ [ "foo", 2 ]
+ */
+ @Multiline
+ private String string3;
+
+ /**
+ [ "foo", "bar", "car" ]
+ */
+ @Multiline
+ private String string4;
+
+ /**
+ [
+ {
+ "foo1":"abc",
+ "bar1":"def"
+ },
+ {
+ "foo2":"ghi",
+ "bar2":"jkl"
+ }
+ ]
+ */
+ @Multiline
+ private String string5;
+
+ @Test
+ public void testToJsonObject() throws Exception {
+ //JSON Object
+ Object ret1 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string1));
+ Assert.assertNotNull(ret1);
+ Assert.assertTrue (ret1 instanceof HashMap);
+
+ Object ret2 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string2));
+ Assert.assertNotNull(ret2);
+ Assert.assertTrue (ret2 instanceof HashMap);
+ Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2)));
+
+ //Simple Arrays
+ Object ret3 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string3));
+ Assert.assertNotNull(ret3);
+ Assert.assertTrue (ret3 instanceof ArrayList);
+ List<Object> result3 = (List<Object>) ret3;
+ Assert.assertEquals(2, result3.get(1));
+
+ Object ret4 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string4));
+ Assert.assertNotNull(ret4);
+ Assert.assertTrue (ret4 instanceof ArrayList);
+ List<Object> result4 = (List<Object>) ret4;
+ Assert.assertEquals("car", result4.get(2));
+
+ //JSON Array
+ Object ret5 = run( "TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string5));
+ Assert.assertNotNull(ret5);
+ Assert.assertTrue (ret5 instanceof ArrayList);
+ List<List<Object>> result5 = (List<List<Object>>) ret5;
+ HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+ Assert.assertEquals("def", results5Map1.get("bar1"));
+ HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+ Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+ // No input
+ boolean thrown = false;
+ try {
+ run("TO_JSON_OBJECT()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Invalid input
+ try {
+ run("TO_JSON_OBJECT('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Malformed JSON String
+ try {
+ run("TO_JSON_OBJECT('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+ }
+
+ @Test
+ public void testToJsonMap() throws Exception {
+ //JSON Object
+ Object ret1 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string1));
+ Assert.assertNotNull(ret1);
+ Assert.assertTrue (ret1 instanceof HashMap);
+
+ Object ret2 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string2));
+ Assert.assertNotNull(ret2);
+ Assert.assertTrue (ret2 instanceof HashMap);
+ Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2)));
+
+ //Simple Arrays
+ boolean thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+ thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string4));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue (thrown);
+
+ //JSON Array
+ thrown = false;
+ try {
+ run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string5));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+
+ // No input
+ try {
+ run("TO_JSON_MAP()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Invalid input
+ try {
+ run("TO_JSON_MAP('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+
+ // Malformed JSON String
+ try {
+ run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ thrown = false;
+ }
+
+ @Test
+ public void testToJsonList() throws Exception {
+ //Simple Arrays
+ Object ret3 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string3));
+ Assert.assertNotNull(ret3);
+ Assert.assertTrue (ret3 instanceof ArrayList);
+ List<Object> result3 = (List<Object>) ret3;
+ Assert.assertEquals(2, result3.get(1));
+
+ Object ret4 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string4));
+ Assert.assertNotNull(ret4);
+ Assert.assertTrue (ret4 instanceof ArrayList);
+ List<Object> result4 = (List<Object>) ret4;
+ Assert.assertEquals("car", result4.get(2));
+
+ //JSON Array
+ Object ret5 = run( "TO_JSON_LIST(msg)", ImmutableMap.of("msg", string5));
+ Assert.assertNotNull(ret5);
+ Assert.assertTrue (ret5 instanceof ArrayList);
+ List<List<Object>> result5 = (List<List<Object>>) ret5;
+ HashMap<String,String> results5Map1 = (HashMap) result5.get(0);
+ Assert.assertEquals("def", results5Map1.get("bar1"));
+ HashMap<String,String> results5Map2 = (HashMap) result5.get(1);
+ Assert.assertEquals("ghi", results5Map2.get("foo2"));
+
+ //JSON Object - throws exception
+ boolean thrown = false;
+ try {
+ run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string1));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue(thrown);
+
+ thrown = false;
+ try {
+ run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string2));
+ } catch (ParseException pe) {
+ thrown = true;
+ }
+ Assert.assertTrue (thrown);
+
+ // No input
+ thrown = false;
+ try {
+ run("TO_JSON_LIST()", Collections.emptyMap());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Unable to parse"));
+ }
+ Assert.assertTrue(thrown);
+
+ // Invalid input
+ thrown = false;
+ try {
+ run("TO_JSON_LIST('123, 456')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+
+ // Malformed JSON String
+ thrown = false;
+ try {
+ run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>());
+ } catch (ParseException pe) {
+ thrown = true;
+ Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ }
+ Assert.assertTrue(thrown);
+ }
+
}