You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by br...@apache.org on 2015/11/18 08:12:17 UTC
[1/2] incubator-apex-malhar git commit: - MLHR-1908 Added helper
method to deserialize JSONObject into a Map.
Repository: incubator-apex-malhar
Updated Branches:
refs/heads/devel-3 d116d9406 -> 2f97e7a40
- MLHR-1908 Added helper method to deserialize JSONObject into a Map.
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/65742586
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/65742586
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/65742586
Branch: refs/heads/devel-3
Commit: 65742586b116c7faa14f75e58745c1cc437ebb4e
Parents: 7803359
Author: Timothy Farkas <ti...@datatorrent.com>
Authored: Tue Nov 17 10:29:12 2015 -0800
Committer: Timothy Farkas <ti...@datatorrent.com>
Committed: Tue Nov 17 16:39:08 2015 -0800
----------------------------------------------------------------------
.../datatorrent/lib/appdata/gpo/GPOUtils.java | 340 +++++++++++++------
.../lib/appdata/gpo/GPOUtilsTest.java | 32 +-
.../test/resources/deserializeToMapTest.json | 5 +
3 files changed, 266 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/65742586/library/src/main/java/com/datatorrent/lib/appdata/gpo/GPOUtils.java
----------------------------------------------------------------------
diff --git a/library/src/main/java/com/datatorrent/lib/appdata/gpo/GPOUtils.java b/library/src/main/java/com/datatorrent/lib/appdata/gpo/GPOUtils.java
index 239dab7..605c6b9 100644
--- a/library/src/main/java/com/datatorrent/lib/appdata/gpo/GPOUtils.java
+++ b/library/src/main/java/com/datatorrent/lib/appdata/gpo/GPOUtils.java
@@ -20,22 +20,22 @@ package com.datatorrent.lib.appdata.gpo;
import java.io.Serializable;
import java.lang.reflect.Array;
-
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.apache.commons.lang3.mutable.MutableInt;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
import com.datatorrent.lib.appdata.schemas.Fields;
import com.datatorrent.lib.appdata.schemas.FieldsDescriptor;
import com.datatorrent.lib.appdata.schemas.ResultFormatter;
@@ -123,6 +123,55 @@ public class GPOUtils
}
/**
+ * This method deserializes the fields in the given {@link FieldsDescriptor} into a map.
+ * @param fieldsDescriptor The {@link FieldsDescriptor} to fetch fields from.
+ * @param dpou The {@link JSONObject} which contains the fields whose values need to be fetched.
+ * @return A {@link Map} whose keys are field names, and whose values are possible values for those fields.
+ */
+ public static Map<String, Set<Object>> deserializeToMap(FieldsDescriptor fieldsDescriptor,
+ JSONObject dpou)
+ {
+ Map<String, Set<Object>> keyToValues = Maps.newHashMap();
+
+ for (String key : fieldsDescriptor.getFields().getFields()) {
+ if (!dpou.has(key)) {
+ throw new IllegalArgumentException("The given key " + key + " is not contained in the given JSON");
+ }
+
+ Set<Object> keyValues;
+ Object keyValue;
+
+ try {
+ keyValue = dpou.get(key);
+ } catch (JSONException ex) {
+ throw new IllegalStateException("This should never happen", ex);
+ }
+
+ if (keyValue instanceof JSONArray) {
+
+ JSONArray ja = (JSONArray) keyValue;
+ keyValues = Sets.newHashSetWithExpectedSize(ja.length());
+
+ Type type = fieldsDescriptor.getType(key);
+
+ for (int index = 0; index < ja.length(); index++) {
+ keyValues.add(getFieldFromJSON(type, ja, index));
+ }
+
+ } else if (keyValue instanceof JSONObject) {
+ throw new UnsupportedOperationException("Cannot extract objects from JSONObjects");
+ } else {
+ keyValues = Sets.newHashSetWithExpectedSize(1);
+ keyValues.add(getFieldFromJSON(fieldsDescriptor, key, dpou));
+ }
+
+ keyToValues.put(key, keyValues);
+ }
+
+ return keyToValues;
+ }
+
+ /**
* This is a helper method for deserialization of a GPOMutable from JSON. It allows you to select a field from
* a JSONObject in a json array and set it on the provided GPOMutable object. The format of the JSONArray should
* be the following:
@@ -178,178 +227,224 @@ public class GPOUtils
*/
public static void setFieldFromJSON(GPOMutable gpo, String field, JSONObject jo)
{
- Type type = gpo.getFieldDescriptor().getType(field);
-
- if(type == Type.BOOLEAN) {
- Boolean val;
-
- try {
- val = jo.getBoolean(field);
- }
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key " + field + " does not have a valid bool value.", ex);
- }
+ Object val = getFieldFromJSON(gpo.getFieldDescriptor(), field, jo);
+ gpo.setFieldGeneric(field, val);
+ }
- gpo.setFieldGeneric(field, val);
- }
- else if(type == Type.BYTE) {
- int val;
+ /**
+ * This method gets the given field from the given {@link JSONObject} and converts the field to an object
+ * of the type specified in the given {@link FieldsDescriptor}.
+ * @param fd The {@link FieldsDescriptor} describing the type of each field.
+ * @param field The field to retrieve from the given {@link JSONObject}.
+ * @param jo The {@link JSONObject} to retrieve a field from.
+ * @return The value of the given field converted to an object of the correct type.
+ */
+ public static Object getFieldFromJSON(FieldsDescriptor fd, String field, JSONObject jo)
+ {
+ Type type = fd.getType(field);
+ int intVal = 0;
+ if(numericTypeIntOrSmaller(type)) {
try {
- val = jo.getInt(field);
- }
- catch(JSONException ex) {
+ intVal = jo.getInt(field);
+ } catch (JSONException ex) {
throw new IllegalArgumentException("The key "
+ field
- + " does not have a valid byte value.", ex);
+ + " does not have a valid "
+ + type
+ + " value.", ex);
}
- if(val < (int)Byte.MIN_VALUE) {
+ if (type != Type.INTEGER && !insideRange(type, intVal)) {
throw new IllegalArgumentException("The key "
+ field
+ " has a value "
- + val
- + " which is too small to fit into a byte.");
+ + intVal
+ + " which is out of range for a "
+ + type
+ + ".");
}
+ }
- if(val > (int)Byte.MAX_VALUE) {
+ if (type == Type.BOOLEAN) {
+ try {
+ return jo.getBoolean(field);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The key " + field + " does not have a valid bool value.", ex);
+ }
+ } else if (type == Type.BYTE) {
+ return ((byte) intVal);
+ } else if (type == Type.SHORT) {
+ return ((short) intVal);
+ } else if (type == Type.INTEGER) {
+ return intVal;
+ } else if (type == Type.LONG) {
+ try {
+ return jo.getLong(field);
+ } catch (JSONException ex) {
throw new IllegalArgumentException("The key "
+ field
- + " has a value "
- + val
- + " which is too larg to fit into a byte.");
+ + " does not have a valid long value.",
+ ex);
}
-
- gpo.setField(field, (byte)val);
- }
- else if(type == Type.SHORT) {
- int val;
+ } else if (type == Type.CHAR) {
+ String val;
try {
- val = jo.getInt(field);
- }
- catch(JSONException ex) {
+ val = jo.getString(field);
+ } catch (JSONException ex) {
throw new IllegalArgumentException("The key "
+ field
- + " does not have a valid short value.",
+ + " does not have a valid character value.",
ex);
}
- if(val < (int)Short.MIN_VALUE) {
+ if (val.length() != 1) {
throw new IllegalArgumentException("The key "
+ field
+ " has a value "
+ val
- + " which is too small to fit into a short.");
+ + " that is not one character long.");
}
- if(val > (int)Short.MAX_VALUE) {
+ return val.charAt(0);
+ } else if (type == Type.STRING) {
+ try {
+ return jo.getString(field);
+ } catch (JSONException ex) {
throw new IllegalArgumentException("The key "
+ field
- + " has a value "
- + val
- + " which is too large to fit into a short.");
+ + " does not have a valid string value.",
+ ex);
}
-
- gpo.setField(field, (short)val);
- }
- else if(type == Type.INTEGER) {
- int val;
-
+ } else if (type == Type.DOUBLE) {
try {
- val = jo.getInt(field);
+ return jo.getDouble(field);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The key "
+ + field
+ + " does not have a valid double value.",
+ ex);
}
- catch(JSONException ex) {
+ } else if (type == Type.FLOAT) {
+ try {
+ return (float)jo.getDouble(field);
+ } catch (JSONException ex) {
throw new IllegalArgumentException("The key "
+ field
- + " does not have a valid int value.",
+ + " does not have a valid double value.",
ex);
}
-
- gpo.setField(field, val);
+ } else {
+ throw new UnsupportedOperationException("The type " + type + " is not supported.");
}
- else if(type == Type.LONG) {
- long val;
+ }
+
+ /**
+ * This method gets an object of the given {@link Type} from the given {@link JSONArray} at the
+ * given index.
+ * @param type The {@link Type} of the object to retrieve from the {@link JSONArray}.
+ * @param ja The {@link JSONArray} to retrieve objects from.
+ * @param index The index of the object in the {@link JSONArray} to retrieve.
+ * @return The object retrieved from the {@link JSONArray}.
+ */
+ public static Object getFieldFromJSON(Type type, JSONArray ja, int index)
+ {
+ int intVal = 0;
+ if(numericTypeIntOrSmaller(type)) {
try {
- val = jo.getLong(field);
+ intVal = ja.getInt(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ + " does not have a valid "
+ + type
+ + " value.", ex);
+ }
+
+ if (type != Type.INTEGER && !insideRange(type, intVal)) {
+ throw new IllegalArgumentException("The index "
+ + index
+ + " has a value "
+ + intVal
+ + " which is out of range for a "
+ + type
+ + ".");
}
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key "
- + field
+ }
+
+ if (type == Type.BOOLEAN) {
+ try {
+ return ja.getBoolean(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index " + index + " does not have a valid bool value.", ex);
+ }
+ } else if (type == Type.BYTE) {
+ return ((byte) intVal);
+ } else if (type == Type.SHORT) {
+ return ((short) intVal);
+ } else if (type == Type.INTEGER) {
+ return intVal;
+ } else if (type == Type.LONG) {
+ try {
+ return ja.getLong(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " does not have a valid long value.",
ex);
}
-
- gpo.setField(field, val);
- }
- else if(type == Type.CHAR) {
+ } else if (type == Type.CHAR) {
String val;
try {
- val = jo.getString(field);
- }
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key "
- + field
+ val = ja.getString(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " does not have a valid character value.",
ex);
}
- if(val.length() != 1) {
- throw new IllegalArgumentException("The key "
- + field
+ if (val.length() != 1) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " has a value "
+ val
+ " that is not one character long.");
}
- gpo.setField(field, val.charAt(0));
- }
- else if(type == Type.STRING) {
- String val;
-
+ return val.charAt(0);
+ } else if (type == Type.STRING) {
try {
- val = jo.getString(field);
- }
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key "
- + field
+ return ja.getString(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " does not have a valid string value.",
ex);
}
-
- gpo.setField(field, val);
- }
- else if(type == Type.DOUBLE) {
- Double val;
-
+ } else if (type == Type.DOUBLE) {
try {
- val = jo.getDouble(field);
- }
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key "
- + field
+ return ja.getDouble(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " does not have a valid double value.",
ex);
}
-
- gpo.setFieldGeneric(field, val);
- }
- else if(type == Type.FLOAT) {
- Float val;
-
+ } else if (type == Type.FLOAT) {
try {
- val = (float)jo.getDouble(field);
- }
- catch(JSONException ex) {
- throw new IllegalArgumentException("The key "
- + field
+ return (float) ja.getDouble(index);
+ } catch (JSONException ex) {
+ throw new IllegalArgumentException("The index "
+ + index
+ " does not have a valid double value.",
ex);
}
-
- gpo.setFieldGeneric(field, val);
+ } else {
+ throw new UnsupportedOperationException("The type " + type + " is not supported.");
}
}
@@ -2690,4 +2785,33 @@ public class GPOUtils
return values;
}
+
+ /**
+ * Determines if the given value is within the range of the specified type.
+ * @param type The type to determine the range of. Valid types can be byte or short.
+ * @param val The value to check the range of.
+ * @return True if the given int value is within the range of the specified type, false otherwise.
+ */
+ public static boolean insideRange(Type type, int val) {
+ switch(type) {
+ case BYTE: {
+ return !(val < (int)Byte.MIN_VALUE || val > (int)Byte.MAX_VALUE);
+ }
+ case SHORT: {
+ return !(val < (int)Short.MIN_VALUE || val > (int)Short.MAX_VALUE);
+ }
+ default:
+ throw new UnsupportedOperationException("This operation is not supported for the type " + type);
+ }
+ }
+
+ /**
+ * Returns true if the given type is of type byte, short, or integer.
+ * @param type The type to check.
+ * @return True if the given type is of type byte, short or integer.
+ */
+ public static boolean numericTypeIntOrSmaller(Type type)
+ {
+ return type == Type.BYTE || type == Type.SHORT || type == Type.INTEGER;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/65742586/library/src/test/java/com/datatorrent/lib/appdata/gpo/GPOUtilsTest.java
----------------------------------------------------------------------
diff --git a/library/src/test/java/com/datatorrent/lib/appdata/gpo/GPOUtilsTest.java b/library/src/test/java/com/datatorrent/lib/appdata/gpo/GPOUtilsTest.java
index e3bb1be..0e1aee3 100644
--- a/library/src/test/java/com/datatorrent/lib/appdata/gpo/GPOUtilsTest.java
+++ b/library/src/test/java/com/datatorrent/lib/appdata/gpo/GPOUtilsTest.java
@@ -20,9 +20,7 @@ package com.datatorrent.lib.appdata.gpo;
import java.util.List;
import java.util.Map;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.util.Set;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
@@ -32,7 +30,12 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.mutable.MutableInt;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
import com.datatorrent.lib.appdata.schemas.FieldsDescriptor;
+import com.datatorrent.lib.appdata.schemas.SchemaUtils;
import com.datatorrent.lib.appdata.schemas.Type;
public class GPOUtilsTest
@@ -253,6 +256,29 @@ public class GPOUtilsTest
}
@Test
+ @SuppressWarnings("AssertEqualsBetweenInconvertibleTypes")
+ public void testDeserializeToMap() throws Exception
+ {
+ Map<String, Type> fieldToType = Maps.newHashMap();
+ fieldToType.put("name", Type.STRING);
+ fieldToType.put("longvals", Type.LONG);
+ fieldToType.put("stringvals", Type.STRING);
+
+ FieldsDescriptor fd = new FieldsDescriptor(fieldToType);
+ String json = SchemaUtils.jarResourceFileToString("deserializeToMapTest.json");
+
+ Map<String, Set<Object>> resultMap = GPOUtils.deserializeToMap(fd, new JSONObject(json));
+
+ Set<Object> names = resultMap.get("name");
+ Set<Object> longvals = resultMap.get("longvals");
+ Set<Object> stringvals = resultMap.get("stringvals");
+
+ Assert.assertEquals(Sets.newHashSet("tim"), names);
+ Assert.assertEquals(Sets.newHashSet(1L, 2L, 3L), longvals);
+ Assert.assertEquals(Sets.newHashSet("a", "b", "c"), stringvals);
+ }
+
+ @Test
public void objectSerdeTest()
{
Map<String, Type> fieldToTypeKey = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/blob/65742586/library/src/test/resources/deserializeToMapTest.json
----------------------------------------------------------------------
diff --git a/library/src/test/resources/deserializeToMapTest.json b/library/src/test/resources/deserializeToMapTest.json
new file mode 100644
index 0000000..b86c747
--- /dev/null
+++ b/library/src/test/resources/deserializeToMapTest.json
@@ -0,0 +1,5 @@
+{
+ "name": "tim",
+ "longvals": [1, 2, 3],
+ "stringvals": ["a", "b", "c"]
+}
[2/2] incubator-apex-malhar git commit: Merge branch 'MLHR-1908' into
devel-3
Posted by br...@apache.org.
Merge branch 'MLHR-1908' into devel-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/commit/2f97e7a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/tree/2f97e7a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-malhar/diff/2f97e7a4
Branch: refs/heads/devel-3
Commit: 2f97e7a4088bc753c97e80974766671b321d5fa4
Parents: d116d94 6574258
Author: bright <br...@bright-mac.lan>
Authored: Tue Nov 17 23:09:51 2015 -0800
Committer: bright <br...@bright-mac.lan>
Committed: Tue Nov 17 23:09:51 2015 -0800
----------------------------------------------------------------------
.../datatorrent/lib/appdata/gpo/GPOUtils.java | 340 +++++++++++++------
.../lib/appdata/gpo/GPOUtilsTest.java | 32 +-
.../test/resources/deserializeToMapTest.json | 5 +
3 files changed, 266 insertions(+), 111 deletions(-)
----------------------------------------------------------------------