You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by jf...@apache.org on 2020/06/12 11:42:38 UTC
[incubator-iotdb] branch feature/IOTDB-742-structured-logical-types
updated: IOTDB-742 Added support for Array Types and arbitrary nesting of
types.
This is an automated email from the ASF dual-hosted git repository.
jfeinauer pushed a commit to branch feature/IOTDB-742-structured-logical-types
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature/IOTDB-742-structured-logical-types by this push:
new 436062f IOTDB-742 Added support for Array Types and arbitrary nesting of types.
436062f is described below
commit 436062f9514cd4b21029d51eeb75643dd937239a
Author: julian <j....@pragmaticminds.de>
AuthorDate: Fri Jun 12 13:42:24 2020 +0200
IOTDB-742 Added support for Array Types and arbitrary nesting of types.
---
.../structured/{MapType.java => ArrayType.java} | 50 ++-------
.../iotdb/db/metadata/structured/MapType.java | 2 +-
.../db/metadata/structured/PrimitiveType.java | 2 +-
.../iotdb/db/metadata/structured/SManager.java | 73 +++++++++----
.../db/metadata/structured/StructuredType.java | 2 +-
.../db/integration/IoTDBInsertStructuredIT.java | 117 ++++++++++++++++++++-
.../iotdb/db/metadata/structured/SManagerTest.java | 66 ++++++++++++
7 files changed, 244 insertions(+), 68 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java b/server/src/main/java/org/apache/iotdb/db/metadata/structured/ArrayType.java
similarity index 60%
copy from server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java
copy to server/src/main/java/org/apache/iotdb/db/metadata/structured/ArrayType.java
index 00bb810..759c9dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/structured/ArrayType.java
@@ -8,21 +8,14 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
import java.util.Set;
-public class MapType implements StructuredType {
+public class ArrayType implements StructuredType {
- private final Map<String, StructuredType> children;
+ private final StructuredType arrayType;
- public MapType() {
- this(new HashMap<>());
- }
-
- public MapType(Map<String, StructuredType> children) {
- this.children = children;
+ public ArrayType(StructuredType arrayType) {
+ this.arrayType = arrayType;
}
@Override
@@ -47,50 +40,27 @@ public class MapType implements StructuredType {
@Override
public boolean isArray() {
- return false;
+ return true;
}
@Override
- public StructuredType getItem(int index) {
- throw new UnsupportedOperationException();
+ public StructuredType getArrayType() {
+ return this.arrayType;
}
@Override
public boolean isMap() {
- return true;
+ return false;
}
@Override
public Set<String> getKeySet() {
- return this.children.keySet();
+ throw new UnsupportedOperationException();
}
@Override
public StructuredType getChild(String name) {
- return this.children.get(name);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- MapType mapType = (MapType) o;
- return Objects.equals(children, mapType.children);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(children);
+ throw new UnsupportedOperationException();
}
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("{\n");
- for (Map.Entry<String, StructuredType> entry : this.children.entrySet()) {
- builder.append("\"" + entry.getKey() + "\": " + entry.getValue().toString() + ",\n");
- }
- builder.append("}\n");
- return builder.toString();
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java b/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java
index 00bb810..a1aa092 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/structured/MapType.java
@@ -51,7 +51,7 @@ public class MapType implements StructuredType {
}
@Override
- public StructuredType getItem(int index) {
+ public StructuredType getArrayType() {
throw new UnsupportedOperationException();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/structured/PrimitiveType.java b/server/src/main/java/org/apache/iotdb/db/metadata/structured/PrimitiveType.java
index a78ac67..c069728 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/structured/PrimitiveType.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/structured/PrimitiveType.java
@@ -49,7 +49,7 @@ public class PrimitiveType implements StructuredType {
}
@Override
- public StructuredType getItem(int index) {
+ public StructuredType getArrayType() {
throw new UnsupportedOperationException();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/structured/SManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/structured/SManager.java
index 668dc7c..497eada 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/structured/SManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/structured/SManager.java
@@ -5,8 +5,8 @@
package org.apache.iotdb.db.metadata.structured;
import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
-import com.sun.tools.corba.se.idl.InvalidArgument;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.NotImplementedException;
@@ -63,10 +63,11 @@ public class SManager {
Object[] values = plan.getValues();
for (int i = 0; i < values.length; i++) {
Object value = values[i];
+ String measurement = plan.getMeasurements()[i];
if (value instanceof String && ((String) value).contains("::")) {
String str = StringUtils.strip(((String) value), "\"");
str = StringEscapeUtils.unescapeJava(str);
- logger.info("Contains structured type for measurement '{}', rewriting...", plan.getMeasurements()[i]);
+ logger.info("Contains structured type for measurement '{}', rewriting...", measurement);
// Now first parse the thing
String jsonString = str.substring(0, str.indexOf("::"));
String structName = str.substring(str.indexOf("::") + 2);
@@ -83,29 +84,17 @@ public class SManager {
JSONObject map = JSON.parseObject(jsonString);
- for (String key : type.getKeySet()) {
- if (!(type.getChild(key) instanceof PrimitiveType)) {
- throw new NotImplementedException("No nesting is supported currently!");
- }
- // Fetch the value
- if (!map.containsKey(key)) {
- throw new IllegalArgumentException("Value String misses the requested field '" + key + "'");
- }
- measurements.add(plan.getMeasurements()[i] + "." + key);
- TSDataType primitiveType = type.getChild(key).getPrimitiveType();
- types.add(primitiveType);
- try {
- newValues.add(CommonUtils.parseValue(primitiveType, map.getString(key)));
- } catch (QueryProcessException e) {
- throw new IllegalArgumentException("Error parsing given structured Object", e);
- }
- }
+ visitMap(measurement, type, map, measurements, types, newValues);
+ } else if (type.isArray()) {
+ JSONArray jsonArray = JSON.parseArray(jsonString);
+
+ visitArray(measurement, type.getArrayType(), jsonArray, measurements, types, newValues);
} else {
throw new NotImplementedException("Only type Map is supported currently!");
}
} else {
// Do inference here
- measurements.add(plan.getMeasurements()[i]);
+ measurements.add(measurement);
types.add(plan.getTypes()[i]);
newValues.add(value);
}
@@ -113,5 +102,49 @@ public class SManager {
return new InsertPlan(plan.getDeviceId(), plan.getTime(), measurements.toArray(new String[0]), types.toArray(new TSDataType[0]), newValues.toArray(new Object[0]));
}
+ private void visitArray(String prefix, StructuredType type, JSONArray jsonArray, List<String> measurements, List<TSDataType> types, List<Object> newValues) {
+ for (int i1 = 0; i1 < jsonArray.size(); i1++) {
+ if (type instanceof PrimitiveType) {
+ visitPrimitive(prefix + "[" + i1 + "]", type, jsonArray.getString(i1), measurements, types, newValues);
+ } else if (type instanceof MapType) {
+ visitMap(prefix + "[" + i1 + "]", type, jsonArray.getJSONObject(i1), measurements, types, newValues);
+ } else if (type instanceof ArrayType) {
+ visitArray(prefix + "[" + i1 + "]", type.getArrayType(), jsonArray.getJSONArray(i1), measurements, types, newValues);
+ } else {
+ throw new NotImplementedException("Not supported type in array");
+ }
+ }
+ }
+
+ private void visitPrimitive(String prefix, StructuredType type, String valueAsString, List<String> measurements, List<TSDataType> types, List<Object> newValues) {
+ measurements.add(prefix);
+ TSDataType primitiveType = type.getPrimitiveType();
+ types.add(primitiveType);
+ try {
+ newValues.add(CommonUtils.parseValue(primitiveType, valueAsString));
+ } catch (QueryProcessException e) {
+ throw new IllegalArgumentException("Unable to parse", e);
+ }
+ }
+
+ private void visitMap(String prefix, StructuredType type, JSONObject map, List<String> measurements, List<TSDataType> types, List<Object> newValues) {
+ for (String key : type.getKeySet()) {
+ StructuredType child = type.getChild(key);
+ if (child instanceof PrimitiveType) {
+ // Fetch the value
+ if (!map.containsKey(key)) {
+ throw new IllegalArgumentException("Value String misses the requested field '" + key + "'");
+ }
+ visitPrimitive(prefix + "." + key, child, map.getString(key), measurements, types, newValues);
+ } else if (child instanceof MapType) {
+ visitMap(prefix + "." + key, child, map.getJSONObject(key), measurements, types, newValues);
+ } else if (child instanceof ArrayType) {
+ visitArray(prefix + "." + key, child.getArrayType(), map.getJSONArray(key), measurements, types, newValues);
+ } else {
+ throw new UnsupportedOperationException("Not supporting nested...");
+ }
+ }
+ }
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/structured/StructuredType.java b/server/src/main/java/org/apache/iotdb/db/metadata/structured/StructuredType.java
index 73900a8..68a7579 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/structured/StructuredType.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/structured/StructuredType.java
@@ -23,7 +23,7 @@ public interface StructuredType {
boolean isArray();
- StructuredType getItem(int index);
+ StructuredType getArrayType();
boolean isMap();
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertStructuredIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertStructuredIT.java
index 8f8cd44..9d69a38 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertStructuredIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBInsertStructuredIT.java
@@ -18,10 +18,8 @@
*/
package org.apache.iotdb.db.integration;
-import org.apache.iotdb.db.metadata.structured.MapType;
-import org.apache.iotdb.db.metadata.structured.PrimitiveType;
-import org.apache.iotdb.db.metadata.structured.SManager;
-import org.apache.iotdb.db.metadata.structured.StructuredType;
+import org.apache.iotdb.db.metadata.structured.*;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -34,6 +32,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.sql.*;
+import java.util.ArrayList;
import java.util.HashMap;
import static org.junit.Assert.assertThat;
@@ -73,7 +72,7 @@ public class IoTDBInsertStructuredIT implements WithAssertions {
}
@Test
- public void showTimeseries() throws ClassNotFoundException {
+ public void insertMap() throws ClassNotFoundException {
String[] retArray = new String[]{
"root.sg1.d1.\"coordinates.lat\",null,root.sg1,DOUBLE,GORILLA,SNAPPY,",
"root.sg1.d1.\"coordinates.long\",null,root.sg1,DOUBLE,GORILLA,SNAPPY,",
@@ -121,6 +120,114 @@ public class IoTDBInsertStructuredIT implements WithAssertions {
}
@Test
+ public void insertArray() throws ClassNotFoundException {
+ String[] retArray = new String[]{
+ "root.sg1.d1.coordinates[0],null,root.sg1,INT32,RLE,SNAPPY,",
+ "root.sg1.d1.coordinates[1],null,root.sg1,INT32,RLE,SNAPPY,",
+ };
+
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ // Prepare type
+ SManager.getInstance().register("two_int", new ArrayType(new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)));
+
+ // Insert value
+ statement.execute("INSERT INTO root.sg1.d1 (timestamp, coordinates) VALUES (NOW(), \"[1,2]::two_int\")");
+
+ boolean hasResultSet = statement.execute(
+ "SHOW TIMESERIES");
+ Assert.assertTrue(hasResultSet);
+
+ try (ResultSet resultSet = statement.getResultSet()) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ StringBuilder header = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ header.append(resultSetMetaData.getColumnName(i)).append(",");
+ }
+ Assert.assertEquals("timeseries,alias,storage group,dataType,encoding,compression,", header.toString());
+ Assert.assertEquals(Types.VARCHAR, resultSetMetaData.getColumnType(1));
+
+ int cnt = 0;
+ while (resultSet.next()) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ builder.append(resultSet.getString(i)).append(",");
+ }
+ Assert.assertEquals(retArray[cnt], builder.toString());
+ cnt++;
+ }
+ Assert.assertEquals(retArray.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void insertComplex() throws ClassNotFoundException {
+ String[] retArray = new String[]{
+ "root.sg1.d1.\"coordinates.max_speed\",null,root.sg1,DOUBLE,GORILLA,SNAPPY,",
+ "root.sg1.d1.\"coordinates.look.color\",null,root.sg1,TEXT,PLAIN,SNAPPY,",
+ "root.sg1.d1.\"coordinates.look.clean\",null,root.sg1,BOOLEAN,RLE,SNAPPY,",
+ "root.sg1.d1.\"coordinates.drivers[0]\",null,root.sg1,TEXT,PLAIN,SNAPPY,",
+ "root.sg1.d1.\"coordinates.drivers[1]\",null,root.sg1,TEXT,PLAIN,SNAPPY,",
+ };
+
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ // Prepare type
+ HashMap<String, StructuredType> lookMap = new HashMap<>();
+ lookMap.put("clean", new PrimitiveType(TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+ lookMap.put("color", new PrimitiveType(TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED));
+
+ HashMap<String, StructuredType> carMap = new HashMap<>();
+ carMap.put("max_speed", new PrimitiveType(TSDataType.DOUBLE, TSEncoding.GORILLA, CompressionType.GZIP));
+ carMap.put("look", new MapType(lookMap));
+ carMap.put("drivers", new ArrayType(new PrimitiveType(TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)));
+
+ SManager.getInstance().register("car", new MapType(carMap));
+
+ // Insert value
+ statement.execute("INSERT INTO root.sg1.d1 (timestamp, coordinates) VALUES (NOW(), \"{\\\"max_speed\\\": 160.0, \\\"look\\\":{\\\"clean\\\":true, \\\"color\\\": \\\"blue\\\"}, \\\"drivers\\\":[\\\"julian\\\", \\\"xiangdong\\\"]}::car\")");
+
+ boolean hasResultSet = statement.execute(
+ "SHOW TIMESERIES");
+ Assert.assertTrue(hasResultSet);
+
+ try (ResultSet resultSet = statement.getResultSet()) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ StringBuilder header = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ header.append(resultSetMetaData.getColumnName(i)).append(",");
+ }
+ Assert.assertEquals("timeseries,alias,storage group,dataType,encoding,compression,", header.toString());
+ Assert.assertEquals(Types.VARCHAR, resultSetMetaData.getColumnType(1));
+
+ int cnt = 0;
+ while (resultSet.next()) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ builder.append(resultSet.getString(i)).append(",");
+ }
+ Assert.assertEquals(retArray[cnt], builder.toString());
+ cnt++;
+ }
+ Assert.assertEquals(retArray.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
public void insertMissingField_fails() throws ClassNotFoundException {
String[] retArray = new String[]{
"root.sg1.d1.\"coordinates.lat\",null,root.sg1,DOUBLE,GORILLA,SNAPPY,",
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/structured/SManagerTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/structured/SManagerTest.java
index ec1747a..01a6826 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/structured/SManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/structured/SManagerTest.java
@@ -10,6 +10,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.junit.Test;
+import java.util.Collections;
import java.util.HashMap;
import static org.junit.Assert.*;
@@ -29,6 +30,71 @@ public class SManagerTest {
assertArrayEquals(new Object[]{ 40.0, 20.0 }, plan.getValues());
}
+ @Test
+ public void arrayType() {
+ SManager sManager = new SManager();
+
+ sManager.register("two_int", new ArrayType(new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)));
+
+ InsertPlan plan = sManager.translate(new InsertPlan("root.sg1.d1", 0, new String[]{"two"}, new String[]{"[1,2]::two_int"}));
+
+ assertArrayEquals(new String[]{ "two[0]", "two[1]" }, plan.getMeasurements());
+ assertArrayEquals(new TSDataType[]{ TSDataType.INT32, TSDataType.INT32 }, plan.getTypes());
+ assertArrayEquals(new Object[]{ 1, 2 }, plan.getValues());
+ }
+
+ @Test
+ public void nestedStructure() {
+ SManager sManager = new SManager();
+
+ sManager.register("nested", new MapType(Collections.singletonMap("a", new MapType(Collections.singletonMap("b", new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))))));
+
+ InsertPlan plan = sManager.translate(new InsertPlan("root.sg1.d1", 0, new String[]{"two"}, new String[]{"{\"a\":{\"b\":1}}::nested"}));
+
+ assertArrayEquals(new String[]{ "two.a.b" }, plan.getMeasurements());
+ assertArrayEquals(new TSDataType[]{ TSDataType.INT32}, plan.getTypes());
+ assertArrayEquals(new Object[]{ 1}, plan.getValues());
+ }
+
+ @Test
+ public void nestedArray() {
+ SManager sManager = new SManager();
+
+ sManager.register("nested", new MapType(Collections.singletonMap("a", new ArrayType(new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)))));
+
+ InsertPlan plan = sManager.translate(new InsertPlan("root.sg1.d1", 0, new String[]{"two"}, new String[]{"{\"a\":[1]}::nested"}));
+
+ assertArrayEquals(new String[]{ "two.a[0]" }, plan.getMeasurements());
+ assertArrayEquals(new TSDataType[]{ TSDataType.INT32}, plan.getTypes());
+ assertArrayEquals(new Object[]{ 1 }, plan.getValues());
+ }
+
+ @Test
+ public void arrayOfStructType() {
+ SManager sManager = new SManager();
+
+ sManager.register("struct_array", new ArrayType(new MapType(Collections.singletonMap("a", new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)))));
+
+ InsertPlan plan = sManager.translate(new InsertPlan("root.sg1.d1", 0, new String[]{"two"}, new String[]{"[{\"a\":1}]::struct_array"}));
+
+ assertArrayEquals(new String[]{ "two[0].a" }, plan.getMeasurements());
+ assertArrayEquals(new TSDataType[]{ TSDataType.INT32}, plan.getTypes());
+ assertArrayEquals(new Object[]{ 1 }, plan.getValues());
+ }
+
+ @Test
+ public void arrayOfArray() {
+ SManager sManager = new SManager();
+
+ sManager.register("array_array", new ArrayType(new ArrayType(new PrimitiveType(TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY))));
+
+ InsertPlan plan = sManager.translate(new InsertPlan("root.sg1.d1", 0, new String[]{"two"}, new String[]{"[[1]]::array_array"}));
+
+ assertArrayEquals(new String[]{ "two[0][0]" }, plan.getMeasurements());
+ assertArrayEquals(new TSDataType[]{ TSDataType.INT32}, plan.getTypes());
+ assertArrayEquals(new Object[]{ 1 }, plan.getValues());
+ }
+
private StructuredType gpsType() {
HashMap<String, StructuredType> children = new HashMap<>();
children.put("lat", new PrimitiveType(TSDataType.DOUBLE, TSEncoding.GORILLA, CompressionType.SNAPPY));