You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2015/08/04 11:09:11 UTC
[1/3] tajo git commit: TAJO-1737: Implement SQL Parser rule for Map
type.
Repository: tajo
Updated Branches:
refs/heads/master 4253f1b60 -> a94936ae5
TAJO-1737: Implement SQL Parser rule for Map type.
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/08efcf27
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/08efcf27
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/08efcf27
Branch: refs/heads/master
Commit: 08efcf279140b4f5bd0e5aca4de19e792fc961a9
Parents: d8ce562
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Aug 3 20:33:17 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Aug 3 20:33:17 2015 +0900
----------------------------------------------------------------------
.../apache/tajo/algebra/ColumnDefinition.java | 9 ++-
.../org/apache/tajo/algebra/DataTypeExpr.java | 81 +++++++++++++++++---
tajo-common/src/main/proto/DataTypes.proto | 1 +
.../org/apache/tajo/engine/parser/SQLLexer.g4 | 1 +
.../org/apache/tajo/engine/parser/SQLParser.g4 | 6 ++
.../apache/tajo/engine/parser/SQLAnalyzer.java | 10 ++-
.../tajo/engine/parser/TestSQLAnalyzer.java | 18 +++++
.../TestSQLAnalyzer/create_table_maptype_1.sql | 1 +
.../TestSQLAnalyzer/create_table_maptype_2.sql | 1 +
.../TestSQLAnalyzer/create_table_maptype_3.sql | 1 +
.../create_table_maptype_1.result | 38 +++++++++
.../create_table_maptype_2.result | 52 +++++++++++++
.../create_table_maptype_3.result | 66 ++++++++++++++++
13 files changed, 272 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
index f8ea0f1..2a829e1 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java
@@ -43,8 +43,13 @@ public class ColumnDefinition extends DataTypeExpr {
}
// nested records
- if (dataType.isNestedRecordType()) {
- this.nestedRecordTypes = dataType.nestedRecordTypes;
+ if (dataType.isRecordType()) {
+ this.recordType = dataType.recordType;
+ }
+
+ // map type
+ if (dataType.isMapType()) {
+ this.mapType = dataType.mapType;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
index b280397..d63532d 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java
@@ -32,30 +32,46 @@ public class DataTypeExpr extends Expr {
@Expose @SerializedName("Scale")
Integer scale;
@Expose @SerializedName("Record")
- ColumnDefinition [] nestedRecordTypes; // not null if the type is RECORD
+ RecordType recordType; // not null if the type is RECORD
+ @Expose @SerializedName("Map")
+ MapType mapType;
public DataTypeExpr(String typeName) {
super(OpType.DataType);
this.typeName = typeName;
}
- public DataTypeExpr(ColumnDefinition [] nestedRecordTypes) {
+ public DataTypeExpr(RecordType record) {
super(OpType.DataType);
- // RECORD = 51 in DataTypes.proto
this.typeName = Type.RECORD.name();
- this.nestedRecordTypes = nestedRecordTypes;
+ this.recordType = record;
+ }
+
+ public DataTypeExpr(MapType map) {
+ super(OpType.DataType);
+ // RECORD = 51 in DataTypes.proto
+ this.typeName = Type.MAP.name();
+ this.mapType = map;
}
public String getTypeName() {
return this.typeName;
}
- public boolean isNestedRecordType() {
+ public boolean isPrimitiveType() {
+ return !this.isRecordType() && !isMapType();
+ }
+
+ public boolean isRecordType() {
return this.typeName.equals(Type.RECORD.name());
}
+ public boolean isMapType() {
+ return this.typeName.equals(Type.MAP.name());
+ }
+
public ColumnDefinition [] getNestedRecordTypes() {
- return nestedRecordTypes;
+ return recordType.schema;
}
public boolean hasLengthOrPrecision() {
@@ -84,7 +100,7 @@ public class DataTypeExpr extends Expr {
@Override
public int hashCode() {
- return Objects.hashCode(typeName, lengthOrPrecision, scale);
+ return Objects.hashCode(typeName, lengthOrPrecision, scale, recordType, mapType);
}
@Override
@@ -93,16 +109,63 @@ public class DataTypeExpr extends Expr {
return typeName.equals(another.typeName) &&
TUtil.checkEquals(lengthOrPrecision, another.lengthOrPrecision) &&
TUtil.checkEquals(scale, another.scale) &&
- TUtil.checkEquals(nestedRecordTypes, another.nestedRecordTypes);
+ TUtil.checkEquals(recordType, another.recordType) &&
+ TUtil.checkEquals(mapType, another.mapType);
}
@Override
public Object clone() throws CloneNotSupportedException {
DataTypeExpr dataType = (DataTypeExpr) super.clone();
dataType.typeName = typeName;
+ // why we copy references? because they are all immutable.
dataType.lengthOrPrecision = lengthOrPrecision;
dataType.scale = scale;
- dataType.nestedRecordTypes = nestedRecordTypes;
+ dataType.recordType = recordType;
+ dataType.mapType = mapType;
return dataType;
}
+
+ public static class RecordType implements JsonSerializable, Cloneable {
+ @Expose @SerializedName("Schema")
+ ColumnDefinition [] schema; // not null if the type is RECORD
+
+ public RecordType(ColumnDefinition [] schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public String toJson() {
+ return JsonHelper.toJson(this);
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ RecordType newRecord = (RecordType) super.clone();
+ newRecord.schema = this.schema;
+ return newRecord;
+ }
+ }
+
+ public static class MapType implements JsonSerializable, Cloneable {
+ @Expose @SerializedName("KeyType")
+ DataTypeExpr keyType;
+ @Expose @SerializedName("ValueType")
+ DataTypeExpr valueType;
+
+ public MapType(DataTypeExpr key, DataTypeExpr value) {
+ this.keyType = key;
+ this.valueType = value;
+ }
+
+ @Override
+ public String toJson() {
+ return JsonHelper.toJson(this);
+ }
+
+ public Object clone() throws CloneNotSupportedException {
+ MapType newMap = (MapType) super.clone();
+ newMap.keyType = keyType;
+ newMap.valueType = valueType;
+ return newMap;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-common/src/main/proto/DataTypes.proto
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto
index fc5ac9a..53c3d0f 100644
--- a/tajo-common/src/main/proto/DataTypes.proto
+++ b/tajo-common/src/main/proto/DataTypes.proto
@@ -60,6 +60,7 @@ enum Type {
BLOB = 45;
RECORD = 51; // nested structure type
+ MAP = 52; // map type
ANY = 61; // Any type
UDT = 62; // user-defined function
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
index d1daeb6..896f627 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLLexer.g4
@@ -258,6 +258,7 @@ LESS : L E S S;
LIST : L I S T;
LOCATION : L O C A T I O N;
+MAP : M A P;
MAX : M A X;
MAXVALUE : M A X V A L U E;
MICROSECONDS : M I C R O S E C O N D S;
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 098994d..149269e 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -282,6 +282,7 @@ nonreserved_keywords
| LESS
| LIST
| LOCATION
+ | MAP
| MAX
| MAXVALUE
| MICROSECONDS
@@ -441,6 +442,7 @@ predefined_type
| binary_type
| network_type
| record_type
+ | map_type
;
character_string_type
@@ -540,6 +542,10 @@ record_type
: RECORD table_elements
;
+map_type
+ : MAP LTH key_type=data_type COMMA value_type=data_type GTH
+ ;
+
/*
===============================================================================
6.3 <value_expression_primary>
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index ffe5d2e..465aaa8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -28,6 +28,7 @@ import org.apache.tajo.SessionVars;
import org.apache.tajo.algebra.*;
import org.apache.tajo.algebra.Aggregation.GroupType;
import org.apache.tajo.algebra.CreateIndex.IndexMethodSpec;
+import org.apache.tajo.algebra.DataTypeExpr.MapType;
import org.apache.tajo.algebra.LiteralValue.LiteralType;
import org.apache.tajo.algebra.Sort.SortSpec;
import org.apache.tajo.engine.parser.SQLParser.*;
@@ -1603,8 +1604,13 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
} else if (checkIfExist(predefined_type.record_type())) {
- ColumnDefinition [] nestedRecordDefines = getDefinitions(predefined_type.record_type().table_elements());
- typeDefinition = new DataTypeExpr(nestedRecordDefines);
+ ColumnDefinition [] nestedRecordDefine = getDefinitions(predefined_type.record_type().table_elements());
+ typeDefinition = new DataTypeExpr(new DataTypeExpr.RecordType(nestedRecordDefine));
+
+ } else if (checkIfExist(predefined_type.map_type())) {
+ Map_typeContext mapTypeContext = predefined_type.map_type();
+ typeDefinition = new DataTypeExpr(
+ new MapType(visitData_type(mapTypeContext.key_type), visitData_type(mapTypeContext.value_type)));
}
return typeDefinition;
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
index bb14aec..e9a9305 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java
@@ -801,4 +801,22 @@ public class TestSQLAnalyzer {
public void testCreateTableWithNested2() throws IOException {
assertParseResult("create_table_nested_2.sql", "create_table_nested_2.result");
}
+
+ @Test
+ public void testCreateTableWithMapType1() throws IOException {
+ // primitive key value map
+ assertParseResult("create_table_maptype_1.sql", "create_table_maptype_1.result");
+ }
+
+ @Test
+ public void testCreateTableWithMapType2() throws IOException {
+ // primitive key and record value map
+ assertParseResult("create_table_maptype_2.sql", "create_table_maptype_2.result");
+ }
+
+ @Test
+ public void testCreateTableWithMapType3() throws IOException {
+ // primitive key and nexted record value map
+ assertParseResult("create_table_maptype_3.sql", "create_table_maptype_3.result");
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql
new file mode 100644
index 0000000..4a9789c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_1.sql
@@ -0,0 +1 @@
+CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, INT8>, F FLOAT8);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql
new file mode 100644
index 0000000..9a828ce
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_2.sql
@@ -0,0 +1 @@
+CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, RECORD (name TEXT, age INT4)>, F FLOAT8);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql
new file mode 100644
index 0000000..fee6829
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSQLAnalyzer/create_table_maptype_3.sql
@@ -0,0 +1 @@
+CREATE TABLE T1 (A TEXT, B INT4, C MAP<TEXT, RECORD (name RECORD (first_name TEXT, last_name TEXT), age INT4)>, F FLOAT8);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
new file mode 100644
index 0000000..fea61b5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
@@ -0,0 +1,38 @@
+{
+ "IsExternal": false,
+ "TableName": "t1",
+ "Attributes": [
+ {
+ "ColumnDefName": "a",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "b",
+ "DataTypeName": "INT4",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "c",
+ "DataTypeName": "MAP",
+ "Map": {
+ "KeyType": {
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ "ValueType": {
+ "DataTypeName": "INT8",
+ "OpType": "DataType"
+ }
+ },
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "f",
+ "DataTypeName": "FLOAT8",
+ "OpType": "DataType"
+ }
+ ],
+ "IfNotExists": false,
+ "OpType": "CreateTable"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
new file mode 100644
index 0000000..81c0b90
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
@@ -0,0 +1,52 @@
+{
+ "IsExternal": false,
+ "TableName": "t1",
+ "Attributes": [
+ {
+ "ColumnDefName": "a",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "b",
+ "DataTypeName": "INT4",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "c",
+ "DataTypeName": "MAP",
+ "Map": {
+ "KeyType": {
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ "ValueType": {
+ "DataTypeName": "RECORD",
+ "Record": {
+ "Schema": [
+ {
+ "ColumnDefName": "name",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "age",
+ "DataTypeName": "INT4",
+ "OpType": "DataType"
+ }
+ ]
+ },
+ "OpType": "DataType"
+ }
+ },
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "f",
+ "DataTypeName": "FLOAT8",
+ "OpType": "DataType"
+ }
+ ],
+ "IfNotExists": false,
+ "OpType": "CreateTable"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/08efcf27/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
new file mode 100644
index 0000000..3259ffb
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
@@ -0,0 +1,66 @@
+{
+ "IsExternal": false,
+ "TableName": "t1",
+ "Attributes": [
+ {
+ "ColumnDefName": "a",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "b",
+ "DataTypeName": "INT4",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "c",
+ "DataTypeName": "MAP",
+ "Map": {
+ "KeyType": {
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ "ValueType": {
+ "DataTypeName": "RECORD",
+ "Record": {
+ "Schema": [
+ {
+ "ColumnDefName": "name",
+ "DataTypeName": "RECORD",
+ "Record": {
+ "Schema": [
+ {
+ "ColumnDefName": "first_name",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "last_name",
+ "DataTypeName": "TEXT",
+ "OpType": "DataType"
+ }
+ ]
+ },
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "age",
+ "DataTypeName": "INT4",
+ "OpType": "DataType"
+ }
+ ]
+ },
+ "OpType": "DataType"
+ }
+ },
+ "OpType": "DataType"
+ },
+ {
+ "ColumnDefName": "f",
+ "DataTypeName": "FLOAT8",
+ "OpType": "DataType"
+ }
+ ],
+ "IfNotExists": false,
+ "OpType": "CreateTable"
+}
\ No newline at end of file
[2/3] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo
Posted by hy...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/89892d0b
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/89892d0b
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/89892d0b
Branch: refs/heads/master
Commit: 89892d0bc7b8bc5ff6cf188c0ca151d003a6df85
Parents: 08efcf2 4253f1b
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Aug 4 18:04:21 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Aug 4 18:04:21 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../tajo/catalog/AbstractCatalogClient.java | 4 +-
.../org/apache/tajo/catalog/CatalogService.java | 5 +-
.../java/org/apache/tajo/catalog/Schema.java | 4 +-
.../exception/AmbiguousFunctionException.java | 6 +
.../exception/AmbiguousTableException.java | 9 +-
.../catalog/exception/CatalogException.java | 9 +-
.../exception/DuplicateColumnException.java | 5 +
.../exception/DuplicateDatabaseException.java | 6 +
.../exception/DuplicateIndexException.java | 5 +
.../exception/DuplicatePartitionException.java | 5 +
.../exception/DuplicateTableException.java | 5 +
.../exception/UndefinedColumnException.java | 5 +
.../exception/UndefinedDatabaseException.java | 5 +
.../exception/UndefinedFunctionException.java | 8 +-
.../exception/UndefinedPartitionException.java | 5 +
.../exception/UndefinedTableException.java | 5 +
.../exception/UndefinedTablespaceException.java | 5 +
.../org/apache/tajo/catalog/TestSchema.java | 3 +-
.../tajo/catalog/store/HiveCatalogUtil.java | 2 +-
.../org/apache/tajo/catalog/CatalogServer.java | 62 +--
.../InfoSchemaMetadataDictionary.java | 12 +-
.../tajo/catalog/store/AbstractDBStore.java | 24 +-
.../org/apache/tajo/catalog/store/MemStore.java | 24 +-
.../catalog/store/XMLCatalogSchemaManager.java | 8 +-
.../org/apache/tajo/cli/tools/TajoAdmin.java | 4 +-
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 7 +-
.../tsql/commands/ConnectDatabaseCommand.java | 3 +-
.../tajo/cli/tsql/commands/SetCommand.java | 5 +-
tajo-client/pom.xml | 4 +-
.../apache/tajo/client/CatalogAdminClient.java | 26 +-
.../tajo/client/CatalogAdminClientImpl.java | 91 ++--
.../apache/tajo/client/ClientExceptionUtil.java | 106 ++++
.../client/InvalidClientSessionException.java | 9 +-
.../org/apache/tajo/client/QueryClient.java | 41 +-
.../org/apache/tajo/client/QueryClientImpl.java | 83 ++--
.../apache/tajo/client/SessionConnection.java | 103 ++--
.../org/apache/tajo/client/TajoClientImpl.java | 46 +-
.../org/apache/tajo/client/TajoClientUtil.java | 2 +-
.../apache/tajo/client/v2/ClientDelegate.java | 41 ++
.../tajo/client/v2/ClientDelegateFactory.java | 42 ++
.../org/apache/tajo/client/v2/ClientUtil.java | 30 ++
.../apache/tajo/client/v2/FutureListener.java | 25 +
.../tajo/client/v2/LegacyClientDelegate.java | 485 +++++++++++++++++++
.../org/apache/tajo/client/v2/QueryFuture.java | 133 +++++
.../org/apache/tajo/client/v2/QueryState.java | 36 ++
.../apache/tajo/client/v2/ServiceDiscovery.java | 28 ++
.../org/apache/tajo/client/v2/TajoClient.java | 154 ++++++
.../v2/exception/ClientConnectionException.java | 28 ++
.../ClientUnableToConnectException.java | 28 ++
.../org/apache/tajo/jdbc/FetchResultSet.java | 2 +-
.../exception/AmbiguousColumnException.java | 9 +-
.../apache/tajo/exception/ErrorMessages.java | 6 +-
.../NoSuchSessionVariableException.java | 33 ++
.../apache/tajo/exception/ReturnStateUtil.java | 3 +-
.../apache/tajo/exception/SQLExceptionUtil.java | 22 +-
.../org/apache/tajo/exception/TajoError.java | 7 +
.../apache/tajo/exception/TajoException.java | 11 +
.../tajo/exception/TajoInternalError.java | 6 +
.../exception/UndefinedOperatorException.java | 5 +
.../tajo/exception/UnsupportedException.java | 5 +
tajo-core/pom.xml | 4 +-
.../org/apache/tajo/benchmark/BenchmarkSet.java | 3 +-
.../java/org/apache/tajo/benchmark/TPCH.java | 5 +-
.../engine/planner/global/GlobalPlanner.java | 107 ++--
.../java/org/apache/tajo/master/TajoMaster.java | 3 +-
.../tajo/master/exec/CreateTableExecutor.java | 15 +-
.../apache/tajo/master/exec/DDLExecutor.java | 36 +-
.../apache/tajo/master/exec/QueryExecutor.java | 3 +-
.../tajo/session/InvalidSessionException.java | 7 +-
.../tajo/webapp/QueryExecutorServlet.java | 2 +-
.../apache/tajo/LocalTajoTestingUtility.java | 6 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 10 +-
.../org/apache/tajo/client/TestTajoClient.java | 62 +--
.../tajo/client/TestTajoClientFailures.java | 16 +-
.../apache/tajo/client/v2/TestTajoClientV2.java | 236 +++++++++
.../engine/codegen/TestEvalCodeGenerator.java | 23 +-
.../apache/tajo/engine/eval/ExprTestBase.java | 31 +-
.../tajo/engine/eval/TestIntervalType.java | 5 +-
.../apache/tajo/engine/eval/TestPredicates.java | 39 +-
.../tajo/engine/eval/TestSQLDateTimeTypes.java | 11 +-
.../tajo/engine/eval/TestSQLExpression.java | 32 +-
.../function/TestConditionalExpressions.java | 11 +-
.../engine/function/TestDateTimeFunctions.java | 21 +-
.../tajo/engine/function/TestJsonFunctions.java | 5 +-
.../tajo/engine/function/TestMathFunctions.java | 49 +-
.../function/TestPatternMatchingPredicates.java | 15 +-
.../engine/function/TestPythonFunctions.java | 5 +-
.../TestStringOperatorsAndFunctions.java | 77 ++-
.../apache/tajo/engine/query/TestIndexScan.java | 3 +-
.../apache/tajo/engine/query/TestJoinQuery.java | 6 +-
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 90 ----
.../testExecuteQueryAsync.result | 7 +
.../testExecuteQueryAsyncWithListener.result | 2 +
.../testExecuteQueryType1.result | 7 +
.../testExecuteQueryType2.result | 4 +
.../testExecuteQueryType3.result | 4 +
.../org/apache/tajo/jdbc/JdbcConnection.java | 28 +-
.../org/apache/tajo/jdbc/MetaDataTuple.java | 17 +-
.../org/apache/tajo/jdbc/TajoStatement.java | 55 ---
.../tajo/plan/expr/BasicEvalNodeVisitor.java | 1 +
.../tajo/plan/expr/SimpleEvalNodeVisitor.java | 3 +-
.../plan/function/stream/CSVLineSerializer.java | 1 -
.../tajo/plan/nameresolver/NameResolver.java | 6 +-
.../plan/nameresolver/ResolverByLegacy.java | 3 +-
.../tajo/plan/nameresolver/ResolverByRels.java | 4 +-
.../nameresolver/ResolverByRelsAndSubExprs.java | 4 +-
.../nameresolver/ResolverBySubExprsAndRels.java | 4 +-
.../rewrite/rules/ProjectionPushDownRule.java | 11 +-
.../tajo/plan/serder/EvalNodeDeserializer.java | 5 +-
.../tajo/plan/verifier/VerificationState.java | 6 +
.../org/apache/tajo/storage/NullScanner.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../org/apache/tajo/storage/Tablespace.java | 3 +-
.../apache/tajo/tuple/offheap/HeapTuple.java | 2 -
.../apache/tajo/tuple/offheap/UnSafeTuple.java | 2 -
116 files changed, 2276 insertions(+), 749 deletions(-)
----------------------------------------------------------------------
[3/3] tajo git commit: TAJO-1723: INSERT INTO statement should allow
nested fields as target columns.
Posted by hy...@apache.org.
TAJO-1723: INSERT INTO statement should allow nested fields as target columns.
Closes #668
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/a94936ae
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/a94936ae
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/a94936ae
Branch: refs/heads/master
Commit: a94936ae569983bb89b4a5270d0203d7b5e591c2
Parents: 89892d0
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Aug 4 18:05:40 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Aug 4 18:05:40 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/algebra/Insert.java | 6 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 25 +---
.../org/apache/tajo/catalog/SchemaUtil.java | 40 ++++-
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 8 +-
.../engine/query/TestSelectNestedRecord.java | 40 ++++-
.../TestSelectNestedRecord/sample0_ddl.sql | 1 +
.../TestSelectNestedRecord/testSelect0.sql | 1 +
.../testInsertType1.result | 5 +
.../testInsertType2.result | 5 +
.../TestSelectNestedRecord/testSelect0.result | 5 +
.../org/apache/tajo/plan/LogicalPlanner.java | 16 +-
.../tajo/storage/json/JsonLineDeserializer.java | 39 +----
.../tajo/storage/json/JsonLineSerializer.java | 148 ++++++++++---------
15 files changed, 208 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 129c7de..0d35d8a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -457,6 +457,9 @@ Release 0.11.0 - unreleased
SUB TASKS
+ TAJO-1723: INSERT INTO statement should allow nested fields as
+ target columns. (hyunsik)
+
TAJO-1302: Support index metadata backup and restore. (jihoon)
TAJO-1484 Apply on ColPartitionStoreExec. (Contributed by Navis,
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java
index f0cd5f9..299a480 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java
@@ -32,7 +32,7 @@ public class Insert extends Expr {
@Expose @SerializedName("TableName")
private String tableName;
@Expose @SerializedName("TargetColumns")
- private String [] targetColumns;
+ private ColumnReferenceExpr [] targetColumns;
@Expose @SerializedName("StorageType")
private String storageType;
@Expose @SerializedName("Location")
@@ -70,11 +70,11 @@ public class Insert extends Expr {
return targetColumns != null;
}
- public String [] getTargetColumns() {
+ public ColumnReferenceExpr [] getTargetColumns() {
return targetColumns;
}
- public void setTargetColumns(String [] targets) {
+ public void setTargetColumns(ColumnReferenceExpr [] targets) {
this.targetColumns = targets;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
index 58b4f26..dec9a55 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -19,11 +19,11 @@
package org.apache.tajo.catalog;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import org.apache.tajo.common.TajoDataTypes.Type;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
/**
* Utility methods for nested field
@@ -65,24 +65,6 @@ public class NestedPathUtil {
return sb.toString();
}
- /**
- * Lookup the actual column corresponding to a given path.
- * We assume that a path starts with the slash '/' and it
- * does not include the root field.
- *
- * @param nestedField Nested column
- * @param path Path which starts with '/';
- * @return Column corresponding to the path
- */
- public static Column lookupPath(Column nestedField, String path) {
- Preconditions.checkArgument(path.charAt(0) == PATH_DELIMITER.charAt(0),
- "A nested field path must start with slash '/'.");
-
- // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
- // So, we should start the index from 1 instead of 0.
- return lookupPath(nestedField, path.split(PATH_DELIMITER));
- }
-
public static Column lookupPath(Column nestedField, String [] paths) {
// We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
// So, we should start the index from 1 instead of 0.
@@ -106,4 +88,5 @@ public class NestedPathUtil {
throw new NoSuchFieldError(makePath(paths));
}
}
+
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index 44973db..09a2e45 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -18,8 +18,11 @@
package org.apache.tajo.catalog;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import org.apache.tajo.util.TUtil;
+import java.util.HashMap;
import java.util.List;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -113,11 +116,44 @@ public class SchemaUtil {
return names;
}
+ public static String [] convertColumnsToPaths(Iterable<Column> columns, boolean onlyLeaves) {
+ List<String> paths = Lists.newArrayList();
+
+ for (Column c : columns) {
+ if (onlyLeaves && c.getDataType().getType() == Type.RECORD) {
+ continue;
+ }
+ paths.add(c.getSimpleName());
+ }
+
+ return paths.toArray(new String [paths.size()]);
+ }
+
+ public static ImmutableMap<String, Type> buildTypeMap(Iterable<Column> schema, String [] targetPaths) {
+
+ HashMap<String, Type> builder = new HashMap<String, Type>();
+ for (Column column : schema) {
+
+ // Keep types which only belong to projected paths
+ // For example, assume that a projected path is 'name/first_name', where name is RECORD and first_name is TEXT.
+ // In this case, we should keep two types:
+ // * name - RECORD
+ // * name/first_name TEXT
+ for (String p : targetPaths) {
+ if (p.startsWith(column.getSimpleName())) {
+ builder.put(column.getSimpleName(), column.getDataType().getType());
+ }
+ }
+ }
+
+ return ImmutableMap.copyOf(builder);
+ }
+
/**
* Column visitor interface
*/
- public static interface ColumnVisitor {
- public void visit(int depth, List<String> path, Column column);
+ public interface ColumnVisitor {
+ void visit(int depth, List<String> path, Column column);
}
/**
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 149269e..41de218 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -1599,7 +1599,7 @@ null_ordering
*/
insert_statement
- : INSERT (OVERWRITE)? INTO table_name (LEFT_PAREN column_name_list RIGHT_PAREN)? query_expression
+ : INSERT (OVERWRITE)? INTO table_name (LEFT_PAREN column_reference_list RIGHT_PAREN)? query_expression
| INSERT (OVERWRITE)? INTO LOCATION path=Character_String_Literal (USING storage_type=identifier (param_clause)?)? query_expression
;
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 465aaa8..51fe819 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1627,10 +1627,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
if (ctx.table_name() != null) {
insertExpr.setTableName(ctx.table_name().getText());
- if (ctx.column_name_list() != null) {
- String[] targetColumns = new String[ctx.column_name_list().identifier().size()];
+ if (ctx.column_reference_list() != null) {
+ ColumnReferenceExpr [] targetColumns =
+ new ColumnReferenceExpr[ctx.column_reference_list().column_reference().size()];
+
for (int i = 0; i < targetColumns.length; i++) {
- targetColumns[i] = ctx.column_name_list().identifier().get(i).getText();
+ targetColumns[i] = visitColumn_reference(ctx.column_reference_list().column_reference(i));
}
insertExpr.setTargetColumns(targetColumns);
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
index 9f8a5fd..ffd6f08 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -30,7 +30,7 @@ import static org.junit.Assert.assertEquals;
public class TestSelectNestedRecord extends QueryTestCaseBase {
@Test
- public final void testSelect1() throws Exception {
+ public final void testSelect0() throws Exception {
List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample1");
assertEquals(TUtil.newList("sample1"), tables);
@@ -40,6 +40,16 @@ public class TestSelectNestedRecord extends QueryTestCaseBase {
}
@Test
+ public final void testSelect1() throws Exception {
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample2");
+ assertEquals(TUtil.newList("sample2"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
public final void testSelect2() throws Exception {
List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
assertEquals(TUtil.newList("tweets"), tables);
@@ -68,4 +78,32 @@ public class TestSelectNestedRecord extends QueryTestCaseBase {
assertResultSet(res);
cleanupQuery(res);
}
+
+ @Test
+ public final void testInsertType1() throws Exception {
+ // all columns
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample3");
+ assertEquals(TUtil.newList("sample3"), tables);
+
+ executeString("CREATE TABLE clone (title TEXT, name RECORD (first_name TEXT, last_name TEXT)) USING JSON;").close();
+
+ executeString("INSERT INTO clone (title, name.first_name, name.last_name) SELECT title, name.first_name, name.last_name from sample3").close();
+ ResultSet res = executeString("select title, name.first_name, name.last_name from clone");
+ assertResultSet(res);
+ res.close();
+ }
+
+ @Test
+ public final void testInsertType2() throws Exception {
+ // some columns
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample4");
+ assertEquals(TUtil.newList("sample4"), tables);
+
+ executeString("CREATE TABLE clone2 (title TEXT, name RECORD (first_name TEXT, last_name TEXT)) USING JSON;").close();
+
+ executeString("INSERT INTO clone2 (title, name.last_name) SELECT title, name.last_name from sample4").close();
+ ResultSet res = executeString("select title, name.first_name, name.last_name from clone2");
+ assertResultSet(res);
+ res.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample0_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample0_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample0_ddl.sql
new file mode 100644
index 0000000..ed6aee1
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample0_ddl.sql
@@ -0,0 +1 @@
+CREATE TABLE clone (title TEXT, name RECORD (first_name TEXT, last_name TEXT)) USING JSON;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect0.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect0.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect0.sql
new file mode 100644
index 0000000..a594bcf
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect0.sql
@@ -0,0 +1 @@
+SELECT title, name.first_name, name.last_name FROM sample1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType1.result
new file mode 100644
index 0000000..ec49cd5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType1.result
@@ -0,0 +1,5 @@
+title,name/first_name,name/last_name
+-------------------------------
+Hand of the King,Eddard,Stark
+Assassin,Arya,Stark
+Dancing Master,Syrio,Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType2.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType2.result
new file mode 100644
index 0000000..a80edfe
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testInsertType2.result
@@ -0,0 +1,5 @@
+title,name/first_name,name/last_name
+-------------------------------
+Hand of the King,null,Stark
+Assassin,null,Stark
+Dancing Master,null,Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect0.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect0.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect0.result
new file mode 100644
index 0000000..ec49cd5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect0.result
@@ -0,0 +1,5 @@
+title,name/first_name,name/last_name
+-------------------------------
+Hand of the King,Eddard,Stark
+Assassin,Arya,Stark
+Dancing Master,Syrio,Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index c0727bb..b1d3e2e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -56,7 +56,6 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode;
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.storage.StorageService;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
@@ -1656,10 +1655,10 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
// See PreLogicalPlanVerifier.visitInsert.
// It guarantees that the equivalence between the numbers of target and projected columns.
- String [] targets = expr.getTargetColumns();
+ ColumnReferenceExpr [] targets = expr.getTargetColumns();
Schema targetColumns = new Schema();
for (int i = 0; i < targets.length; i++) {
- Column targetColumn = desc.getLogicalSchema().getColumn(targets[i]);
+ Column targetColumn = desc.getLogicalSchema().getColumn(targets[i].getCanonicalName().replace(".", "/"));
if (targetColumn == null) {
throw makeSyntaxError("column '" + targets[i] + "' of relation '" + desc.getName() + "' does not exist");
@@ -1702,15 +1701,20 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
if (child instanceof Projectable) {
- Projectable projectionNode = (Projectable)insertNode.getChild();
+ Projectable projectionNode = insertNode.getChild();
// Modifying projected columns by adding NULL constants
// It is because that table appender does not support target columns to be written.
List<Target> targets = TUtil.newList();
- for (int i = 0; i < tableSchema.size(); i++) {
- Column column = tableSchema.getColumn(i);
+ for (Column column : tableSchema.getAllColumns()) {
int idxInProjectionNode = targetColumns.getIndex(column);
+
+ // record type itself cannot be projected yet.
+ if (column.getDataType().getType() == TajoDataTypes.Type.RECORD) {
+ continue;
+ }
+
if (idxInProjectionNode >= 0 && idxInProjectionNode < projectionNode.getTargets().length) {
targets.add(projectionNode.getTargets()[idxInProjectionNode]);
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
index 73b7592..7ef483c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
@@ -19,6 +19,7 @@
package org.apache.tajo.storage.json;
+import com.facebook.presto.hive.shaded.com.google.common.collect.Lists;
import io.netty.buffer.ByteBuf;
import net.minidev.json.JSONObject;
import net.minidev.json.parser.JSONParser;
@@ -35,57 +36,29 @@ import org.apache.tajo.exception.UnimplementedException;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.text.TextLineDeserializer;
import org.apache.tajo.storage.text.TextLineParsingError;
-import org.apache.tajo.util.TUtil;
import java.io.IOException;
import java.util.Map;
public class JsonLineDeserializer extends TextLineDeserializer {
private JSONParser parser;
+
// Full Path -> Type
- private Map<String, Type> types;
- private String [] projectedPaths;
+ private final Map<String, Type> types;
+ private final String [] projectedPaths;
public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) {
super(schema, meta);
- projectedPaths = new String[projected.length];
- for (int i = 0; i < projected.length; i++) {
- this.projectedPaths[i] = projected[i].getSimpleName();
- }
+ projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true);
+ types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths);
}
@Override
public void init() {
- types = TUtil.newHashMap();
- for (Column column : schema.getAllColumns()) {
-
- // Keep types which only belong to projected paths
- // For example, assume that a projected path is 'name/first_name', where name is RECORD and first_name is TEXT.
- // In this case, we should keep two types:
- // * name - RECORD
- // * name/first_name TEXT
- for (String p :projectedPaths) {
- if (p.startsWith(column.getSimpleName())) {
- types.put(column.getSimpleName(), column.getDataType().getType());
- }
- }
- }
parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE | JSONParser.IGNORE_CONTROL_CHAR);
}
- private static String makePath(String [] path, int depth) {
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i <= depth; i++) {
- sb.append(path[i]);
- if (i < depth) {
- sb.append(NestedPathUtil.PATH_DELIMITER);
- }
- }
-
- return sb.toString();
- }
-
/**
*
*
http://git-wip-us.apache.org/repos/asf/tajo/blob/a94936ae/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
index 4d5d2e6..99f81a2 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
@@ -21,6 +21,7 @@ package org.apache.tajo.storage.json;
import net.minidev.json.JSONObject;
import org.apache.commons.net.util.Base64;
+import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.catalog.TableMeta;
@@ -32,87 +33,103 @@ import org.apache.tajo.storage.text.TextLineSerializer;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Map;
public class JsonLineSerializer extends TextLineSerializer {
- private Type [] types;
- private String [] simpleNames;
- private int columnNum;
+ // Full Path -> Type
+ private final Map<String, Type> types;
+ private final String [] projectedPaths;
public JsonLineSerializer(Schema schema, TableMeta meta) {
super(schema, meta);
+
+ projectedPaths = SchemaUtil.convertColumnsToPaths(schema.getAllColumns(), true);
+ types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths);
}
@Override
public void init() {
- types = SchemaUtil.toTypes(schema);
- simpleNames = SchemaUtil.toSimpleNames(schema);
- columnNum = schema.size();
+ }
+
+ private void putValue(JSONObject json,
+ String fullPath,
+ String [] pathElements,
+ int depth,
+ int fieldIndex,
+ Tuple input) throws IOException {
+ String fieldName = pathElements[depth];
+
+ if (input.isBlankOrNull(fieldIndex)) {
+ return;
+ }
+
+ switch (types.get(fullPath)) {
+
+ case BOOLEAN:
+ json.put(fieldName, input.getBool(fieldIndex));
+ break;
+
+ case INT1:
+ case INT2:
+ json.put(fieldName, input.getInt2(fieldIndex));
+ break;
+
+ case INT4:
+ json.put(fieldName, input.getInt4(fieldIndex));
+ break;
+
+ case INT8:
+ json.put(fieldName, input.getInt8(fieldIndex));
+ break;
+
+ case FLOAT4:
+ json.put(fieldName, input.getFloat4(fieldIndex));
+ break;
+
+ case FLOAT8:
+ json.put(fieldName, input.getFloat8(fieldIndex));
+ break;
+
+ case CHAR:
+ case TEXT:
+ case VARCHAR:
+ case INET4:
+ case TIMESTAMP:
+ case DATE:
+ case TIME:
+ case INTERVAL:
+ json.put(fieldName, input.getText(fieldIndex));
+ break;
+
+ case BIT:
+ case BINARY:
+ case BLOB:
+ case VARBINARY:
+ json.put(fieldName, Base64.encodeBase64String(input.getBytes(fieldIndex)));
+ break;
+
+ case NULL_TYPE:
+ break;
+
+ case RECORD:
+ JSONObject record = json.containsKey(fieldName) ? (JSONObject) json.get(fieldName) : new JSONObject();
+ json.put(fieldName, record);
+ putValue(record, fullPath + "/" + pathElements[depth + 1], pathElements, depth + 1, fieldIndex, input);
+ break;
+
+ default:
+ throw new UnimplementedException(fieldName + "(" + types.get(fullPath).name() + ") is not supported.");
+ }
}
@Override
public int serialize(OutputStream out, Tuple input) throws IOException {
JSONObject jsonObject = new JSONObject();
- for (int i = 0; i < columnNum; i++) {
- if (input.isBlankOrNull(i)) {
- continue;
- }
-
- String fieldName = simpleNames[i];
- Type type = types[i];
-
- switch (type) {
-
- case BOOLEAN:
- jsonObject.put(fieldName, input.getBool(i));
- break;
-
- case INT1:
- case INT2:
- jsonObject.put(fieldName, input.getInt2(i));
- break;
-
- case INT4:
- jsonObject.put(fieldName, input.getInt4(i));
- break;
-
- case INT8:
- jsonObject.put(fieldName, input.getInt8(i));
- break;
-
- case FLOAT4:
- jsonObject.put(fieldName, input.getFloat4(i));
- break;
-
- case FLOAT8:
- jsonObject.put(fieldName, input.getFloat8(i));
- break;
-
- case CHAR:
- case TEXT:
- case VARCHAR:
- case INET4:
- case TIMESTAMP:
- case DATE:
- case TIME:
- case INTERVAL:
- jsonObject.put(fieldName, input.getText(i));
- break;
-
- case BIT:
- case BINARY:
- case BLOB:
- case VARBINARY:
- jsonObject.put(fieldName, Base64.encodeBase64String(input.getBytes(i)));
- break;
-
- case NULL_TYPE:
- break;
-
- default:
- throw new UnimplementedException(types[i].name() + " is not supported.");
- }
+ for (int i = 0; i < projectedPaths.length; i++) {
+ String [] paths = projectedPaths[i].split(NestedPathUtil.PATH_DELIMITER);
+ putValue(jsonObject, paths[0], paths, 0, i, input);
}
String jsonStr = jsonObject.toJSONString();
@@ -123,6 +140,5 @@ public class JsonLineSerializer extends TextLineSerializer {
@Override
public void release() {
-
}
}