You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/09/23 03:12:18 UTC
[1/3] tajo git commit: TAJO-1832: Well support for self-describing
data formats.
Repository: tajo
Updated Branches:
refs/heads/master 4a9628858 -> 5a1558613
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
index d90f1b0..608fa4c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java
@@ -510,13 +510,15 @@ public class LogicalNodeDeserializer {
createTable.setPartitionMethod(new PartitionMethodDesc(storeTableNodeSpec.getPartitionMethod()));
}
- createTable.setTableSchema(convertSchema(storeTableNodeSpec.getTableSchema()));
+ if (storeTableNodeSpec.hasTableSchema()) {
+ createTable.setTableSchema(convertSchema(storeTableNodeSpec.getTableSchema()));
+ }
if (createTableNodeSpec.hasTablespaceName()) {
createTable.setTableSpaceName(createTableNodeSpec.getTablespaceName());
}
createTable.setExternal(createTableNodeSpec.getExternal());
- if (createTableNodeSpec.getExternal() && storeTableNodeSpec.hasUri()) {
+ if (storeTableNodeSpec.hasUri()) {
createTable.setUri(URI.create(storeTableNodeSpec.getUri()));
}
createTable.setIfNotExists(createTableNodeSpec.getIfNotExists());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
index 7e61f30..a0f1fcc 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java
@@ -711,7 +711,10 @@ public class LogicalNodeSerializer extends BasicLogicalPlanVisitor<LogicalNodeSe
if (node.hasUri()) {
storeTableBuilder.setUri(node.getUri().toString());
}
- storeTableBuilder.setTableSchema(node.getTableSchema().getProto());
+
+ if (node.hasTableSchema()) {
+ storeTableBuilder.setTableSchema(node.getTableSchema().getProto());
+ }
if (node.hasPartition()) {
storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java
index 461ffd6..57a7644 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java
@@ -34,8 +34,8 @@ import java.util.Stack;
public class ExprFinder extends SimpleAlgebraVisitor<ExprFinder.Context, Object> {
- static class Context {
- List<Expr> set = TUtil.newList();
+ static class Context<T> {
+ List<T> set = TUtil.newList();
OpType targetType;
Context(OpType type) {
@@ -44,18 +44,18 @@ public class ExprFinder extends SimpleAlgebraVisitor<ExprFinder.Context, Object>
}
public static <T extends Expr> Set<T> finds(Expr expr, OpType type) {
- return (Set<T>) new HashSet<Expr>(findsInOrder(expr, type));
+ return (Set<T>) new HashSet<>(findsInOrder(expr, type));
}
public static <T extends Expr> List<T> findsInOrder(Expr expr, OpType type) {
- Context context = new Context(type);
+ Context<T> context = new Context<>(type);
ExprFinder finder = new ExprFinder();
try {
finder.visit(context, new Stack<Expr>(), expr);
} catch (TajoException e) {
throw new TajoInternalError(e);
}
- return (List<T>) context.set;
+ return context.set;
}
public Object visit(Context ctx, Stack<Expr> stack, Expr expr) throws TajoException {
@@ -71,7 +71,7 @@ public class ExprFinder extends SimpleAlgebraVisitor<ExprFinder.Context, Object>
super.visit(ctx, stack, expr);
}
- if (ctx.targetType == expr.getType()) {
+ if (expr != null && ctx.targetType == expr.getType()) {
ctx.set.add(expr);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 0210865..7ba31bc 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -992,4 +992,13 @@ public class PlannerUtil {
finder.visit(null, qual, new Stack<EvalNode>());
return finder.getEvalNodes();
}
+
+ public static boolean hasAsterisk(NamedExpr [] namedExprs) {
+ for (NamedExpr eachTarget : namedExprs) {
+ if (eachTarget.getExpr().getType() == OpType.Asterisk) {
+ return true;
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index f40548c..2c7feac 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -19,6 +19,7 @@
package org.apache.tajo.plan.verifier;
import com.google.common.base.Preconditions;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.OverridableConf;
import org.apache.tajo.SessionVars;
import org.apache.tajo.TajoConstants;
@@ -249,6 +250,24 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
if (expr.hasTableElements()) {
assertRelationSchema(context, expr);
+ } else {
+ if (expr.getStorageType() != null) {
+ if (expr.hasSelfDescSchema()) {
+ // TODO: support other types like Parquet and ORC.
+ if (!expr.getStorageType().equalsIgnoreCase(BuiltinStorages.JSON)) {
+ if (expr.getStorageType().equalsIgnoreCase(BuiltinStorages.PARQUET) ||
+ expr.getStorageType().equalsIgnoreCase(BuiltinStorages.ORC)) {
+ throw new NotImplementedException(expr.getStorageType());
+ } else {
+ throw new UnsupportedException(expr.getStorageType());
+ }
+ }
+ } else {
+ if (expr.getLikeParentTableName() == null && expr.getSubQuery() == null) {
+ throw new TajoInternalError(expr.getTableName() + " does not have pre-defined or self-describing schema");
+ }
+ }
+ }
}
if (expr.hasStorageType()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java
index ab1f74d..6d43a69 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java
@@ -30,6 +30,9 @@ import java.util.Stack;
public abstract class SimpleAlgebraVisitor<CONTEXT, RESULT> extends BaseAlgebraVisitor<CONTEXT, RESULT> {
public RESULT visit(CONTEXT ctx, Stack<Expr> stack, Expr expr) throws TajoException {
+ if (expr == null) {
+ return null;
+ }
RESULT result = null;
if (expr instanceof UnaryOperator) {
preHook(ctx, stack, expr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/proto/Plan.proto
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto
index 7815cac..8a8ecb1 100644
--- a/tajo-plan/src/main/proto/Plan.proto
+++ b/tajo-plan/src/main/proto/Plan.proto
@@ -244,7 +244,7 @@ message PersistentStoreNode {
message StoreTableNodeSpec { // required PersistentStoreNode
optional string tableName = 1; // 'INSERT INTO LOCATION' does not require 'table name'.
optional string uri = 2;
- required SchemaProto table_schema = 3;
+ optional SchemaProto table_schema = 3; // optional for self-describing formats
optional PartitionMethodProto partitionMethod = 4;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4
index 87b897d..c125352 100644
--- a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4
+++ b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4
@@ -117,6 +117,7 @@ create_table_statement
table_elements
: LEFT_PAREN field_element (COMMA field_element)* RIGHT_PAREN
+ | LEFT_PAREN asterisk RIGHT_PAREN
;
field_element
@@ -1312,7 +1313,11 @@ derived_column
;
qualified_asterisk
- : (tb_name=identifier DOT)? MULTIPLY
+ : (tb_name=identifier DOT)? asterisk
+ ;
+
+asterisk
+ : MULTIPLY
;
set_qualifier
[2/3] tajo git commit: TAJO-1832: Well support for self-describing
data formats.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testInSubquery.1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testInSubquery.1.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testInSubquery.1.result
new file mode 100644
index 0000000..c700a7d
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testInSubquery.1.result
@@ -0,0 +1,4 @@
+l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment
+-------------------------------
+1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold
+1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.result
new file mode 100644
index 0000000..9e68a19
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.result
@@ -0,0 +1,3 @@
+?cast
+-------------------------------
+0
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.result
new file mode 100644
index 0000000..1754d7f
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.result
@@ -0,0 +1,4 @@
+?cast,l_linenumber,l_comment
+-------------------------------
+0,1,egular courts above the
+0,2,ly final dependencies: slyly bold
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.result
new file mode 100644
index 0000000..ef95de3
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.result
@@ -0,0 +1,3 @@
+?cast,l_linenumber,l_comment
+-------------------------------
+0,2,ly final dependencies: slyly bold
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.result
new file mode 100644
index 0000000..e186cad
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.result
@@ -0,0 +1,5 @@
+?cast,l_linenumber,l_comment
+-------------------------------
+0,2,ly final dependencies: slyly bold
+0,2,ly final dependencies: slyly bold
+0,2,ly final dependencies: slyly bold
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect.result
new file mode 100644
index 0000000..9dd2ede
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect.result
@@ -0,0 +1,3 @@
+glossary/title,glossary/GlossDiv/title,glossary/GlossDiv/null_expected,glossary/GlossDiv/GlossList/GlossEntry/SortAs,glossary/GlossDiv/GlossList/GlossEntry/Abbrev
+-------------------------------
+example glossary,S,null,SGML,ISO 8879:1986
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect2.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect2.result
new file mode 100644
index 0000000..ae223f2
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSelect2.result
@@ -0,0 +1,3 @@
+glossary/title,glossary/GlossDiv/title,glossary/GlossDiv/null_expected,glossary/GlossDiv/GlossList/GlossEntry/SortAs
+-------------------------------
+example glossary,S,null,SGML
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSort.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSort.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSort.result
new file mode 100644
index 0000000..ba68534
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testSort.result
@@ -0,0 +1,6 @@
+created_at,id,user/profile_sidebar_fill_color
+-------------------------------
+Fri Sep 21 22:51:18 +0000 2012,249279667666817024,BFAC83
+Fri Sep 21 23:30:20 +0000 2012,249289491129438208,99CC33
+Fri Sep 21 23:40:54 +0000 2012,249292149810667520,DDFFCC
+Mon Sep 24 03:35:21 +0000 2012,250075927172759552,DDEEF6
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/case_sensitivity1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/case_sensitivity1.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/case_sensitivity1.result
index 94ba2cd..fd64965 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/case_sensitivity1.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/case_sensitivity1.result
@@ -9,5 +9,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_like_1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_like_1.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_like_1.result
index c6fbaf2..4f1033f 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_like_1.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_like_1.result
@@ -3,5 +3,6 @@
"TableName": "new_table",
"IfNotExists": false,
"LikeParentTable": "orig_name",
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
index fea61b5..7a7a1a6 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_1.result
@@ -34,5 +34,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
index 81c0b90..e1923fd 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_2.result
@@ -48,5 +48,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
index 3259ffb..e3c8047 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_maptype_3.result
@@ -62,5 +62,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_1.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_1.result
index 55f79da..200dcd8 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_1.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_1.result
@@ -38,5 +38,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_2.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_2.result
index 8a95d2c..844621f 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_2.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_nested_2.result
@@ -57,5 +57,6 @@
}
],
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_column.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_column.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_column.result
index 22dc423..f74465e 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_column.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_column.result
@@ -35,5 +35,6 @@
"PartitionType": "COLUMN"
},
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_1.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_1.result
index f4f3716..897d753 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_1.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_1.result
@@ -28,5 +28,6 @@
"PartitionType": "HASH"
},
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_2.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_2.result
index 030a437..d9c0e4d 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_2.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_hash_2.result
@@ -34,5 +34,6 @@
"PartitionType": "HASH"
},
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_list.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_list.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_list.result
index 994f408..3a86d96 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_list.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_list.result
@@ -61,5 +61,6 @@
"PartitionType": "LIST"
},
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_range.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_range.result b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_range.result
index 9ebb8ab..4112f66 100644
--- a/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_range.result
+++ b/tajo-core-tests/src/test/resources/results/TestSQLAnalyzer/create_table_partition_by_range.result
@@ -51,5 +51,6 @@
"PartitionType": "RANGE"
},
"IfNotExists": false,
+ "HasSelfDescSchema": false,
"OpType": "CreateTable"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestUnionQuery/testUnionAndFilter.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestUnionQuery/testUnionAndFilter.result b/tajo-core-tests/src/test/resources/results/TestUnionQuery/testUnionAndFilter.result
index 8eb747b..7ef21a8 100644
--- a/tajo-core-tests/src/test/resources/results/TestUnionQuery/testUnionAndFilter.result
+++ b/tajo-core-tests/src/test/resources/results/TestUnionQuery/testUnionAndFilter.result
@@ -1,7 +1,7 @@
c_custkey,ret
-------------------------------
-4,2866.83
1,711.56
-5,794.47
-3,7498.12
2,121.65
+3,7498.12
+4,2866.83
+5,794.47
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index cd43add..227af0b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -870,7 +870,11 @@ public class TajoMasterClientService extends AbstractService {
throw new UnavailableTableLocationException(path.toString(), "no such a directory");
}
- Schema schema = new Schema(request.getSchema());
+ Schema schema = null;
+ if (request.hasSchema()) {
+ schema = new Schema(request.getSchema());
+ }
+
TableMeta meta = new TableMeta(request.getMeta());
PartitionMethodDesc partitionDesc = null;
if (request.hasPartition()) {
@@ -908,7 +912,7 @@ public class TajoMasterClientService extends AbstractService {
QueryContext queryContext = new QueryContext(conf, session);
context.getGlobalEngine().getDDLExecutor().dropTable(queryContext, dropTable.getName(), false,
- dropTable.getPurge());
+ dropTable.getPurge());
return OK;
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
index f7b7bf2..fffee80 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java
@@ -24,9 +24,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.exception.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.exception.DuplicateTableException;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.UndefinedTableException;
+import org.apache.tajo.exception.UndefinedTablespaceException;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.plan.logical.CreateTableNode;
import org.apache.tajo.plan.util.PlannerUtil;
@@ -78,7 +81,7 @@ public class CreateTableExecutor {
public TableDesc create(QueryContext queryContext,
String tableName,
@Nullable String tableSpaceName,
- Schema schema,
+ @Nullable Schema schema,
TableMeta meta,
@Nullable URI uri,
boolean isExternal,
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java
index 4e3e369..6190cdc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java
@@ -1294,13 +1294,18 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
if (checkIfExist(ctx.EXTERNAL())) {
createTable.setExternal();
+ if (checkIfExist(ctx.table_elements().asterisk())) {
+ createTable.setHasSelfDescSchema();
+ } else {
+ ColumnDefinition[] elements = getDefinitions(ctx.table_elements());
+ createTable.setTableElements(elements);
+ }
+
if (checkIfExist(ctx.TABLESPACE())) {
throw new TajoRuntimeException(new SQLSyntaxError("Tablespace clause is not allowed for an external table."));
}
- ColumnDefinition[] elements = getDefinitions(ctx.table_elements());
String storageType = ctx.storage_type.getText();
- createTable.setTableElements(elements);
createTable.setStorageType(storageType);
if (checkIfExist(ctx.LOCATION())) {
@@ -1311,8 +1316,12 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
}
} else {
if (checkIfExist(ctx.table_elements())) {
- ColumnDefinition[] elements = getDefinitions(ctx.table_elements());
- createTable.setTableElements(elements);
+ if (checkIfExist(ctx.table_elements().asterisk())) {
+ createTable.setHasSelfDescSchema();
+ } else {
+ ColumnDefinition[] elements = getDefinitions(ctx.table_elements());
+ createTable.setTableElements(elements);
+ }
}
if (checkIfExist(ctx.TABLESPACE())) {
@@ -1328,6 +1337,7 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
if (checkIfExist(ctx.query_expression())) {
Expr subquery = visitQuery_expression(ctx.query_expression());
createTable.setSubQuery(subquery);
+ createTable.unsetHasSelfDescSchema();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-docs/src/main/sphinx/sql_language/ddl.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/sql_language/ddl.rst b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
index 011f42e..c1e9bdb 100644
--- a/tajo-docs/src/main/sphinx/sql_language/ddl.rst
+++ b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
@@ -37,17 +37,19 @@ CREATE TABLE
.. code-block:: sql
- CREATE TABLE [IF NOT EXISTS] <table_name> [(<column_name> <data_type>, ... )] [TABLESPACE tablespace_name]
+ CREATE TABLE [IF NOT EXISTS] <table_name> [(column_list)] [TABLESPACE tablespace_name]
[using <storage_type> [with (<key> = <value>, ...)]] [AS <select_statement>]
- CREATE EXTERNAL TABLE [IF NOT EXISTS] <table_name> (<column_name> <data_type>, ... )
+ CREATE EXTERNAL TABLE [IF NOT EXISTS] <table_name> (column_list)
using <storage_type> [with (<key> = <value>, ...)] LOCATION '<path>'
*Description*
-In Tajo, there are two types of tables, `managed table` and `external table` (For more information, please refer to :doc:`/table_management/table_overview`.)
+In Tajo, there are two types of tables, `managed table` and `external table`.
+Managed tables are placed on some predefined tablespaces. The ``TABLESPACE`` clause is to specify a tablespace for this table. For external tables, Tajo allows an arbitrary table location with the ``LOCATION`` clause.
+For more information about tables and tablespace, please refer to :doc:`/table_management/table_overview` and :doc:`/table_management/tablespaces`.
-Managed tables are placed on some predefined tablespaces. The ``TABLESPACE`` clause is to specify a tablespace for this table. For more information about tablespace, please refer to :doc:`/table_management/tablespaces`. For external tables, Tajo allows an arbitrary table location with the ``LOCATION`` clause.
+``column_list`` is a sequence of the column name and its type like ``<column_name> <data_type>, ...``. Additionally, the `asterisk (*)` is allowed for external tables when their data format is `JSON`. You can find more details at :doc:`/table_management/json`.
``IF NOT EXISTS`` allows ``CREATE [EXTERNAL] TABLE`` statement to avoid an error which occurs when the table does not exist.
@@ -65,7 +67,7 @@ If you want to add an external table that contains compressed data, you should g
...
L_COMMENT text)
- USING TEXT WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec')
+ USING TEXT WITH ('text.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.SnappyCodec')
LOCATION 'hdfs://localhost:9010/tajo/warehouse/lineitem_100_snappy';
`compression.codec` parameter can have one of the following compression codecs:
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-docs/src/main/sphinx/table_management/file_formats.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/table_management/file_formats.rst b/tajo-docs/src/main/sphinx/table_management/file_formats.rst
index 7768920..966903c 100644
--- a/tajo-docs/src/main/sphinx/table_management/file_formats.rst
+++ b/tajo-docs/src/main/sphinx/table_management/file_formats.rst
@@ -2,12 +2,13 @@
File Formats
*************************************
-Currently, Tajo provides four file formats as follows:
+Currently, Tajo provides following file formats:
.. toctree::
:maxdepth: 1
text
+ json
rcfile
parquet
orc
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-docs/src/main/sphinx/table_management/json.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/table_management/json.rst b/tajo-docs/src/main/sphinx/table_management/json.rst
new file mode 100644
index 0000000..06d1fe6
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/table_management/json.rst
@@ -0,0 +1,100 @@
+****
+JSON
+****
+
+JSON(JavaScript Object Notation) is an open standard format for data (de)serialization. Since it is simple and human-readable, it is popularly used in many fields.
+Tajo supports JSON as its data format. In this section, you will get an overview of how to create JSON tables and query on them.
+
+============================
+How to Create a JSON Table ?
+============================
+
+You can create a JSON table using the ``CREATE TABLE`` statement. (For more information, please refer to :doc:`/sql_language/ddl`.)
+For example, please consider an example data as follows:
+
+.. code-block:: bash
+
+ $ hdfs dfs -cat /table1/table.json
+ { "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}
+ { "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}
+ { "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}
+
+Tajo provides two ways to create a table for this data. First is a traditional way to create tables. Here is an example.
+
+.. code-block:: sql
+
+ CREATE EXTERNAL TABLE table1 (
+ title TEXT,
+ name RECORD (
+ first_name TEXT,
+ last_name TEXT
+ )
+ ) USING JSON LOCATION '/table1/table.json';
+
+With this way, you need to specify every column which they want to use. This will be a tedious work, and not appropriate for flexible JSON schema.
+Second is a simpler alternative to alleviate this problem. When you create an external table of JSON format, you can simply omit the column specification as follows:
+
+.. code-block:: sql
+
+ CREATE EXTERNAL TABLE table1 (*) USING JSON LOCATION '/table1/table.json';
+
+No matter which way you choose, you can submit any queries on this table.
+
+.. code-block:: sql
+
+ > SELECT title, name.last_name from table1 where name.first_name = 'Arya';
+ title,name/last_name
+ -------------------------------
+ Assassin,Stark
+
+.. warning::
+
+ If you create a table with the second way, every column is assumed as the ``TEXT`` type.
+ So, you need to perform type casting if you want to handle them as other types.
+
+===================
+Physical Properties
+===================
+
+Some table storage formats provide parameters for enabling or disabling features and adjusting physical parameters.
+The ``WITH`` clause in the CREATE TABLE statement allows users to set those parameters.
+
+The JSON format provides the following physical properties.
+
+* ``text.delimiter``: delimiter character. ``|`` or ``\u0001`` is usually used, and the default field delimiter is ``|``.
+* ``text.null``: ``NULL`` character. The default ``NULL`` character is an empty string ``''``. Hive's default ``NULL`` character is ``'\\N'``.
+* ``compression.codec``: Compression codec. You can enable compression feature and set specified compression algorithm. The compression algorithm used to compress files. The compression codec name should be the fully qualified class name inherited from `org.apache.hadoop.io.compress.CompressionCodec <https://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/compress/CompressionCodec.html>`_. By default, compression is disabled.
+* ``timezone``: the time zone that the table uses for writting. When table rows are read or written, ```timestamp``` and ```time``` column values are adjusted by this timezone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'UTC+9' or a location-based form like 'Asia/Seoul'.
+* ``text.error-tolerance.max-num``: the maximum number of permissible parsing errors. This value should be an integer value. By default, ``text.error-tolerance.max-num`` is ``0``. According to the value, parsing errors will be handled in different ways.
+
+ * If ``text.error-tolerance.max-num < 0``, all parsing errors are ignored.
+ * If ``text.error-tolerance.max-num == 0``, any parsing error is not allowed. If any error occurs, the query will be failed. (default)
+ * If ``text.error-tolerance.max-num > 0``, the given number of parsing errors in each task will be pemissible.
+
+* ``text.skip.headerlines``: Number of header lines to be skipped. Some text files often have a header which has a kind of metadata(e.g.: column names), thus this option can be useful.
+
+The following example is to set a custom field delimiter, ``NULL`` character, and compression codec:
+
+.. code-block:: sql
+
+ CREATE TABLE table1 (
+ id int,
+ name text,
+ score float,
+ type text
+ ) USING JSON WITH('text.delimiter'='\u0001',
+ 'text.null'='\\N',
+ 'compression.codec'='org.apache.hadoop.io.compress.SnappyCodec');
+
+.. warning::
+
+ Be careful when using ``\n`` as the field delimiter because *TEXT* format tables use ``\n`` as the line delimiter.
+ At the moment, Tajo does not provide a way to specify the line delimiter.
+
+==========================
+Null Value Handling Issues
+==========================
+In default, ``NULL`` character in *TEXT* format is an empty string ``''``.
+In other words, an empty field is basically recognized as a ``NULL`` value in Tajo.
+If a field domain is ``TEXT``, an empty field is recognized as a string value ``''`` instead of ``NULL`` value.
+Besides, You can also use your own ``NULL`` character by specifying a physical property ``text.null``.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-docs/src/main/sphinx/table_management/text.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/table_management/text.rst b/tajo-docs/src/main/sphinx/table_management/text.rst
index 4755334..b79b0e2 100644
--- a/tajo-docs/src/main/sphinx/table_management/text.rst
+++ b/tajo-docs/src/main/sphinx/table_management/text.rst
@@ -2,8 +2,8 @@
TEXT
****
-A character-separated values plain-text file represents a tabular data set consisting of rows and columns.
-Each row is a plan-text line. A line is usually broken by a character line feed ``\n`` or carriage-return ``\r``.
+A character-separated values plain text file represents a tabular data set consisting of rows and columns.
+Each row is a plain text line. A line is usually broken by a character line feed ``\n`` or carriage-return ``\r``.
The line feed ``\n`` is the default delimiter in Tajo. Each record consists of multiple fields, separated by
some other character or string, most commonly a literal vertical bar ``|``, comma ``,`` or tab ``\t``.
The vertical bar is used as the default field delimiter in Tajo.
@@ -72,8 +72,8 @@ Custom (De)serializer
=====================
The *TEXT* format not only provides reading and writing interfaces for text data but also allows users to process custom
-plan-text file formats with user-defined (De)serializer classes.
-For example, with custom (de)serializers, Tajo can process JSON file formats or any specialized plan-text file formats.
+plain text file formats with user-defined (De)serializer classes.
+With custom (de)serializers, Tajo can process any text files no matter which the internal structure is.
In order to specify a custom (De)serializer, set a physical property ``text.serde``.
The property value should be a fully qualified class name.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
index b062e8e..89089e0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java
@@ -76,20 +76,27 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
LogicalPlan plan;
LogicalPlan.QueryBlock currentBlock;
NameResolvingMode columnRsvLevel;
+ boolean includeSelfDescTable;
- public Context(LogicalPlanner.PlanContext planContext, NameResolvingMode colRsvLevel) {
+ public Context(LogicalPlanner.PlanContext planContext, NameResolvingMode colRsvLevel, boolean includeSeflDescTable) {
this.queryContext = planContext.queryContext;
this.timeZone = planContext.timeZone;
this.plan = planContext.plan;
this.currentBlock = planContext.queryBlock;
this.columnRsvLevel = colRsvLevel;
+ this.includeSelfDescTable = includeSeflDescTable;
}
}
public EvalNode createEvalNode(LogicalPlanner.PlanContext planContext, Expr expr,
NameResolvingMode colRsvLevel) throws TajoException {
- Context context = new Context(planContext, colRsvLevel);
+ return createEvalNode(planContext, expr, colRsvLevel, false);
+ }
+
+ public EvalNode createEvalNode(LogicalPlanner.PlanContext planContext, Expr expr,
+ NameResolvingMode colRsvLevel, boolean includeSeflDescTable) throws TajoException {
+ Context context = new Context(planContext, colRsvLevel, includeSeflDescTable);
return planContext.evalOptimizer.optimize(planContext, visit(context, new Stack<Expr>(), expr));
}
@@ -542,12 +549,10 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva
switch (ctx.columnRsvLevel) {
case LEGACY:
- column = ctx.plan.resolveColumn(ctx.currentBlock, expr);
- break;
case RELS_ONLY:
case RELS_AND_SUBEXPRS:
case SUBEXPRS_AND_RELS:
- column = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, ctx.columnRsvLevel);
+ column = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, ctx.columnRsvLevel, ctx.includeSelfDescTable);
break;
default:
throw new TajoInternalError("Unsupported column resolving level: " + ctx.columnRsvLevel.name());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 504c625..3531ce7 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1376,7 +1376,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
private static LinkedHashSet<Target> createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode,
- Set<String> newlyEvaluatedRefNames) {
+ Set<String> newlyEvaluatedRefNames) {
LinkedHashSet<Target> targets = Sets.newLinkedHashSet();
for (Column column : relationNode.getLogicalSchema().getAllColumns()) {
@@ -2000,8 +2000,12 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
return createTableNode;
} else { // if CREATE AN EMPTY TABLE
- Schema tableSchema = convertColumnsToSchema(expr.getTableElements());
- createTableNode.setTableSchema(tableSchema);
+ if (!expr.hasSelfDescSchema()) {
+ Schema tableSchema = convertColumnsToSchema(expr.getTableElements());
+ createTableNode.setTableSchema(tableSchema);
+ } else {
+ createTableNode.setSelfDescSchema(true);
+ }
if (expr.isExternal()) {
createTableNode.setExternal(true);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
index dfa8079..af413ca 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java
@@ -31,6 +31,8 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.function.FunctionUtil;
+import org.apache.tajo.plan.nameresolver.NameResolver;
+import org.apache.tajo.plan.nameresolver.NameResolvingMode;
import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
import java.util.Stack;
@@ -142,7 +144,7 @@ public class TypeDeterminant extends SimpleAlgebraVisitor<LogicalPlanner.PlanCon
public DataType visitColumnReference(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, ColumnReferenceExpr expr)
throws TajoException {
stack.push(expr);
- Column column = ctx.plan.resolveColumn(ctx.queryBlock, expr);
+ Column column = NameResolver.resolve(ctx.plan, ctx.queryBlock, expr, NameResolvingMode.LEGACY, true);
stack.pop();
return column.getDataType();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
index 2d200fc..135f9ac 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java
@@ -508,6 +508,7 @@ public class BaseAlgebraVisitor<CONTEXT, RESULT> implements AlgebraVisitor<CONTE
// Insert or Update Section
///////////////////////////////////////////////////////////////////////////////////////////////////////////
+ @Override
public RESULT visitInsert(CONTEXT ctx, Stack<Expr> stack, Insert expr) throws TajoException {
stack.push(expr);
RESULT child = visit(ctx, stack, expr.getSubQuery());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java
index 46ea458..9a169dd 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java
@@ -30,6 +30,7 @@ public class CreateTableNode extends StoreTableNode implements Cloneable {
@Expose private String tableSpaceName;
@Expose private boolean external;
@Expose private boolean ifNotExists;
+ @Expose private boolean selfDescSchema = false;
public CreateTableNode(int pid) {
super(pid, NodeType.CREATE_TABLE);
@@ -82,6 +83,14 @@ public class CreateTableNode extends StoreTableNode implements Cloneable {
return ifNotExists;
}
+ public void setSelfDescSchema(boolean selfDescSchema) {
+ this.selfDescSchema = selfDescSchema;
+ }
+
+ public boolean hasSelfDescSchema() {
+ return selfDescSchema;
+ }
+
@Override
public PlanString getPlanString() {
return new PlanString(this);
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
index 170e13c..262d497 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java
@@ -74,6 +74,10 @@ public class StoreTableNode extends PersistentStoreNode implements Cloneable {
return this.uri;
}
+ public boolean hasTableSchema() {
+ return this.tableSchema != null;
+ }
+
public void setTableSchema(Schema schema) {
this.tableSchema = schema;
}
@@ -117,10 +121,10 @@ public class StoreTableNode extends PersistentStoreNode implements Cloneable {
if (obj instanceof StoreTableNode) {
StoreTableNode other = (StoreTableNode) obj;
boolean eq = super.equals(other);
- eq = eq && TUtil.checkEquals(this.tableName, other.tableName);
- eq = eq && TUtil.checkEquals(uri, other.uri);
- eq = tableSchema.equals(other.tableSchema);
- eq = eq && TUtil.checkEquals(partitionDesc, other.partitionDesc);
+ eq &= TUtil.checkEquals(this.tableName, other.tableName);
+ eq &= TUtil.checkEquals(uri, other.uri);
+ eq &= TUtil.checkEquals(tableSchema, other.tableSchema);
+ eq &= TUtil.checkEquals(partitionDesc, other.partitionDesc);
return eq;
} else {
return false;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index b41ef76..58ec60c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -26,9 +26,11 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.*;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.RelationNode;
+import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
@@ -68,13 +70,19 @@ public abstract class NameResolver {
public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
NameResolvingMode mode) throws TajoException {
+ return resolve(plan, block, column, mode, false);
+ }
+
+ public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
+ NameResolvingMode mode, boolean includeSelfDescTable) throws TajoException {
if (!resolverMap.containsKey(mode)) {
throw new RuntimeException("Unsupported name resolving level: " + mode.name());
}
- return resolverMap.get(mode).resolve(plan, block, column);
+ return resolverMap.get(mode).resolve(plan, block, column, includeSelfDescTable);
}
- abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef,
+ boolean includeSelfDescTable)
throws TajoException;
/**
@@ -143,7 +151,7 @@ public abstract class NameResolver {
* @return The found column
*/
static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ ColumnReferenceExpr columnRef, boolean includeSeflDescTable)
throws AmbiguousColumnException, AmbiguousTableException, UndefinedColumnException, UndefinedTableException {
String qualifier;
String canonicalName;
@@ -151,7 +159,7 @@ public abstract class NameResolver {
if (columnRef.hasQualifier()) {
Pair<String, String> normalized;
try {
- normalized = lookupQualifierAndCanonicalName(block, columnRef);
+ normalized = lookupQualifierAndCanonicalName(block, columnRef, includeSeflDescTable);
} catch (UndefinedColumnException udce) {
// is it correlated subquery?
// if the search column is not found at the current block, find it at all ancestors of the block.
@@ -178,22 +186,28 @@ public abstract class NameResolver {
throw new UndefinedTableException(qualifier);
}
- // Please consider a query case:
- // select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
- //
- // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available.
- // The below code makes it possible. Otherwise, it cannot find any match in the relation schema.
- if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
- canonicalName =
- CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
- }
+ Column column;
+ if (includeSeflDescTable && describeSchemaByItself(relationOp)) {
+ column = guessColumn(CatalogUtil.buildFQName(normalized.getFirst(), normalized.getSecond()));
+
+ } else {
+ // Please consider a query case:
+ // select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
+ //
+ // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available.
+ // The below code makes it possible. Otherwise, it cannot find any match in the relation schema.
+ if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
+ canonicalName =
+ CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
+ }
- Schema schema = relationOp.getLogicalSchema();
- Column column = schema.getColumn(canonicalName);
+ Schema schema = relationOp.getLogicalSchema();
+ column = schema.getColumn(canonicalName);
+ }
return column;
} else {
- return lookupColumnFromAllRelsInBlock(block, columnRef.getName());
+ return lookupColumnFromAllRelsInBlock(block, columnRef.getName(), includeSeflDescTable);
}
}
@@ -231,7 +245,7 @@ public abstract class NameResolver {
* @return The found column
*/
static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block,
- String columnName) throws AmbiguousColumnException {
+ String columnName, boolean includeSelfDescTable) throws AmbiguousColumnException {
Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName));
List<Column> candidates = TUtil.newList();
@@ -248,10 +262,36 @@ public abstract class NameResolver {
if (!candidates.isEmpty()) {
return ensureUniqueColumn(candidates);
} else {
+ if (includeSelfDescTable) {
+ List<RelationNode> candidateRels = TUtil.newList();
+ for (RelationNode rel : block.getRelations()) {
+ if (describeSchemaByItself(rel)) {
+ candidateRels.add(rel);
+ }
+ }
+ if (candidateRels.size() == 1) {
+ return guessColumn(CatalogUtil.buildFQName(candidateRels.get(0).getCanonicalName(), columnName));
+ } else if (candidateRels.size() > 1) {
+ throw new AmbiguousColumnException(columnName);
+ }
+ }
+
return null;
}
}
+ static boolean describeSchemaByItself(RelationNode relationNode) {
+ if (relationNode instanceof ScanNode && ((ScanNode) relationNode).getTableDesc().hasEmptySchema()) {
+ return true;
+ }
+ return false;
+ }
+
+ static Column guessColumn(String qualifiedName) {
+ // TODO: other data types must be supported.
+ return new Column(qualifiedName, Type.TEXT);
+ }
+
/**
* Trying to find a column from all relations in other blocks
*
@@ -319,7 +359,7 @@ public abstract class NameResolver {
* @return A pair of normalized qualifier and column name
*/
static Pair<String, String> lookupQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ ColumnReferenceExpr columnRef, boolean includeSeflDescTable)
throws AmbiguousColumnException, AmbiguousTableException, UndefinedColumnException {
Preconditions.checkArgument(columnRef.hasQualifier(), "ColumnReferenceExpr must be qualified.");
@@ -374,13 +414,30 @@ public abstract class NameResolver {
// throw exception if no column cannot be founded or two or more than columns are founded
if (guessedRelations.size() == 0) {
- throw new UndefinedColumnException(columnRef.getCanonicalName());
+ if (includeSeflDescTable) {
+ // check self-describing relations
+ for (RelationNode rel : block.getRelations()) {
+ if (describeSchemaByItself(rel)) {
+ columnNamePosition = 0;
+ guessedRelations.add(rel);
+ }
+ }
+
+ if (guessedRelations.size() > 1) {
+ throw new AmbiguousColumnException(columnRef.getCanonicalName());
+ } else if (guessedRelations.size() == 0) {
+ throw new UndefinedColumnException(columnRef.getCanonicalName());
+ }
+ } else {
+ throw new UndefinedColumnException(columnRef.getCanonicalName());
+ }
+
} else if (guessedRelations.size() > 1) {
throw new AmbiguousColumnException(columnRef.getCanonicalName());
}
String qualifier = guessedRelations.iterator().next().getCanonicalName();
- String columnName = "";
+ String columnName;
if (columnNamePosition >= qualifierParts.length) { // if there is no column in qualifierParts
columnName = columnRef.getName();
@@ -398,7 +455,7 @@ public abstract class NameResolver {
columnName += NestedPathUtil.PATH_DELIMITER + columnRef.getName();
}
- return new Pair<String, String>(qualifier, columnName);
+ return new Pair<>(qualifier, columnName);
}
static Column ensureUniqueColumn(List<Column> candidates) throws AmbiguousColumnException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java
index 4d9f9a5..6ea2f57 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java
@@ -73,8 +73,8 @@ package org.apache.tajo.plan.nameresolver;
* </ol>
*/
public enum NameResolvingMode {
- RELS_ONLY, // finding from only relations
- RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place
- SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place
- LEGACY // Finding in a legacy manner (globally)
+ RELS_ONLY, // finding from only relations excluding self-describing ones
+ RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place
+ SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place
+ LEGACY // Finding in a legacy manner (globally)
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
index 8991b60..94e38be 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
@@ -22,9 +22,9 @@ import org.apache.tajo.algebra.ColumnReferenceExpr;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.exception.UndefinedColumnException;
import org.apache.tajo.exception.AmbiguousColumnException;
import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.UndefinedColumnException;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
@@ -36,26 +36,30 @@ import java.util.List;
public class ResolverByLegacy extends NameResolver {
@Override
- public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ public Column resolve(LogicalPlan plan,
+ LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef,
+ boolean includeSeflDescTable)
throws TajoException {
if (columnRef.hasQualifier()) {
- return resolveColumnWithQualifier(plan, block, columnRef);
+ return resolveColumnWithQualifier(plan, block, columnRef, includeSeflDescTable);
} else {
- return resolveColumnWithoutQualifier(plan, block, columnRef);
+ return resolveColumnWithoutQualifier(plan, block, columnRef, includeSeflDescTable);
}
}
private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef) throws TajoException {
+ ColumnReferenceExpr columnRef, boolean includeSeflDescTable)
+ throws TajoException {
final String qualifier;
final String qualifiedName;
- Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef, includeSeflDescTable);
qualifier = normalized.getFirst();
qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
- Column found = resolveFromRelsWithinBlock(plan, block, columnRef);
+ Column found = resolveFromRelsWithinBlock(plan, block, columnRef, includeSeflDescTable);
if (found == null) {
throw new UndefinedColumnException(columnRef.getCanonicalName());
}
@@ -84,7 +88,7 @@ public class ResolverByLegacy extends NameResolver {
List<Column> candidates = TUtil.newList();
if (block.getNamedExprsManager().isAliased(qualifiedName)) {
String alias = block.getNamedExprsManager().getAlias(qualifiedName);
- found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY);
+ found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY, includeSeflDescTable);
if (found != null) {
candidates.add(found);
}
@@ -98,10 +102,10 @@ public class ResolverByLegacy extends NameResolver {
}
static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ ColumnReferenceExpr columnRef, boolean includeSeflDescTable)
throws AmbiguousColumnException, UndefinedColumnException {
- Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName());
+ Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName(), includeSeflDescTable);
if (found != null) {
return found;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java
index 82fbc21..4572143 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java
@@ -28,13 +28,17 @@ import org.apache.tajo.plan.LogicalPlan;
public class ResolverByRels extends NameResolver {
@Override
- public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ public Column resolve(LogicalPlan plan,
+ LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef,
+ boolean includeSeflDescTable)
throws AmbiguousColumnException, AmbiguousTableException, UndefinedColumnException, UndefinedTableException {
- Column column = resolveFromRelsWithinBlock(plan, block, columnRef);
- if (column == null) {
- throw new UndefinedColumnException(columnRef.getCanonicalName());
+ Column column = resolveFromRelsWithinBlock(plan, block, columnRef, includeSeflDescTable);
+ if (column != null) {
+ return column;
}
- return column;
+
+ throw new UndefinedColumnException(columnRef.getCanonicalName());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java
index 4bfdaf6..6ce7c67 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java
@@ -28,17 +28,22 @@ import org.apache.tajo.plan.LogicalPlan;
public class ResolverByRelsAndSubExprs extends NameResolver {
@Override
- public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ public Column resolve(LogicalPlan plan,
+ LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef,
+ boolean includeSeflDescTable)
throws AmbiguousColumnException, AmbiguousTableException, UndefinedColumnException, UndefinedTableException {
- Column column = resolveFromRelsWithinBlock(plan, block, columnRef);
- if (column == null) {
- column = resolveFromCurrentAndChildNode(block, columnRef);
+ Column column = resolveFromRelsWithinBlock(plan, block, columnRef, includeSeflDescTable);
+ if (column != null) {
+ return column;
}
- if (column == null) {
- throw new UndefinedColumnException(columnRef.getCanonicalName());
+ column = resolveFromCurrentAndChildNode(block, columnRef);
+ if (column != null) {
+ return column;
}
- return column;
+
+ throw new UndefinedColumnException(columnRef.getCanonicalName());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java
index ea291f4..4ed884e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java
@@ -28,17 +28,22 @@ import org.apache.tajo.plan.LogicalPlan;
public class ResolverBySubExprsAndRels extends NameResolver {
@Override
- public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
+ public Column resolve(LogicalPlan plan,
+ LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef,
+ boolean includeSeflDescTable)
throws AmbiguousColumnException, AmbiguousTableException, UndefinedColumnException, UndefinedTableException {
Column column = resolveFromCurrentAndChildNode(block, columnRef);
- if (column == null) {
- column = resolveFromRelsWithinBlock(plan, block, columnRef);
+ if (column != null) {
+ return column;
}
- if (column == null) {
- throw new UndefinedColumnException(columnRef.getCanonicalName());
+ column = resolveFromRelsWithinBlock(plan, block, columnRef, includeSeflDescTable);
+ if (column != null) {
+ return column;
}
- return column;
+
+ throw new UndefinedColumnException(columnRef.getCanonicalName());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java
index 92af709..3f64ae6 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanPreprocessPhaseProvider.java
@@ -27,7 +27,8 @@ public class BaseLogicalPlanPreprocessPhaseProvider extends LogicalPlanPreproces
@Override
public Collection<Class<? extends LogicalPlanPreprocessPhase>> getPhases() {
List phases = TUtil.newList(
- BaseSchemaBuildPhase.class
+ BaseSchemaBuildPhase.class,
+ SelfDescSchemaBuildPhase.class
);
return phases;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java
index 81e9b2c..f26e190 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java
@@ -40,6 +40,13 @@ import org.apache.tajo.util.TUtil;
import java.util.*;
+/**
+ * BaseSchemaBuildPhase builds a basic schema information of tables which have pre-defined schema.
+ * For example, tables like the below example have pre-defined schema.
+ *
+ * CREATE TABLE t1 (id int8, name text);
+ * CREATE EXTERNAL TABLE t2 (id int8, score int8, dept text);
+ */
public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
private final Processor processor;
@@ -178,15 +185,6 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
return newTargetExprs;
}
- private static boolean hasAsterisk(NamedExpr [] namedExprs) {
- for (NamedExpr eachTarget : namedExprs) {
- if (eachTarget.getExpr().getType() == OpType.Asterisk) {
- return true;
- }
- }
- return false;
- }
-
private static NamedExpr [] voidResolveAsteriskNamedExpr(LogicalPlanner.PlanContext context,
NamedExpr [] namedExprs) throws TajoException {
List<NamedExpr> rewrittenTargets = TUtil.newList();
@@ -222,7 +220,7 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
LogicalNode child = visit(ctx, stack, expr.getChild());
// Resolve the asterisk expression
- if (hasAsterisk(expr.getNamedExprs())) {
+ if (PlannerUtil.hasAsterisk(expr.getNamedExprs())) {
expr.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, expr.getNamedExprs()));
}
@@ -239,7 +237,7 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
namedExpr.getAlias());
} else if (OpType.isLiteralType(namedExpr.getExpr().getType()) && namedExpr.hasAlias()) {
Expr constExpr = namedExpr.getExpr();
- ConstEval constEval = (ConstEval) annotator.createEvalNode(ctx, constExpr, NameResolvingMode.RELS_ONLY);
+ ConstEval constEval = (ConstEval) annotator.createEvalNode(ctx, constExpr, NameResolvingMode.RELS_ONLY, true);
ctx.getQueryBlock().addConstReference(namedExpr.getAlias(), constExpr, constEval);
}
}
@@ -320,13 +318,13 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
int finalTargetNum = projection.getNamedExprs().length;
Target [] targets = new Target[finalTargetNum];
- if (hasAsterisk(projection.getNamedExprs())) {
+ if (PlannerUtil.hasAsterisk(projection.getNamedExprs())) {
projection.setNamedExprs(voidResolveAsteriskNamedExpr(ctx, projection.getNamedExprs()));
}
for (int i = 0; i < finalTargetNum; i++) {
NamedExpr namedExpr = projection.getNamedExprs()[i];
- EvalNode evalNode = annotator.createEvalNode(ctx, namedExpr.getExpr(), NameResolvingMode.SUBEXPRS_AND_RELS);
+ EvalNode evalNode = annotator.createEvalNode(ctx, namedExpr.getExpr(), NameResolvingMode.SUBEXPRS_AND_RELS, true);
if (namedExpr.hasAlias()) {
targets[i] = new Target(evalNode, namedExpr.getAlias());
@@ -564,7 +562,7 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
return insertNode;
}
- static class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor<PlanContext, Object> {
+ public static class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor<PlanContext, Object> {
private static final NameRefInSelectListNormalizer instance;
static {
@@ -581,7 +579,7 @@ public class BaseSchemaBuildPhase extends LogicalPlanPreprocessPhase {
throws TajoException {
String normalized = NameResolver.resolve(ctx.getPlan(), ctx.getQueryBlock(), expr,
- NameResolvingMode.RELS_ONLY).getQualifiedName();
+ NameResolvingMode.RELS_ONLY, true).getQualifiedName();
expr.setName(normalized);
return expr;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java
index 21f8b5c..dfdcb7a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanPreprocessPhase.java
@@ -49,7 +49,7 @@ public abstract class LogicalPlanPreprocessPhase {
* @param expr
* @return
*/
- public abstract boolean isEligible(PlanContext context, Expr expr);
+ public abstract boolean isEligible(PlanContext context, Expr expr) throws TajoException;
/**
* Do a pre-process phase for an expression tree and returns it.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java
new file mode 100644
index 0000000..310f238
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/SelfDescSchemaBuildPhase.java
@@ -0,0 +1,489 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan.rewrite;
+
+import com.google.common.base.Objects;
+import org.apache.tajo.SessionVars;
+import org.apache.tajo.algebra.*;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.exception.TajoInternalError;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.ExprAnnotator;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlan.QueryBlock;
+import org.apache.tajo.plan.LogicalPlanner.PlanContext;
+import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
+import org.apache.tajo.plan.logical.*;
+import org.apache.tajo.plan.nameresolver.NameResolver;
+import org.apache.tajo.plan.nameresolver.NameResolvingMode;
+import org.apache.tajo.plan.rewrite.BaseSchemaBuildPhase.Processor.NameRefInSelectListNormalizer;
+import org.apache.tajo.plan.util.ExprFinder;
+import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
+import org.apache.tajo.util.StringUtils;
+import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.graph.DirectedGraphVisitor;
+import org.apache.tajo.util.graph.SimpleDirectedGraph;
+
+import java.util.*;
+
+/**
+ * SelfDescSchemaBuildPhase builds the schema information of tables of self-describing data formats,
+ * such as JSON, Parquet, and ORC.
+ */
+public class SelfDescSchemaBuildPhase extends LogicalPlanPreprocessPhase {
+
+ private Processor processor;
+
+ public SelfDescSchemaBuildPhase(CatalogService catalog, ExprAnnotator annotator) {
+ super(catalog, annotator);
+ }
+
+ @Override
+ public String getName() {
+ return "Self-describing schema build phase";
+ }
+
+ private static String getQualifiedRelationName(PlanContext context, Relation relation) {
+ return CatalogUtil.isFQTableName(relation.getName()) ?
+ relation.getName() :
+ CatalogUtil.buildFQName(context.getQueryContext().get(SessionVars.CURRENT_DATABASE), relation.getName());
+ }
+
+ @Override
+ public boolean isEligible(PlanContext context, Expr expr) throws TajoException {
+ Set<Relation> relations = ExprFinderIncludeSubquery.finds(expr, OpType.Relation);
+ for (Relation eachRelation : relations) {
+ TableDesc tableDesc = catalog.getTableDesc(getQualifiedRelationName(context, eachRelation));
+ if (tableDesc.hasEmptySchema()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static class FinderContext<T> {
+ Set<T> set = new HashSet<>();
+ OpType targetType;
+
+ FinderContext(OpType type) {
+ this.targetType = type;
+ }
+ }
+
+ private static class ExprFinderIncludeSubquery extends SimpleAlgebraVisitor<FinderContext, Object> {
+
+ public static <T extends Expr> Set<T> finds(Expr expr, OpType type) throws TajoException {
+ FinderContext<T> context = new FinderContext<>(type);
+ ExprFinderIncludeSubquery finder = new ExprFinderIncludeSubquery();
+ finder.visit(context, new Stack<Expr>(), expr);
+ return context.set;
+ }
+
+ @Override
+ public Object visit(FinderContext ctx, Stack<Expr> stack, Expr expr) throws TajoException {
+ if (expr instanceof Selection) {
+ preHook(ctx, stack, expr);
+ visit(ctx, stack, ((Selection) expr).getQual());
+ visitUnaryOperator(ctx, stack, (UnaryOperator) expr);
+ postHook(ctx, stack, expr, null);
+ } else if (expr instanceof UnaryOperator) {
+ preHook(ctx, stack, expr);
+ visitUnaryOperator(ctx, stack, (UnaryOperator) expr);
+ postHook(ctx, stack, expr, null);
+ } else if (expr instanceof BinaryOperator) {
+ preHook(ctx, stack, expr);
+ visitBinaryOperator(ctx, stack, (BinaryOperator) expr);
+ postHook(ctx, stack, expr, null);
+ } else if (expr instanceof SimpleTableSubquery) {
+ preHook(ctx, stack, expr);
+ visit(ctx, stack, ((SimpleTableSubquery) expr).getSubQuery());
+ postHook(ctx, stack, expr, null);
+ } else if (expr instanceof TablePrimarySubQuery) {
+ preHook(ctx, stack, expr);
+ visit(ctx, stack, ((TablePrimarySubQuery) expr).getSubQuery());
+ postHook(ctx, stack, expr, null);
+ } else {
+ super.visit(ctx, stack, expr);
+ }
+
+ if (expr != null && ctx.targetType == expr.getType()) {
+ ctx.set.add(expr);
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ public LogicalNode process(PlanContext context, Expr expr) throws TajoException {
+ if (processor == null) {
+ processor = new Processor();
+ }
+ return processor.visit(new ProcessorContext(context), new Stack<Expr>(), expr);
+ }
+
+ static class ProcessorContext {
+ final PlanContext planContext;
+ final Map<String, List<ColumnReferenceExpr>> projectColumns = new HashMap<>();
+
+ public ProcessorContext(PlanContext planContext) {
+ this.planContext = planContext;
+ }
+ }
+
+ static class Processor extends BaseAlgebraVisitor<ProcessorContext, LogicalNode> {
+
+ private static <T extends LogicalNode> T getNodeFromExpr(LogicalPlan plan, Expr expr) {
+ return plan.getBlockByExpr(expr).getNodeFromExpr(expr);
+ }
+
+ private static <T extends LogicalNode> T getNonRelationListExpr(LogicalPlan plan, Expr expr) {
+ if (expr instanceof RelationList) {
+ return getNodeFromExpr(plan, ((RelationList) expr).getRelations()[0]);
+ } else {
+ return getNodeFromExpr(plan, expr);
+ }
+ }
+
+ @Override
+ public LogicalNode visitProjection(ProcessorContext ctx, Stack<Expr> stack, Projection expr) throws TajoException {
+ if (PlannerUtil.hasAsterisk(expr.getNamedExprs())) {
+ throw new UnsupportedException("Asterisk for self-describing data formats");
+ }
+
+ for (NamedExpr eachNamedExpr : expr.getNamedExprs()) {
+ Set<ColumnReferenceExpr> columns = ExprFinder.finds(eachNamedExpr, OpType.Column);
+ for (ColumnReferenceExpr col : columns) {
+ TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col);
+ }
+ }
+
+ super.visitProjection(ctx, stack, expr);
+
+ ProjectionNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitLimit(ProcessorContext ctx, Stack<Expr> stack, Limit expr) throws TajoException {
+ super.visitLimit(ctx, stack, expr);
+
+ LimitNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitSort(ProcessorContext ctx, Stack<Expr> stack, Sort expr) throws TajoException {
+ super.visitSort(ctx, stack, expr);
+
+ SortNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitHaving(ProcessorContext ctx, Stack<Expr> stack, Having expr) throws TajoException {
+ super.visitHaving(ctx, stack, expr);
+
+ HavingNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitGroupBy(ProcessorContext ctx, Stack<Expr> stack, Aggregation expr) throws TajoException {
+ super.visitGroupBy(ctx, stack, expr);
+
+ GroupbyNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitJoin(ProcessorContext ctx, Stack<Expr> stack, Join expr) throws TajoException {
+ super.visitJoin(ctx, stack, expr);
+
+ JoinNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode leftChild = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getLeft());
+ LogicalNode rightChild = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getRight());
+ node.setInSchema(SchemaUtil.merge(leftChild.getOutSchema(), rightChild.getOutSchema()));
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitFilter(ProcessorContext ctx, Stack<Expr> stack, Selection expr) throws TajoException {
+ Set<ColumnReferenceExpr> columnSet = ExprFinder.finds(expr.getQual(), OpType.Column);
+ for (ColumnReferenceExpr col : columnSet) {
+ NameRefInSelectListNormalizer.normalize(ctx.planContext, col);
+ TUtil.putToNestedList(ctx.projectColumns, col.getQualifier(), col);
+ }
+
+ super.visitFilter(ctx, stack, expr);
+
+ SelectionNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getChild());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitUnion(ProcessorContext ctx, Stack<Expr> stack, SetOperation expr) throws TajoException {
+ super.visitUnion(ctx, stack, expr);
+
+ UnionNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getLeft());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitExcept(ProcessorContext ctx, Stack<Expr> stack, SetOperation expr) throws TajoException {
+ super.visitExcept(ctx, stack, expr);
+
+ ExceptNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getLeft());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitIntersect(ProcessorContext ctx, Stack<Expr> stack, SetOperation expr) throws TajoException {
+ super.visitIntersect(ctx, stack, expr);
+
+ IntersectNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getLeft());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitSimpleTableSubquery(ProcessorContext ctx, Stack<Expr> stack, SimpleTableSubquery expr)
+ throws TajoException {
+ super.visitSimpleTableSubquery(ctx, stack, expr);
+
+ TableSubQueryNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getSubQuery());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitTableSubQuery(ProcessorContext ctx, Stack<Expr> stack, TablePrimarySubQuery expr)
+ throws TajoException {
+ super.visitTableSubQuery(ctx, stack, expr);
+
+ TableSubQueryNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getSubQuery());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitCreateTable(ProcessorContext ctx, Stack<Expr> stack, CreateTable expr) throws TajoException {
+ super.visitCreateTable(ctx, stack, expr);
+ CreateTableNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+
+ if (expr.hasSubQuery()) {
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getSubQuery());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ }
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitInsert(ProcessorContext ctx, Stack<Expr> stack, Insert expr) throws TajoException {
+ super.visitInsert(ctx, stack, expr);
+
+ InsertNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr);
+ LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getSubQuery());
+ node.setInSchema(child.getOutSchema());
+ node.setOutSchema(node.getInSchema());
+ return node;
+ }
+
+ @Override
+ public LogicalNode visitRelation(ProcessorContext ctx, Stack<Expr> stack, Relation expr) throws TajoException {
+ LogicalPlan plan = ctx.planContext.getPlan();
+ QueryBlock queryBlock = plan.getBlockByExpr(expr);
+ ScanNode scan = queryBlock.getNodeFromExpr(expr);
+ TableDesc desc = scan.getTableDesc();
+
+ if (desc.hasEmptySchema()) {
+ if (ctx.projectColumns.containsKey(getQualifiedRelationName(ctx.planContext, expr))) {
+ Set<Column> columns = new HashSet<>();
+ for (ColumnReferenceExpr col : ctx.projectColumns.get(getQualifiedRelationName(ctx.planContext, expr))) {
+ columns.add(NameResolver.resolve(plan, queryBlock, col, NameResolvingMode.RELS_ONLY, true));
+ }
+
+ desc.setSchema(buildSchemaFromColumnSet(columns));
+ scan.init(desc);
+ } else {
+ // error
+ throw new TajoInternalError(
+ "Columns projected from " + getQualifiedRelationName(ctx.planContext, expr) + " is not found.");
+ }
+ }
+
+ return scan;
+ }
+
+ /**
+ * This method creates a schema from a set of columns.
+ * For a nested column, its ancestors are guessed and added to the schema.
+ * For example, given a column 'glossary.title', the columns of (glossary RECORD (title TEXT)) will be added
+ * to the schema.
+ *
+ * @param columns a set of columns
+ * @return schema build from columns
+ */
+ private Schema buildSchemaFromColumnSet(Set<Column> columns) {
+ SchemaGraph schemaGraph = new SchemaGraph();
+ Set<ColumnVertex> rootVertexes = new HashSet<>();
+ Schema schema = new Schema();
+
+ for (Column eachColumn : columns) {
+ String simpleName = eachColumn.getSimpleName();
+ if (NestedPathUtil.isPath(simpleName)) {
+ String[] paths = simpleName.split(NestedPathUtil.PATH_DELIMITER);
+ for (int i = 0; i < paths.length-1; i++) {
+ String parentName = paths[i];
+ if (i == 0) {
+ parentName = CatalogUtil.buildFQName(eachColumn.getQualifier(), parentName);
+ }
+ // Leaf column type is TEXT; otherwise, RECORD.
+ Type childDataType = (i == paths.length-2) ? Type.TEXT : Type.RECORD;
+ ColumnVertex parentVertex = new ColumnVertex(
+ parentName,
+ StringUtils.join(paths, NestedPathUtil.PATH_DELIMITER, 0, i+1),
+ Type.RECORD);
+ schemaGraph.addEdge(
+ new ColumnEdge(
+ new ColumnVertex(
+ paths[i+1],
+ StringUtils.join(paths, NestedPathUtil.PATH_DELIMITER, 0, i+2), childDataType
+ ),
+ parentVertex));
+ if (i == 0) {
+ rootVertexes.add(parentVertex);
+ }
+ }
+ } else {
+ schema.addColumn(eachColumn);
+ }
+ }
+
+ // Build record columns
+ RecordColumnBuilder builder = new RecordColumnBuilder(schemaGraph);
+ for (ColumnVertex eachRoot : rootVertexes) {
+ schemaGraph.accept(eachRoot, builder);
+ schema.addColumn(eachRoot.column);
+ }
+
+ return schema;
+ }
+
+ private static class ColumnVertex {
+ private final String path;
+ private final String name;
+ private final Type type;
+ private Column column;
+
+ public ColumnVertex(String name, String path, Type type) {
+ this.name = name;
+ this.path = path;
+ this.type = type;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof ColumnVertex) {
+ ColumnVertex other = (ColumnVertex) o;
+ return this.name.equals(other.name) &&
+ this.type.equals(other.type) &&
+ this.path.equals(other.path);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(name, type, path);
+ }
+ }
+
+ private static class ColumnEdge {
+ private final ColumnVertex parent;
+ private final ColumnVertex child;
+
+ public ColumnEdge(ColumnVertex child, ColumnVertex parent) {
+ this.child = child;
+ this.parent = parent;
+ }
+ }
+
+ private static class SchemaGraph extends SimpleDirectedGraph<ColumnVertex, ColumnEdge> {
+ public void addEdge(ColumnEdge edge) {
+ this.addEdge(edge.child, edge.parent, edge);
+ }
+ }
+
+ private static class RecordColumnBuilder implements DirectedGraphVisitor<ColumnVertex> {
+ private final SchemaGraph graph;
+
+ public RecordColumnBuilder(SchemaGraph graph) {
+ this.graph = graph;
+ }
+
+ @Override
+ public void visit(Stack<ColumnVertex> stack, ColumnVertex schemaVertex) {
+ if (graph.isLeaf(schemaVertex)) {
+ schemaVertex.column = new Column(schemaVertex.name, schemaVertex.type);
+ } else {
+ Schema schema = new Schema();
+ for (ColumnVertex eachChild : graph.getChilds(schemaVertex)) {
+ schema.addColumn(eachChild.column);
+ }
+ schemaVertex.column = new Column(schemaVertex.name, new TypeDesc(schema));
+ }
+ }
+ }
+ }
+}
[3/3] tajo git commit: TAJO-1832: Well support for self-describing
data formats.
Posted by ji...@apache.org.
TAJO-1832: Well support for self-describing data formats.
Closes #756
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5a155861
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5a155861
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5a155861
Branch: refs/heads/master
Commit: 5a15586139c5fed29b66cbfc03b54e0bed09f5b7
Parents: 4a96288
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Sep 23 10:12:15 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Sep 23 10:12:15 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/algebra/CreateTable.java | 14 +
.../java/org/apache/tajo/catalog/TableDesc.java | 36 +-
.../src/main/proto/CatalogProtos.proto | 2 +-
.../dictionary/AbstractTableDescriptor.java | 7 +-
.../tajo/catalog/store/AbstractDBStore.java | 46 +-
.../src/main/resources/schemas/derby/derby.xml | 4 +-
.../main/resources/schemas/mariadb/mariadb.xml | 4 +-
.../src/main/resources/schemas/mysql/mysql.xml | 4 +-
.../main/resources/schemas/oracle/oracle.xml | 4 +-
.../resources/schemas/postgresql/postgresql.xml | 4 +-
.../tajo/client/CatalogAdminClientImpl.java | 9 +-
tajo-client/src/main/proto/ClientProtos.proto | 2 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 6 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 123 ++++-
.../tajo/engine/query/TestCreateTable.java | 31 ++
.../engine/query/TestQueryOnSelfDescTable.java | 184 +++++++
.../tajo/engine/query/TestUnionQuery.java | 2 +-
.../TestCreateTable/json_table/table1.json | 4 +
.../TestQueryOnSelfDescTable/github/github.json | 4 +
.../TestQueryOnSelfDescTable/sample1/table.json | 3 +
.../sample2/sample2.json | 1 +
.../tweets/sample1.json | 4 +
.../TestCreateTable/testSelfDescTable1.sql | 1 +
.../TestQueryOnSelfDescTable/testCrossJoin.sql | 6 +
.../TestQueryOnSelfDescTable/testGroupby.sql | 7 +
.../TestQueryOnSelfDescTable/testGroupby2.sql | 7 +
.../TestQueryOnSelfDescTable/testGroupby3.sql | 7 +
...JoinOfSelfDescTablesWithQualifiedColumns.sql | 6 +
.../testJoinWithSchemaFullTable.sql | 8 +
.../testJoinWithSchemaFullTable2.sql | 8 +
.../testJoinWithSchemaFullTable4.sql | 8 +
.../TestQueryOnSelfDescTable/testSelect.sql | 8 +
.../TestQueryOnSelfDescTable/testSelect2.sql | 9 +
.../TestQueryOnSelfDescTable/testSort.sql | 7 +
.../positive/create_self_desc_table1.sql | 1 +
.../positive/create_self_desc_table2.sql | 1 +
.../testCrossJoin.result | 22 +
.../TestQueryOnSelfDescTable/testGroupby.result | 5 +
.../testGroupby2.result | 3 +
.../testGroupby3.result | 5 +
.../testInSubquery.1.result | 4 +
...nOfSelfDescTablesWithQualifiedColumns.result | 3 +
.../testJoinWithSchemaFullTable.result | 4 +
.../testJoinWithSchemaFullTable2.result | 3 +
.../testJoinWithSchemaFullTable4.result | 5 +
.../TestQueryOnSelfDescTable/testSelect.result | 3 +
.../TestQueryOnSelfDescTable/testSelect2.result | 3 +
.../TestQueryOnSelfDescTable/testSort.result | 6 +
.../TestSQLAnalyzer/case_sensitivity1.result | 1 +
.../TestSQLAnalyzer/create_table_like_1.result | 1 +
.../create_table_maptype_1.result | 1 +
.../create_table_maptype_2.result | 1 +
.../create_table_maptype_3.result | 1 +
.../create_table_nested_1.result | 1 +
.../create_table_nested_2.result | 1 +
.../create_table_partition_by_column.result | 1 +
.../create_table_partition_by_hash_1.result | 1 +
.../create_table_partition_by_hash_2.result | 1 +
.../create_table_partition_by_list.result | 1 +
.../create_table_partition_by_range.result | 1 +
.../TestUnionQuery/testUnionAndFilter.result | 6 +-
.../tajo/master/TajoMasterClientService.java | 8 +-
.../tajo/master/exec/CreateTableExecutor.java | 7 +-
.../org/apache/tajo/parser/sql/SQLAnalyzer.java | 18 +-
tajo-docs/src/main/sphinx/sql_language/ddl.rst | 12 +-
.../sphinx/table_management/file_formats.rst | 3 +-
.../src/main/sphinx/table_management/json.rst | 100 ++++
.../src/main/sphinx/table_management/text.rst | 8 +-
.../org/apache/tajo/plan/ExprAnnotator.java | 15 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 10 +-
.../org/apache/tajo/plan/TypeDeterminant.java | 4 +-
.../tajo/plan/algebra/BaseAlgebraVisitor.java | 1 +
.../tajo/plan/logical/CreateTableNode.java | 9 +
.../tajo/plan/logical/StoreTableNode.java | 12 +-
.../tajo/plan/nameresolver/NameResolver.java | 99 +++-
.../plan/nameresolver/NameResolvingMode.java | 8 +-
.../plan/nameresolver/ResolverByLegacy.java | 24 +-
.../tajo/plan/nameresolver/ResolverByRels.java | 14 +-
.../nameresolver/ResolverByRelsAndSubExprs.java | 19 +-
.../nameresolver/ResolverBySubExprsAndRels.java | 17 +-
.../BaseLogicalPlanPreprocessPhaseProvider.java | 3 +-
.../tajo/plan/rewrite/BaseSchemaBuildPhase.java | 28 +-
.../rewrite/LogicalPlanPreprocessPhase.java | 2 +-
.../plan/rewrite/SelfDescSchemaBuildPhase.java | 489 +++++++++++++++++++
.../plan/serder/LogicalNodeDeserializer.java | 6 +-
.../tajo/plan/serder/LogicalNodeSerializer.java | 5 +-
.../org/apache/tajo/plan/util/ExprFinder.java | 12 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 9 +
.../plan/verifier/PreLogicalPlanVerifier.java | 19 +
.../tajo/plan/visitor/SimpleAlgebraVisitor.java | 3 +
tajo-plan/src/main/proto/Plan.proto | 2 +-
.../org/apache/tajo/parser/sql/SQLParser.g4 | 7 +-
93 files changed, 1470 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index fb0c8a6..63af5dd 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,8 @@ Release 0.11.0 - unreleased
NEW FEATURES
+ TAJO-1832: Well support for self-describing data formats. (jihoon)
+
TAJO-1730: JDBC Tablespace support. (hyunsik)
TAJO-1812: Timezone support in JSON file format. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
index 8e081ff..6131b8d 100644
--- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
+++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java
@@ -54,6 +54,8 @@ public class CreateTable extends Expr {
private boolean ifNotExists;
@Expose @SerializedName("LikeParentTable")
private String likeParentTable;
+ @Expose @SerializedName("HasSelfDescSchema")
+ private boolean selfDescSchema = false;
public CreateTable(final String tableName, boolean ifNotExists) {
super(OpType.CreateTable);
@@ -174,6 +176,18 @@ public class CreateTable extends Expr {
return likeParentTable;
}
+ public void setHasSelfDescSchema() {
+ selfDescSchema = true;
+ }
+
+ public void unsetHasSelfDescSchema() {
+ selfDescSchema = false;
+ }
+
+ public boolean hasSelfDescSchema() {
+ return selfDescSchema;
+ }
+
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index add77f2..ccf084e 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -18,10 +18,12 @@
package org.apache.tajo.catalog;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.Expose;
+import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
@@ -35,22 +37,22 @@ import java.net.URI;
public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Cloneable {
@Expose protected String tableName; // required
- @Expose protected Schema schema;
+ @Expose protected Schema schema; // optional for self-describing tables
@Expose protected TableMeta meta; // required
/** uri is set if external flag is TRUE. */
- @Expose protected URI uri; // optional
+ @Expose protected URI uri; // required
@Expose protected TableStats stats; // optional
/** the description of table partition */
@Expose protected PartitionMethodDesc partitionMethodDesc; // optional
/** True if it is an external table. False if it is a managed table. */
@Expose protected Boolean external; // optional
+ @VisibleForTesting
public TableDesc() {
}
- public TableDesc(String tableName, Schema schema, TableMeta meta,
- URI uri, boolean external) {
- this();
+ public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta,
+ @Nullable URI uri, boolean external) {
this.tableName = tableName;
this.schema = schema;
this.meta = meta;
@@ -58,16 +60,17 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
this.external = external;
}
- public TableDesc(String tableName, Schema schema, TableMeta meta, URI path) {
+ public TableDesc(String tableName, @Nullable Schema schema, TableMeta meta, @Nullable URI path) {
this(tableName, schema, meta, path, true);
}
- public TableDesc(String tableName, Schema schema, String storeType, KeyValueSet options, URI path) {
+ public TableDesc(String tableName, @Nullable Schema schema, String storeType, KeyValueSet options,
+ @Nullable URI path) {
this(tableName, schema, new TableMeta(storeType, options), path);
}
public TableDesc(TableDescProto proto) {
- this(proto.getTableName(), new Schema(proto.getSchema()),
+ this(proto.getTableName(), proto.hasSchema() ? new Schema(proto.getSchema()) : null,
new TableMeta(proto.getMeta()), proto.hasPath() ? URI.create(proto.getPath()) : null, proto.getIsExternal());
if(proto.hasStats()) {
this.stats = new TableStats(proto.getStats());
@@ -84,10 +87,6 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
public String getName() {
return this.tableName;
}
-
- public boolean hasUri() {
- return this.uri != null;
- }
public void setUri(URI uri) {
this.uri = uri;
@@ -108,6 +107,14 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
public void setSchema(Schema schem) {
this.schema = schem;
}
+
+ public boolean hasSchema() {
+ return schema != null;
+ }
+
+ public boolean hasEmptySchema() {
+ return schema.size() == 0;
+ }
public Schema getSchema() {
return schema;
@@ -170,7 +177,8 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
eq = eq && TUtil.checkEquals(uri, other.uri);
eq = eq && TUtil.checkEquals(partitionMethodDesc, other.partitionMethodDesc);
eq = eq && TUtil.checkEquals(external, other.external);
- return eq && TUtil.checkEquals(stats, other.stats);
+ eq = eq && TUtil.checkEquals(stats, other.stats);
+ return eq;
}
return false;
@@ -179,7 +187,7 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
public Object clone() throws CloneNotSupportedException {
TableDesc desc = (TableDesc) super.clone();
desc.tableName = tableName;
- desc.schema = (Schema) schema.clone();
+ desc.schema = schema != null ? (Schema) schema.clone() : null;
desc.meta = (TableMeta) meta.clone();
desc.uri = uri;
desc.stats = stats != null ? (TableStats) stats.clone() : null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index cec528a..0b7a1b4 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -98,7 +98,7 @@ message TableDescProto {
required string table_name = 2;
optional string path = 3;
required TableProto meta = 4;
- required SchemaProto schema = 5;
+ optional SchemaProto schema = 5;
optional TableStatsProto stats = 6;
optional PartitionMethodProto partition = 7;
optional bool isExternal = 8 [default = false];
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
index f2dc3cf..f9b7c62 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java
@@ -19,12 +19,7 @@
package org.apache.tajo.catalog.dictionary;
import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto;
abstract class AbstractTableDescriptor implements TableDescriptor {
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index c4d1828..d46ab3c 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -835,7 +835,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
conn.setAutoCommit(false);
String sql = "INSERT INTO TABLES (DB_ID, " + COL_TABLES_NAME +
- ", TABLE_TYPE, PATH, STORE_TYPE) VALUES(?, ?, ?, ?, ?) ";
+ ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA) VALUES(?, ?, ?, ?, ?, ?) ";
if (LOG.isDebugEnabled()) {
LOG.debug(sql);
@@ -851,6 +851,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
}
pstmt.setString(4, table.getPath());
pstmt.setString(5, table.getMeta().getStoreType());
+ pstmt.setBoolean(6, table.getSchema() == null);
pstmt.executeUpdate();
pstmt.close();
@@ -1672,7 +1673,7 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
// Geting Table Description
//////////////////////////////////////////
String sql =
- "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, STORE_TYPE FROM TABLES " +
+ "SELECT TID, " + COL_TABLES_NAME + ", TABLE_TYPE, PATH, STORE_TYPE, HAS_SELF_DESCRIBE_SCHEMA FROM TABLES " +
"WHERE DB_ID = ? AND " + COL_TABLES_NAME + "=?";
if (LOG.isDebugEnabled()) {
@@ -1698,34 +1699,37 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo
tableBuilder.setPath(res.getString(4).trim());
storeType = res.getString(5).trim();
+ boolean hasSelfDescSchema = res.getBoolean(6);
res.close();
pstmt.close();
- //////////////////////////////////////////
- // Geting Column Descriptions
- //////////////////////////////////////////
- CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder();
- sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS +
- " WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC";
+ if (!hasSelfDescSchema) {
+ //////////////////////////////////////////
+ // Geting Column Descriptions
+ //////////////////////////////////////////
+ CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder();
+ sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS +
+ " WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC";
- if (LOG.isDebugEnabled()) {
- LOG.debug(sql);
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(sql);
+ }
- pstmt = conn.prepareStatement(sql);
- pstmt.setInt(1, tableId);
- res = pstmt.executeQuery();
+ pstmt = conn.prepareStatement(sql);
+ pstmt.setInt(1, tableId);
+ res = pstmt.executeQuery();
- while (res.next()) {
- schemaBuilder.addFields(resultToColumnProto(res));
- }
+ while (res.next()) {
+ schemaBuilder.addFields(resultToColumnProto(res));
+ }
- tableBuilder.setSchema(
- CatalogUtil.getQualfiedSchema(databaseName + "." + tableName, schemaBuilder.build()));
+ tableBuilder.setSchema(
+ CatalogUtil.getQualfiedSchema(databaseName + "." + tableName, schemaBuilder.build()));
- res.close();
- pstmt.close();
+ res.close();
+ pstmt.close();
+ }
//////////////////////////////////////////
// Geting Table Properties
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
index c48c078..5280127 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
* 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813)
* 8 - 2015-09-02: Wrong table type problem in catalog (TAJO-1808)
* 7 - 2015-07-30: Add a column and index for partition keys (TAJO-1346)
@@ -29,7 +30,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="9">
+ <tns:base version="10">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -67,6 +68,7 @@
TABLE_TYPE VARCHAR(128) NOT NULL,
PATH VARCHAR(4096),
STORE_TYPE CHAR(16),
+ HAS_SELF_DESCRIBE_SCHEMA BOOLEAN,
CONSTRAINT TABLES_PK PRIMARY KEY (TID),
CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME)
)]]>
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
index 0bde634..0a7bfa2 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
* 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813)
* 8 - 2015-09-02: Wrong table type problem in catalog (TAJO-1808)
* 7 - 2015-07-30: Add a column and index for partition keys (TAJO-1346)
@@ -29,7 +30,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="9">
+ <tns:base version="10">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -65,6 +66,7 @@
TABLE_TYPE VARCHAR(128) NOT NULL,
PATH VARCHAR(4096) BINARY,
STORE_TYPE CHAR(16),
+ HAS_SELF_DESCRIBE_SCHEMA BOOLEAN,
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID),
INDEX IDX_DB_ID (DB_ID),
UNIQUE INDEX IDX_TABLE_ID (DB_ID, TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
index 1251693..1113b3e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
* 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813)
* 8 - 2015-09-02: Wrong table type problem in catalog (TAJO-1808)
* 7 - 2015-07-30: Add a column and index for partition keys (TAJO-1346)
@@ -29,7 +30,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="9">
+ <tns:base version="10">
<tns:objects>
<tns:Object order="0" type="table" name="META">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -65,6 +66,7 @@
TABLE_TYPE VARCHAR(128) NOT NULL,
PATH VARCHAR(4096) BINARY,
STORE_TYPE CHAR(16),
+ HAS_SELF_DESCRIBE_SCHEMA BOOLEAN,
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID),
INDEX IDX_DB_ID (DB_ID),
UNIQUE INDEX IDX_TABLE_ID (DB_ID, TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
index 1c09231..45d072e 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml
@@ -19,6 +19,7 @@
<tns:store xmlns:tns="http://tajo.apache.org/catalogstore" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
* 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813)
* 8 - 2015-09-02: Wrong table type problem in catalog (TAJO-1808)
* 7 - 2015-07-30: Add a column and index for partition keys (TAJO-1346)
@@ -29,7 +30,7 @@
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="9">
+ <tns:base version="10">
<tns:objects>
<tns:Object order="0" type="table" name="meta">
<tns:sql><![CDATA[
@@ -95,6 +96,7 @@
TABLE_NAME VARCHAR2(128) NOT NULL,
TABLE_TYPE VARCHAR2(128) NOT NULL,
PATH VARCHAR2(4000),
+ HAS_SELF_DESCRIBE_SCHEMA CHAR NOT NULL,
STORE_TYPE CHAR(16),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID),
CONSTRAINT C_TABLE_ID_UNIQ UNIQUE (DB_ID, TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
index feb6656..231dc20 100644
--- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
+++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml
@@ -21,6 +21,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.xsd ">
<!--
Catalog base version history
+ * 10 - 2015-09-22: Well support for self-describing data formats (TAJO-1832)
* 9 - 2015-09-12: Allow external catalog store for unit testing (TAJO-1813)
* 8 - 2015-09-02: Wrong table type problem in catalog (TAJO-1808)
* 7 - 2015-07-30: Add a column and index for partition keys (TAJO-1346)
@@ -32,7 +33,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
* 2 - 2014-06-09: First versioning
* 1- Before 2013-03-20
-->
- <tns:base version="9">
+ <tns:base version="10">
<tns:objects>
<tns:Object name="META" type="table" order="0">
<tns:sql><![CDATA[CREATE TABLE META (VERSION INT NOT NULL)]]></tns:sql>
@@ -72,6 +73,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition.
TABLE_NAME VARCHAR(128) NOT NULL,
TABLE_TYPE VARCHAR(128) NOT NULL,
PATH VARCHAR(4096),
+ HAS_SELF_DESCRIBE_SCHEMA BOOLEAN,
STORE_TYPE CHAR(16),
FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID),
UNIQUE (DB_ID, TABLE_NAME)
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
index 102b39e..22e5cc7 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java
@@ -135,12 +135,13 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
}
@Override
- public TableDesc createExternalTable(String tableName, Schema schema, URI path, TableMeta meta)
+ public TableDesc createExternalTable(String tableName, @Nullable Schema schema, URI path, TableMeta meta)
throws DuplicateTableException, UnavailableTableLocationException, InsufficientPrivilegeException {
return createExternalTable(tableName, schema, path, meta, null);
}
- public TableDesc createExternalTable(final String tableName, final Schema schema, final URI path,
+ @Override
+ public TableDesc createExternalTable(final String tableName, @Nullable final Schema schema, final URI path,
final TableMeta meta, final PartitionMethodDesc partitionMethodDesc)
throws DuplicateTableException, InsufficientPrivilegeException, UnavailableTableLocationException {
@@ -151,7 +152,9 @@ public class CatalogAdminClientImpl implements CatalogAdminClient {
final ClientProtos.CreateTableRequest.Builder builder = ClientProtos.CreateTableRequest.newBuilder();
builder.setSessionId(conn.sessionId);
builder.setName(tableName);
- builder.setSchema(schema.getProto());
+ if (schema != null) {
+ builder.setSchema(schema.getProto());
+ }
builder.setMeta(meta.getProto());
builder.setPath(path.toString());
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-client/src/main/proto/ClientProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto
index 13cb04b..5cf9639 100644
--- a/tajo-client/src/main/proto/ClientProtos.proto
+++ b/tajo-client/src/main/proto/ClientProtos.proto
@@ -194,7 +194,7 @@ message GetClusterInfoResponse {
message CreateTableRequest {
optional SessionIdProto sessionId = 1;
required string name = 2;
- required SchemaProto schema = 3;
+ optional SchemaProto schema = 3;
required TableProto meta = 4;
required string path = 5;
optional PartitionMethodProto partition = 6;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 9abfbd3..2fde31c 100644
--- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -915,7 +915,7 @@ public class QueryTestCaseBase {
return resultPath;
}
- private Path getDataSetFile(String fileName) throws IOException {
+ protected Path getDataSetFile(String fileName) throws IOException {
Path dataFilePath = StorageUtil.concatPath(currentDatasetPath, fileName);
FileSystem fs = currentDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(dataFilePath)) {
@@ -923,10 +923,10 @@ public class QueryTestCaseBase {
dataFilePath = StorageUtil.concatPath(namedDatasetPath, fileName);
fs = namedDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(dataFilePath)) {
- throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
+ throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
}
} else {
- throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
+ throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
}
}
return dataFilePath;
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index cfb1ed8..38d02aa 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -34,13 +34,14 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType;
import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.datum.Int4Datum;
import org.apache.tajo.datum.TextDatum;
import org.apache.tajo.engine.function.FunctionLoader;
import org.apache.tajo.engine.function.builtin.SumInt;
import org.apache.tajo.engine.json.CoreGsonHelper;
-import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
@@ -1357,4 +1358,124 @@ public class TestLogicalPlanner {
assertEquals(alterTableNode.getPartitionValues()[1], "01");
assertEquals(alterTableNode.getPartitionValues()[2], "11");
}
+
+ String[] SELF_DESC = {
+ "select id, name, dept from default.self_desc_table1", // 0
+ "select name, dept from default.self_desc_table1 where id > 10",
+ };
+
+ @Test
+ public void testSelectFromSelfDescTable() throws Exception {
+ TableDesc tableDesc = new TableDesc("default.self_desc_table1", null, CatalogUtil.newTableMeta("TEXT"),
+ CommonTestingUtil.getTestDir().toUri(), true);
+ catalog.createTable(tableDesc);
+ assertTrue(catalog.existsTable("default.self_desc_table1"));
+ tableDesc = catalog.getTableDesc("default.self_desc_table1");
+ assertTrue(tableDesc.hasEmptySchema());
+
+ QueryContext context = createQueryContext();
+ Expr expr = sqlAnalyzer.parse(SELF_DESC[0]);
+ LogicalPlan logicalPlan = planner.createPlan(context, expr);
+
+ LogicalNode node = logicalPlan.getRootNode();
+ assertEquals(NodeType.ROOT, node.getType());
+ LogicalRootNode root = (LogicalRootNode) node;
+ testJsonSerDerObject(root);
+ testCloneLogicalNode(root);
+
+ assertEquals(NodeType.PROJECTION, root.getChild().getType());
+ ProjectionNode projectionNode = root.getChild();
+ testJsonSerDerObject(projectionNode);
+ testCloneLogicalNode(projectionNode);
+
+ // projection column test
+ Target[] targets = projectionNode.getTargets();
+ Arrays.sort(targets, new Comparator<Target>() {
+ @Override
+ public int compare(Target o1, Target o2) {
+ return o1.getCanonicalName().compareTo(o2.getCanonicalName());
+ }
+ });
+ assertEquals(3, targets.length);
+ assertEquals("default.self_desc_table1.dept", targets[0].getCanonicalName());
+ assertEquals("default.self_desc_table1.id", targets[1].getCanonicalName());
+ assertEquals("default.self_desc_table1.name", targets[2].getCanonicalName());
+
+ // scan column test
+ assertEquals(NodeType.SCAN, projectionNode.getChild().getType());
+ ScanNode scanNode = projectionNode.getChild();
+ targets = scanNode.getTargets();
+ Arrays.sort(targets, new Comparator<Target>() {
+ @Override
+ public int compare(Target o1, Target o2) {
+ return o1.getCanonicalName().compareTo(o2.getCanonicalName());
+ }
+ });
+ assertEquals(3, targets.length);
+ assertEquals("default.self_desc_table1.dept", targets[0].getCanonicalName());
+ assertEquals("default.self_desc_table1.id", targets[1].getCanonicalName());
+ assertEquals("default.self_desc_table1.name", targets[2].getCanonicalName());
+
+ catalog.dropTable("default.self_desc_table1");
+ }
+
+ @Test
+ public void testSelectWhereFromSelfDescTable() throws Exception {
+ TableDesc tableDesc = new TableDesc("default.self_desc_table1", null, CatalogUtil.newTableMeta("TEXT"),
+ CommonTestingUtil.getTestDir().toUri(), true);
+ catalog.createTable(tableDesc);
+ assertTrue(catalog.existsTable("default.self_desc_table1"));
+ tableDesc = catalog.getTableDesc("default.self_desc_table1");
+ assertTrue(tableDesc.hasEmptySchema());
+
+ QueryContext context = createQueryContext();
+ Expr expr = sqlAnalyzer.parse(SELF_DESC[1]);
+ LogicalPlan logicalPlan = planner.createPlan(context, expr);
+
+ LogicalNode node = logicalPlan.getRootNode();
+ assertEquals(NodeType.ROOT, node.getType());
+ LogicalRootNode root = (LogicalRootNode) node;
+ testJsonSerDerObject(root);
+ testCloneLogicalNode(root);
+
+ assertEquals(NodeType.PROJECTION, root.getChild().getType());
+ ProjectionNode projectionNode = root.getChild();
+ testJsonSerDerObject(projectionNode);
+ testCloneLogicalNode(projectionNode);
+
+ // projection column test
+ Target[] targets = projectionNode.getTargets();
+ Arrays.sort(targets, new Comparator<Target>() {
+ @Override
+ public int compare(Target o1, Target o2) {
+ return o1.getCanonicalName().compareTo(o2.getCanonicalName());
+ }
+ });
+ assertEquals(2, targets.length);
+ assertEquals("default.self_desc_table1.dept", targets[0].getCanonicalName());
+ assertEquals("default.self_desc_table1.name", targets[1].getCanonicalName());
+
+ assertEquals(NodeType.SELECTION, projectionNode.getChild().getType());
+ SelectionNode selectionNode = projectionNode.getChild();
+ assertEquals(new BinaryEval(EvalType.GTH, new FieldEval("default.self_desc_table1.id", CatalogUtil.newSimpleDataType(Type.TEXT)), new ConstEval(new Int4Datum(10))),
+ selectionNode.getQual());
+
+ // scan column test
+ assertEquals(NodeType.SCAN, selectionNode.getChild().getType());
+ ScanNode scanNode = selectionNode.getChild();
+ targets = scanNode.getTargets();
+ Arrays.sort(targets, new Comparator<Target>() {
+ @Override
+ public int compare(Target o1, Target o2) {
+ return o1.getCanonicalName().compareTo(o2.getCanonicalName());
+ }
+ });
+ assertEquals(4, targets.length);
+ assertEquals("?greaterthan", targets[0].getCanonicalName());
+ assertEquals("default.self_desc_table1.dept", targets[1].getCanonicalName());
+ assertEquals("default.self_desc_table1.id", targets[2].getCanonicalName());
+ assertEquals("default.self_desc_table1.name", targets[3].getCanonicalName());
+
+ catalog.dropTable("default.self_desc_table1");
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index e33f173..f2f54ba 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -649,4 +649,35 @@ public class TestCreateTable extends QueryTestCaseBase {
executeString("DROP TABLE D9.nested_table2");
executeString("DROP DATABASE D9").close();
}
+
+ @Test
+ public final void testSelfDescTable1() throws Exception {
+ executeString("create database d9;").close();
+
+ assertTableNotExists("d9.schemaless");
+ executeQuery();
+ assertTableExists("d9.schemaless");
+ TableDesc desc = getClient().getTableDesc("d9.schemaless");
+ assertTrue(desc.hasEmptySchema());
+
+ executeString("drop table d9.schemaless").close();
+ executeString("drop database d9").close();
+ }
+
+ @Test
+ public final void testSelfDescTable2() throws Exception {
+ executeString("create database d10;").close();
+
+ String className = getClass().getSimpleName();
+ Path currentDatasetPath = new Path(datasetBasePath, className);
+ Path filePath = StorageUtil.concatPath(currentDatasetPath, "table1");
+ String sql = "create external table d10.schemaless (*) using json with ('compression.codec'='none') location '" + filePath.toString() + "'";
+ executeString(sql).close();
+ assertTableExists("d10.schemaless");
+ TableDesc desc = getClient().getTableDesc("d10.schemaless");
+ assertTrue(desc.hasEmptySchema());
+
+ executeString("drop table d10.schemaless").close();
+ executeString("drop database d10").close();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestQueryOnSelfDescTable.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestQueryOnSelfDescTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestQueryOnSelfDescTable.java
new file mode 100644
index 0000000..9ce46bc
--- /dev/null
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestQueryOnSelfDescTable.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.exception.AmbiguousColumnException;
+import org.apache.tajo.exception.TajoException;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+public class TestQueryOnSelfDescTable extends QueryTestCaseBase {
+
+ public TestQueryOnSelfDescTable() throws IOException, TajoException, SQLException {
+ super();
+
+ executeString(String.format("create external table if not exists self_desc_table1 (*) using json location '%s'",
+ getDataSetFile("sample1"))).close();
+
+ executeString(String.format("create external table if not exists self_desc_table2 (*) using json location '%s'",
+ getDataSetFile("sample2"))).close();
+
+ executeString(String.format("create external table if not exists self_desc_table3 (*) using json location '%s'",
+ getDataSetFile("tweets"))).close();
+
+ executeString(String.format("create external table if not exists github (*) using json location '%s'",
+ getDataSetFile("github"))).close();
+ }
+
+ @After
+ public void teardown() throws TajoException, SQLException {
+ executeString("drop table if exists self_desc_table1").close();
+ executeString("drop table if exists self_desc_table2").close();
+ executeString("drop table if exists self_desc_table3").close();
+ executeString("drop table if exists github").close();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testSelect() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testSelect2() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testGroupby() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testGroupby2() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testGroupby3() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @SimpleTest
+ public final void testSort() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testCrossJoin() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testJoinWithSchemaFullTable() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testJoinWithSchemaFullTable2() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test(expected = AmbiguousColumnException.class)
+ public final void testJoinWithSchemaFullTable3() throws Exception {
+ executeString("" +
+ "select " +
+ " user.favourites_count::int8, " +
+ " l_linenumber, " +
+ " l_comment " +
+ "from " +
+ " default.lineitem, " +
+ " self_desc_table1, " +
+ " self_desc_table3, " +
+ " default.orders, " +
+ " default.supplier " +
+ "where " +
+ " user.favourites_count::int8 = (l_orderkey - 1) and " +
+ " l_orderkey = o_orderkey and " +
+ " l_linenumber = s_suppkey and " +
+ " self_desc_table3.user.favourites_count = self_desc_table1.name.first_name");
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testJoinWithSchemaFullTable4() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test(expected = AmbiguousColumnException.class)
+ public final void testJoinOfSelfDescTables() throws Exception {
+ executeString("" +
+ "select " +
+ " user.favourites_count::int8 " +
+ "from " +
+ " self_desc_table1, " +
+ " self_desc_table3 " +
+ "where " +
+ " user.favourites_count = name.first_name");
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest
+ public final void testJoinOfSelfDescTablesWithQualifiedColumns() throws Exception {
+ runSimpleTests();
+ }
+
+ @Test(expected = AmbiguousColumnException.class)
+ public final void testJoinWithSingleQualifiedColumn() throws Exception {
+ executeString("" +
+ "select " +
+ " self_desc_table3.user.favourites_count::int8 " +
+ "from " +
+ " self_desc_table1, " +
+ " self_desc_table3 " +
+ "where " +
+ " self_desc_table3.user.favourites_count = name.first_name");
+ }
+
+ @Test
+ @Option(sort = true)
+ @SimpleTest(queries = {
+ @QuerySpec("select * from default.lineitem where l_orderkey in (select user.favourites_count::int8 + 1 from self_desc_table3)")
+ })
+ public final void testInSubquery() throws Exception {
+ runSimpleTests();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java
index 5d1321a..ce22782 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java
@@ -692,7 +692,7 @@ public class TestUnionQuery extends QueryTestCaseBase {
}
@Test
- @Option(withExplain = true)
+ @Option(withExplain = true, sort = true)
@SimpleTest
public void testUnionAndFilter() throws Exception {
runSimpleTests();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/dataset/TestCreateTable/json_table/table1.json
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/dataset/TestCreateTable/json_table/table1.json b/tajo-core-tests/src/test/resources/dataset/TestCreateTable/json_table/table1.json
new file mode 100644
index 0000000..0b597b2
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/dataset/TestCreateTable/json_table/table1.json
@@ -0,0 +1,4 @@
+{"id":"2937257753","type":"PushEvent","actor":{"id":5266949,"login":"hardrubic","gravatar_id":"","url":"https://api.github.com/users/hardrubic","avatar_url":"https://avatars.githubusercontent.com/u/5266949?"},"repo":{"id":38299397,"name":"hardrubic/rxJavaTest","url":"https://api.github.com/repos/hardrubic/rxJavaTest"},"payload":{"push_id":712081726,"size":1,"distinct_size":1,"ref":"refs/heads/master","head":"ea79d7a424f2693b70b9496726f315a5711b6fe7","before":"613f05557ad353f4bedc6df54128f8091ed1f1e9","commits":[{"sha":"ea79d7a424f2693b70b9496726f315a5711b6fe7","author":{"email":"dgzx106@163.com","name":"hardrubic"},"message":"增加rxJava例子","distinct":true,"url":"https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7"}]},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257758","type":"WatchEvent","actor":{"id":11455393,"login":"chrischjh","gravatar_id":"","url":"https://api.github.com/users/chrischjh","avatar_url":"https://avatars.githubusercontent.com/u/11455393?"},"repo":{"id":18218031,"name":"dead-horse/co-and-koa-talk","url":"https://api.github.com/repos/dead-horse/co-and-koa-talk"},"payload":{"action":"started"},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257759","type":"CreateEvent","actor":{"id":206379,"login":"gvn","gravatar_id":"","url":"https://api.github.com/users/gvn","avatar_url":"https://avatars.githubusercontent.com/u/206379?"},"repo":{"id":24345476,"name":"gvn/webmaker-android","url":"https://api.github.com/repos/gvn/webmaker-android"},"payload":{"ref":"use-self-building","ref_type":"branch","master_branch":"master","description":"Webmaker for Firefox OS & Android","pusher_type":"user"},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257761","type":"ForkEvent","actor":{"id":1088854,"login":"CAOakleyII","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","avatar_url":"https://avatars.githubusercontent.com/u/1088854?"},"repo":{"id":11909954,"name":"skycocker/chromebrew","url":"https://api.github.com/repos/skycocker/chromebrew"},"payload":{"forkee":{"id":38339291,"name":"chromebrew","full_name":"CAOakleyII/chromebrew","owner":{"login":"CAOakleyII","id":1088854,"avatar_url":"https://avatars.githubusercontent.com/u/1088854?v=3","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","html_url":"https://github.com/CAOakleyII","followers_url":"https://api.github.com/users/CAOakleyII/followers","following_url":"https://api.github.com/users/CAOakleyII/following{/other_user}","gists_url":"https://api.github.com/users/CAOakleyII/gists{/gist_id}","starred_url":"https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CAOakleyII/subscri
ptions","organizations_url":"https://api.github.com/users/CAOakleyII/orgs","repos_url":"https://api.github.com/users/CAOakleyII/repos","events_url":"https://api.github.com/users/CAOakleyII/events{/privacy}","received_events_url":"https://api.github.com/users/CAOakleyII/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CAOakleyII/chromebrew","description":"Package manager for Chrome OS","fork":true,"url":"https://api.github.com/repos/CAOakleyII/chromebrew","forks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/forks","keys_url":"https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CAOakleyII/chromebrew/teams","hooks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/hooks","issue_events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}","
events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/events","assignees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}","branches_url":"https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}","tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/tags","blobs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}","trees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}","languages_url":"https://api.github.com/repos/CAOakleyII/chromebrew/languages","stargazers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/stargazers","contributors_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contributors","subscribers_url":"https://a
pi.github.com/repos/CAOakleyII/chromebrew/subscribers","subscription_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscription","commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}","git_commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}","comments_url":"https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}","issue_comment_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}","contents_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}","compare_url":"https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CAOakleyII/chromebrew/merges","archive_url":"https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CAOakleyII/chromebrew/downloads","issues_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}","pu
lls_url":"https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}","milestones_url":"https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}","notifications_url":"https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}","releases_url":"https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}","created_at":"2015-07-01T00:00:00Z","updated_at":"2015-06-28T10:11:09Z","pushed_at":"2015-06-09T07:46:57Z","git_url":"git://github.com/CAOakleyII/chromebrew.git","ssh_url":"git@github.com:CAOakleyII/chromebrew.git","clone_url":"https://github.com/CAOakleyII/chromebrew.git","svn_url":"https://github.com/CAOakleyII/chromebrew","homepage":"http://skycocker.github.io/chromebrew/","size":846,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"
open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master","public":true}},"public":true,"created_at":"2015-07-01T00:00:01Z"}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/github/github.json
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/github/github.json b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/github/github.json
new file mode 100644
index 0000000..0b597b2
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/github/github.json
@@ -0,0 +1,4 @@
+{"id":"2937257753","type":"PushEvent","actor":{"id":5266949,"login":"hardrubic","gravatar_id":"","url":"https://api.github.com/users/hardrubic","avatar_url":"https://avatars.githubusercontent.com/u/5266949?"},"repo":{"id":38299397,"name":"hardrubic/rxJavaTest","url":"https://api.github.com/repos/hardrubic/rxJavaTest"},"payload":{"push_id":712081726,"size":1,"distinct_size":1,"ref":"refs/heads/master","head":"ea79d7a424f2693b70b9496726f315a5711b6fe7","before":"613f05557ad353f4bedc6df54128f8091ed1f1e9","commits":[{"sha":"ea79d7a424f2693b70b9496726f315a5711b6fe7","author":{"email":"dgzx106@163.com","name":"hardrubic"},"message":"增加rxJava例子","distinct":true,"url":"https://api.github.com/repos/hardrubic/rxJavaTest/commits/ea79d7a424f2693b70b9496726f315a5711b6fe7"}]},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257758","type":"WatchEvent","actor":{"id":11455393,"login":"chrischjh","gravatar_id":"","url":"https://api.github.com/users/chrischjh","avatar_url":"https://avatars.githubusercontent.com/u/11455393?"},"repo":{"id":18218031,"name":"dead-horse/co-and-koa-talk","url":"https://api.github.com/repos/dead-horse/co-and-koa-talk"},"payload":{"action":"started"},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257759","type":"CreateEvent","actor":{"id":206379,"login":"gvn","gravatar_id":"","url":"https://api.github.com/users/gvn","avatar_url":"https://avatars.githubusercontent.com/u/206379?"},"repo":{"id":24345476,"name":"gvn/webmaker-android","url":"https://api.github.com/repos/gvn/webmaker-android"},"payload":{"ref":"use-self-building","ref_type":"branch","master_branch":"master","description":"Webmaker for Firefox OS & Android","pusher_type":"user"},"public":true,"created_at":"2015-07-01T00:00:01Z"}
+{"id":"2937257761","type":"ForkEvent","actor":{"id":1088854,"login":"CAOakleyII","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","avatar_url":"https://avatars.githubusercontent.com/u/1088854?"},"repo":{"id":11909954,"name":"skycocker/chromebrew","url":"https://api.github.com/repos/skycocker/chromebrew"},"payload":{"forkee":{"id":38339291,"name":"chromebrew","full_name":"CAOakleyII/chromebrew","owner":{"login":"CAOakleyII","id":1088854,"avatar_url":"https://avatars.githubusercontent.com/u/1088854?v=3","gravatar_id":"","url":"https://api.github.com/users/CAOakleyII","html_url":"https://github.com/CAOakleyII","followers_url":"https://api.github.com/users/CAOakleyII/followers","following_url":"https://api.github.com/users/CAOakleyII/following{/other_user}","gists_url":"https://api.github.com/users/CAOakleyII/gists{/gist_id}","starred_url":"https://api.github.com/users/CAOakleyII/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CAOakleyII/subscri
ptions","organizations_url":"https://api.github.com/users/CAOakleyII/orgs","repos_url":"https://api.github.com/users/CAOakleyII/repos","events_url":"https://api.github.com/users/CAOakleyII/events{/privacy}","received_events_url":"https://api.github.com/users/CAOakleyII/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CAOakleyII/chromebrew","description":"Package manager for Chrome OS","fork":true,"url":"https://api.github.com/repos/CAOakleyII/chromebrew","forks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/forks","keys_url":"https://api.github.com/repos/CAOakleyII/chromebrew/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CAOakleyII/chromebrew/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CAOakleyII/chromebrew/teams","hooks_url":"https://api.github.com/repos/CAOakleyII/chromebrew/hooks","issue_events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/events{/number}","
events_url":"https://api.github.com/repos/CAOakleyII/chromebrew/events","assignees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/assignees{/user}","branches_url":"https://api.github.com/repos/CAOakleyII/chromebrew/branches{/branch}","tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/tags","blobs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/refs{/sha}","trees_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CAOakleyII/chromebrew/statuses/{sha}","languages_url":"https://api.github.com/repos/CAOakleyII/chromebrew/languages","stargazers_url":"https://api.github.com/repos/CAOakleyII/chromebrew/stargazers","contributors_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contributors","subscribers_url":"https://a
pi.github.com/repos/CAOakleyII/chromebrew/subscribers","subscription_url":"https://api.github.com/repos/CAOakleyII/chromebrew/subscription","commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/commits{/sha}","git_commits_url":"https://api.github.com/repos/CAOakleyII/chromebrew/git/commits{/sha}","comments_url":"https://api.github.com/repos/CAOakleyII/chromebrew/comments{/number}","issue_comment_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues/comments{/number}","contents_url":"https://api.github.com/repos/CAOakleyII/chromebrew/contents/{+path}","compare_url":"https://api.github.com/repos/CAOakleyII/chromebrew/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CAOakleyII/chromebrew/merges","archive_url":"https://api.github.com/repos/CAOakleyII/chromebrew/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CAOakleyII/chromebrew/downloads","issues_url":"https://api.github.com/repos/CAOakleyII/chromebrew/issues{/number}","pu
lls_url":"https://api.github.com/repos/CAOakleyII/chromebrew/pulls{/number}","milestones_url":"https://api.github.com/repos/CAOakleyII/chromebrew/milestones{/number}","notifications_url":"https://api.github.com/repos/CAOakleyII/chromebrew/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CAOakleyII/chromebrew/labels{/name}","releases_url":"https://api.github.com/repos/CAOakleyII/chromebrew/releases{/id}","created_at":"2015-07-01T00:00:00Z","updated_at":"2015-06-28T10:11:09Z","pushed_at":"2015-06-09T07:46:57Z","git_url":"git://github.com/CAOakleyII/chromebrew.git","ssh_url":"git@github.com:CAOakleyII/chromebrew.git","clone_url":"https://github.com/CAOakleyII/chromebrew.git","svn_url":"https://github.com/CAOakleyII/chromebrew","homepage":"http://skycocker.github.io/chromebrew/","size":846,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"
open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master","public":true}},"public":true,"created_at":"2015-07-01T00:00:01Z"}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample1/table.json
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample1/table.json b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample1/table.json
new file mode 100644
index 0000000..db3ad6c
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample1/table.json
@@ -0,0 +1,3 @@
+{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}
+{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}
+{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample2/sample2.json
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample2/sample2.json b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample2/sample2.json
new file mode 100644
index 0000000..20bab62
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/sample2/sample2.json
@@ -0,0 +1 @@
+{ "glossary": { "title": "example glossary", "GlossDiv": { "title": "S", "GlossList": { "GlossEntry": { "ID": "SGML", "SortAs": "SGML", "GlossTerm": "Standard Generalized Markup Language", "Acronym": "SGML", "Abbrev": "ISO 8879:1986", "GlossDef": { "para": "A meta-markup language, used to create markup languages such as DocBook.", "GlossSeeAlso": ["GML", "XML"] }, "GlossSee": "markup" } } } } }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/tweets/sample1.json
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/tweets/sample1.json b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/tweets/sample1.json
new file mode 100644
index 0000000..78a9071
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/dataset/TestQueryOnSelfDescTable/tweets/sample1.json
@@ -0,0 +1,4 @@
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Mon Sep 24 03:35:21 +0000 2012","id_str":"250075927172759552","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Aggressive Ponytail #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":1,"in_reply_to_status_id_str":null,"id":250075927172759552,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDEEF6","profile_sidebar_border_color":"C0DEED","profile_background_tile":false,"name":"Sean Cummings","profile_image_url":"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon Apr 26 06:01:55 +0000 2010","location":"LA, CA","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"137238150","entities":{"url":{"urls":[{"expanded_url":null,"url":"","i
ndices":[0,0]}]},"description":{"urls":[]}},"default_profile":true,"contributors_enabled":false,"favourites_count":0,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific Time (US & Canada)","description":"Born 330 Live 310","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"<a>Twitter for Mac<\/a>","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:40:54 +0000 2012","id_str":"249292149810667520","entities":{"urls":[],"hashtags":[{"text":"FreeBandNames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Thee Namaste Nerdz. #FreeBandNames","metadata":{"iso_language_code":"pl","result_type":"recent"},"retweet_count":2,"in_reply_to_status_id_str":null,"id":249292149810667520,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDFFCC","profile_sidebar_border_color":"BDDCAD","profile_background_tile":true,"name":"Chaz Martenstein","profile_image_url":"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue Apr 07 19:05:07 +0000 2009","location":"Durham, NC","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"29516238","entities":{"url":{"urls":[{"expanded_url":null,"url":"http://bu
llcityrecords.com/wnng/","indices":[0,32]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":8,"url":"http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c
ount":7579,"friends_count":348,"following":null,"show_all_inline_media":true,"screen_name":"bullcityrecords"},"in_reply_to_screen_name":null,"source":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:30:20 +0000 2012","id_str":"249289491129438208","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[29,43]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Mexican Heaven, Mexican Hell #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":3,"in_reply_to_status_id_str":null,"id":249289491129438208,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"99CC33","profile_sidebar_border_color":"829D5E","profile_background_tile":false,"name":"Thomas John Wakeman","profile_image_url":"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue Sep 01 21:21:35 +0000 2009","location":"Kingston New York","follow_request_sent":null,"profile_link_color":"D02B55","is_translator":false,"id_str":"70789458","entities":{"url":{"urls":[{"expanded_url":n
ull,"url":"","indices":[0,0]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":19,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour
ce":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 22:51:18 +0000 2012","id_str":"249279667666817024","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"The Foolish Mortals #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":4,"in_reply_to_status_id_str":null,"id":249279667666817024,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"BFAC83","profile_sidebar_border_color":"615A44","profile_background_tile":true,"name":"Marty Elmer","profile_image_url":"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon May 04 00:05:00 +0000 2009","location":"Wisconsin, USA","follow_request_sent":null,"profile_link_color":"3B2A26","is_translator":false,"id_str":"37539828","entities":{"url":{"urls":[{"expanded_url":null,"url":"ht
tp://www.omnitarian.me","indices":[0,24]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":647,"url":"http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central Time (US & Canada)","description":"Cartoonist, Illustrator, and T-Shirt connoisseur","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree
n_name":"Omnitarian"},"in_reply_to_screen_name":null,"source":"<a>Twitter for iPhone<\/a>","in_reply_to_status_id":null}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestCreateTable/testSelfDescTable1.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testSelfDescTable1.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testSelfDescTable1.sql
new file mode 100644
index 0000000..6dacfbf
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testSelfDescTable1.sql
@@ -0,0 +1 @@
+create table d9.schemaless (*) using json with ('compression.codec'='none') partition by column (id int8)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testCrossJoin.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testCrossJoin.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testCrossJoin.sql
new file mode 100644
index 0000000..4a49790
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testCrossJoin.sql
@@ -0,0 +1,6 @@
+select
+ user.favourites_count::int8,
+ l_linenumber,
+ l_comment
+from
+ default.lineitem, self_desc_table3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby.sql
new file mode 100644
index 0000000..9c7b33a
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby.sql
@@ -0,0 +1,7 @@
+select
+ name.first_name,
+ count(*)
+from
+ self_desc_table1
+group by
+ name.first_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby2.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby2.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby2.sql
new file mode 100644
index 0000000..5b302cd
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby2.sql
@@ -0,0 +1,7 @@
+select
+ coordinates,
+ avg(retweet_count::int4)
+from
+ self_desc_table3
+group by
+ coordinates
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby3.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby3.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby3.sql
new file mode 100644
index 0000000..4d752fb
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testGroupby3.sql
@@ -0,0 +1,7 @@
+select
+ user.time_zone,
+ sum(user.favourites_count::int8)
+from
+ self_desc_table3
+group by
+ user.time_zone
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.sql
new file mode 100644
index 0000000..3e0feff
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinOfSelfDescTablesWithQualifiedColumns.sql
@@ -0,0 +1,6 @@
+select
+ self_desc_table3.user.favourites_count::int8
+from
+ github, self_desc_table3
+where
+ self_desc_table3.user.favourites_count = (github.actor.id::int8 - 206379)::text
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.sql
new file mode 100644
index 0000000..2bb8ab8
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable.sql
@@ -0,0 +1,8 @@
+select
+ user.favourites_count::int8,
+ l_linenumber,
+ l_comment
+from
+ default.lineitem, self_desc_table3
+where
+ user.favourites_count::int8 = (l_orderkey - 1)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.sql
new file mode 100644
index 0000000..fd7eb6a
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable2.sql
@@ -0,0 +1,8 @@
+select
+ user.favourites_count::int8,
+ l_linenumber,
+ l_comment
+from
+ default.lineitem, self_desc_table3, default.orders, default.supplier
+where
+ user.favourites_count::int8 = (l_orderkey - 1) and l_orderkey = o_orderkey and l_linenumber = s_suppkey
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.sql
new file mode 100644
index 0000000..de2679c
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testJoinWithSchemaFullTable4.sql
@@ -0,0 +1,8 @@
+select
+ self_desc_table3.user.favourites_count::int8,
+ l_linenumber,
+ l_comment
+from
+ default.lineitem, self_desc_table1, self_desc_table3, default.orders, default.supplier
+where
+ self_desc_table3.user.favourites_count::int8 = (l_orderkey - 1) and l_orderkey = o_orderkey and l_linenumber = s_suppkey and self_desc_table3.user.favourites_count <> self_desc_table1.name.first_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect.sql
new file mode 100644
index 0000000..1b31c8a
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect.sql
@@ -0,0 +1,8 @@
+select
+ glossary.title,
+ glossary."GlossDiv".title,
+ glossary."GlossDiv".null_expected,
+ glossary."GlossDiv"."GlossList"."GlossEntry"."SortAs",
+ glossary."GlossDiv"."GlossList"."GlossEntry"."Abbrev"
+from
+ self_desc_table2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect2.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect2.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect2.sql
new file mode 100644
index 0000000..35a342d
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSelect2.sql
@@ -0,0 +1,9 @@
+select
+ glossary.title,
+ glossary."GlossDiv".title,
+ glossary."GlossDiv".null_expected,
+ glossary."GlossDiv"."GlossList"."GlossEntry"."SortAs"
+from
+ self_desc_table2
+where
+ glossary."GlossDiv"."GlossList"."GlossEntry"."Abbrev" = 'ISO 8879:1986'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSort.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSort.sql b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSort.sql
new file mode 100644
index 0000000..98a011b
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestQueryOnSelfDescTable/testSort.sql
@@ -0,0 +1,7 @@
+select
+ created_at,
+ id,
+ user.profile_sidebar_fill_color
+from
+ self_desc_table3
+order by created_at
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table1.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table1.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table1.sql
new file mode 100644
index 0000000..f636399
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table1.sql
@@ -0,0 +1 @@
+create external table schemaless (*) using json with ('compression.codec'='none') partition by column (id int8) location 'file:///schemaless'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table2.sql
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table2.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table2.sql
new file mode 100644
index 0000000..5f2cd18
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_self_desc_table2.sql
@@ -0,0 +1 @@
+create table schemaless (*) using json with ('compression.codec'='none') partition by column (id int8)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testCrossJoin.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testCrossJoin.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testCrossJoin.result
new file mode 100644
index 0000000..52f385a
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testCrossJoin.result
@@ -0,0 +1,22 @@
+?cast,l_linenumber,l_comment
+-------------------------------
+0,1,egular courts above the
+0,1,ongside of the furiously brave acco
+0,1,ven requests. deposits breach a
+0,2, unusual accounts. eve
+0,2,ly final dependencies: slyly bold
+19,1,egular courts above the
+19,1,ongside of the furiously brave acco
+19,1,ven requests. deposits breach a
+19,2, unusual accounts. eve
+19,2,ly final dependencies: slyly bold
+647,1,egular courts above the
+647,1,ongside of the furiously brave acco
+647,1,ven requests. deposits breach a
+647,2, unusual accounts. eve
+647,2,ly final dependencies: slyly bold
+8,1,egular courts above the
+8,1,ongside of the furiously brave acco
+8,1,ven requests. deposits breach a
+8,2, unusual accounts. eve
+8,2,ly final dependencies: slyly bold
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby.result
new file mode 100644
index 0000000..0dc413f
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby.result
@@ -0,0 +1,5 @@
+name/first_name,?count
+-------------------------------
+Arya,1
+Eddard,1
+Syrio,1
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby2.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby2.result
new file mode 100644
index 0000000..e7b795f
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby2.result
@@ -0,0 +1,3 @@
+coordinates,?avg_1
+-------------------------------
+null,2.5
http://git-wip-us.apache.org/repos/asf/tajo/blob/5a155861/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby3.result b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby3.result
new file mode 100644
index 0000000..2004343
--- /dev/null
+++ b/tajo-core-tests/src/test/resources/results/TestQueryOnSelfDescTable/testGroupby3.result
@@ -0,0 +1,5 @@
+user/time_zone,?sum_1
+-------------------------------
+Central Time (US & Canada),647
+Eastern Time (US & Canada),27
+Pacific Time (US & Canada),0