You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/05/12 05:14:11 UTC
[12/14] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
TAJO-1359: Add nested field projector and language extension to project nested record. (hyunsik)
Closes #422
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0d1bf41f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0d1bf41f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0d1bf41f
Branch: refs/heads/index_support
Commit: 0d1bf41f3aff119af5d95270d735101a81d5721c
Parents: ddd3921
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon May 11 14:34:07 2015 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon May 11 14:35:09 2015 -0700
----------------------------------------------------------------------
CHANGES | 7 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 12 +-
.../org/apache/tajo/catalog/NestedPathUtil.java | 109 ++++++
.../java/org/apache/tajo/catalog/Schema.java | 109 +++++-
.../org/apache/tajo/catalog/SchemaUtil.java | 33 +-
.../tajo/catalog/store/HiveCatalogStore.java | 4 +-
.../cli/tsql/commands/DescTableCommand.java | 4 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Tuple.java | 2 +-
.../java/org/apache/tajo/util/StringUtils.java | 70 ++++
.../main/java/org/apache/tajo/util/TUtil.java | 32 --
.../org/apache/tajo/engine/parser/SQLParser.g4 | 2 +-
.../apache/tajo/engine/parser/SQLAnalyzer.java | 17 +-
.../engine/planner/PhysicalPlannerImpl.java | 17 +-
.../tajo/engine/planner/global/DataChannel.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 6 +-
.../global/builder/DistinctGroupbyBuilder.java | 6 +-
.../DistinctGroupbyThirdAggregationExec.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 2 +-
.../engine/planner/physical/SeqScanExec.java | 27 +-
.../tajo/engine/utils/TupleCacheScanner.java | 2 +-
.../org/apache/tajo/engine/utils/TupleUtil.java | 8 +-
.../exec/NonForwardQueryResultFileScanner.java | 2 +-
.../NonForwardQueryResultSystemScanner.java | 20 +-
.../resources/webapps/admin/catalogview.jsp | 5 +-
.../engine/function/TestFunctionLoader.java | 5 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 8 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 4 +-
.../tajo/engine/query/TestJoinBroadcast.java | 2 +-
.../engine/query/TestSelectNestedRecord.java | 71 ++++
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +-
.../rs/resources/TestQueryResultResource.java | 6 +-
.../TestSelectNestedRecord/sample1/table.json | 3 +
.../TestSelectNestedRecord/tweets/sample1.json | 4 +
.../TestSelectNestedRecord/sample1_ddl.sql | 7 +
.../TestSelectNestedRecord/sample2_ddl.sql | 19 ++
.../testNestedFieldAsGroupbyKey1.sql | 7 +
.../testNestedFieldAsJoinKey1.sql | 7 +
.../TestSelectNestedRecord/testSelect1.sql | 1 +
.../TestSelectNestedRecord/testSelect2.sql | 61 ++++
.../TestSelectNestedRecord/tweets_ddl.sql | 74 +++++
.../testSelectWithParentheses2.sql | 2 +-
.../testNestedFieldAsGroupbyKey1.result | 6 +
.../testNestedFieldAsJoinKey1.result | 6 +
.../TestSelectNestedRecord/testSelect1.result | 5 +
.../TestSelectNestedRecord/testSelect2.result | 6 +
.../apache/tajo/jdbc/TajoDatabaseMetaData.java | 2 +-
.../tajo/plan/LogicalPlanPreprocessor.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 32 +-
.../apache/tajo/plan/expr/RowConstantEval.java | 3 +-
.../tajo/plan/expr/WindowFunctionEval.java | 3 +-
.../function/python/PythonScriptEngine.java | 12 +-
.../GreedyHeuristicJoinOrderAlgorithm.java | 8 +-
.../apache/tajo/plan/joinorder/JoinEdge.java | 4 +-
.../apache/tajo/plan/joinorder/JoinGraph.java | 6 +-
.../tajo/plan/logical/DistinctGroupbyNode.java | 4 +-
.../apache/tajo/plan/logical/EvalExprNode.java | 3 +-
.../apache/tajo/plan/logical/GroupbyNode.java | 10 +-
.../tajo/plan/logical/ProjectionNode.java | 3 +-
.../apache/tajo/plan/logical/RelationNode.java | 12 +
.../tajo/plan/logical/ShuffleFileWriteNode.java | 3 +-
.../tajo/plan/logical/TruncateTableNode.java | 4 +-
.../apache/tajo/plan/logical/WindowAggNode.java | 7 +-
.../tajo/plan/nameresolver/NameResolver.java | 194 ++++++++---
.../plan/nameresolver/ResolverByLegacy.java | 8 +-
.../plan/rewrite/rules/FilterPushDownRule.java | 4 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
.../org/apache/tajo/plan/util/PlannerUtil.java | 2 +-
.../tajo/plan/verifier/LogicalPlanVerifier.java | 24 +-
.../plan/verifier/PreLogicalPlanVerifier.java | 6 +-
.../org/apache/tajo/storage/MergeScanner.java | 2 +-
.../org/apache/tajo/storage/RowStoreUtil.java | 2 +-
.../java/org/apache/tajo/storage/Scanner.java | 9 +
.../org/apache/tajo/storage/StorageManager.java | 4 +-
.../storage/hbase/AbstractHBaseAppender.java | 2 -
.../tajo/storage/hbase/ColumnMapping.java | 2 +-
.../HBaseBinarySerializerDeserializer.java | 10 +-
.../apache/tajo/storage/hbase/HBaseScanner.java | 22 +-
.../tajo/storage/hbase/HBaseStorageManager.java | 6 +-
.../java/org/apache/tajo/storage/CSVFile.java | 2 +-
.../org/apache/tajo/storage/FileScanner.java | 2 +-
.../apache/tajo/storage/FileStorageManager.java | 2 +-
.../apache/tajo/storage/avro/AvroScanner.java | 25 +-
.../tajo/storage/json/JsonLineDeserializer.java | 331 +++++++++++--------
.../apache/tajo/storage/json/JsonLineSerDe.java | 5 +-
.../tajo/storage/json/JsonLineSerializer.java | 2 -
.../storage/parquet/TajoRecordConverter.java | 7 +-
.../org/apache/tajo/storage/rcfile/RCFile.java | 10 +-
.../sequencefile/SequenceFileScanner.java | 16 +-
.../tajo/storage/text/CSVLineDeserializer.java | 26 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 5 +-
.../tajo/storage/text/DelimitedTextFile.java | 11 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/TextLineSerDe.java | 3 +-
.../apache/tajo/storage/TestMergeScanner.java | 29 +-
.../org/apache/tajo/storage/TestStorages.java | 35 +-
96 files changed, 1279 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0aad306..84be66a 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,8 +27,8 @@ Release 0.11.0 - unreleased
TAJO-1452: Improve function listing order (Contributed Dongjoon Hyun,
Committed by hyunsik)
- TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage() eliminates
- an important kind of information.
+ TAJO-1576: Sometimes DefaultTajoCliOutputFormatter.parseErrorMessage()
+ eliminates an important kind of information.
(Contributed by Jongyoung Park, Committed by jihoon)
TAJO-1408: Make IntermediateEntryProto more compact.
@@ -277,6 +277,9 @@ Release 0.11.0 - unreleased
SUB TASKS
+ TAJO-1359: Add nested field projector and language extension to project
+ nested record. (hyunsik)
+
TAJO-1529: Implement json_extract_path_text(string, string) function.
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 12f36b1..8e5b657 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -171,6 +171,16 @@ public class CatalogUtil {
return openQuote && closeQuote;
}
+ /**
+ * True if a given name is a simple identifier, meaning is not a dot-chained name.
+ *
+ * @param columnOrTableName Column or Table name to be checked
+ * @return True if a given name is a simple identifier. Otherwise, it will return False.
+ */
+ public static boolean isSimpleIdentifier(String columnOrTableName) {
+ return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1;
+ }
+
public static boolean isFQColumnName(String tableName) {
return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3;
}
@@ -662,7 +672,7 @@ public class CatalogUtil {
if (types[i].getType() != Type.NULL_TYPE) {
Type candidate = TUtil.getFromNestedMap(OPERATION_CASTING_MAP, widest.getType(), types[i].getType());
if (candidate == null) {
- throw new InvalidOperationException("No matched operation for those types: " + TUtil.arrayToString
+ throw new InvalidOperationException("No matched operation for those types: " + StringUtils.join
(types));
}
widest = newSimpleDataType(candidate);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
new file mode 100644
index 0000000..58b4f26
--- /dev/null
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.catalog;
+
+import com.google.common.base.Preconditions;
+import org.apache.tajo.common.TajoDataTypes.Type;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Utility methods for nested field
+ */
+public class NestedPathUtil {
+ public static final String PATH_DELIMITER = "/";
+
+ public static final List<String> ROOT_PATH = Collections.unmodifiableList(new ArrayList<String>());
+
+ public static boolean isPath(String name) {
+ return name.indexOf(PATH_DELIMITER.charAt(0)) >= 0;
+ }
+
+ public static String makePath(String[] parts) {
+ return makePath(parts, 0);
+ }
+
+ public static String makePath(String[] parts, int startIndex) {
+ return makePath(parts, startIndex, parts.length);
+ }
+
+ /**
+ * Make a nested field path
+ *
+ * @param parts path parts
+ * @param startIndex startIndex
+ * @param depth Depth
+ * @return Path
+ */
+ public static String makePath(String[] parts, int startIndex, int depth) {
+ Preconditions.checkArgument(startIndex <= (parts.length - 1));
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = startIndex; i < depth; i++) {
+ sb.append(PATH_DELIMITER);
+ sb.append(parts[i].toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Lookup the actual column corresponding to a given path.
+ * We assume that a path starts with the slash '/' and it
+ * does not include the root field.
+ *
+ * @param nestedField Nested column
+ * @param path Path which starts with '/';
+ * @return Column corresponding to the path
+ */
+ public static Column lookupPath(Column nestedField, String path) {
+ Preconditions.checkArgument(path.charAt(0) == PATH_DELIMITER.charAt(0),
+ "A nested field path must start with slash '/'.");
+
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupPath(nestedField, path.split(PATH_DELIMITER));
+ }
+
+ public static Column lookupPath(Column nestedField, String [] paths) {
+ // We assume that path starts with '/', causing an empty string "" at 0 in the path splits.
+ // So, we should start the index from 1 instead of 0.
+ return lookupColumnInternal(nestedField, paths, 1);
+ }
+
+ private static Column lookupColumnInternal(Column currentColumn, String [] paths, int depth) {
+ Column found = null;
+
+ if (currentColumn.getDataType().getType() == Type.RECORD) {
+ found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]);
+ }
+
+ if (found != null) {
+ if (found.getDataType().getType() == Type.RECORD) {
+ return lookupColumnInternal(found, paths, depth + 1);
+ } else {
+ return found;
+ }
+ } else {
+ throw new NoSuchFieldError(makePath(paths));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
index 054cc2c..0e4b741 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java
@@ -32,6 +32,7 @@ import org.apache.tajo.common.ProtoObject;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.json.GsonObject;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -135,7 +136,8 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @param qualifier The qualifier
*/
public void setQualifier(String qualifier) {
- List<Column> columns = getColumns();
+ // only change root fields, and must keep each nested field simple name
+ List<Column> columns = getRootColumns();
fields.clear();
fieldsByQualifiedName.clear();
@@ -180,14 +182,39 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
* @return The column matched to a given column name.
*/
public Column getColumn(String name) {
- String [] parts = name.split("\\.");
- // Some of the string can includes database name and table name and column name.
- // For example, it can be 'default.table1.id'.
- // Therefore, spilt string array length can be 3.
- if (parts.length >= 2) {
- return getColumnByQName(name);
+
+ if (NestedPathUtil.isPath(name)) {
+
+ // TODO - to be refactored
+ if (fieldsByQualifiedName.containsKey(name)) {
+ Column flattenColumn = fields.get(fieldsByQualifiedName.get(name));
+ if (flattenColumn != null) {
+ return flattenColumn;
+ }
+ }
+
+ String [] paths = name.split(NestedPathUtil.PATH_DELIMITER);
+ Column column = getColumn(paths[0]);
+ if (column == null) {
+ return null;
+ }
+ Column actualColumn = NestedPathUtil.lookupPath(column, paths);
+
+ Column columnPath = new Column(
+ column.getQualifiedName() + NestedPathUtil.makePath(paths, 1),
+ actualColumn.typeDesc);
+
+ return columnPath;
} else {
- return getColumnByName(name);
+ String[] parts = name.split("\\.");
+ // Some of the string can includes database name and table name and column name.
+ // For example, it can be 'default.table1.id'.
+ // Therefore, spilt string array length can be 3.
+ if (parts.length >= 2) {
+ return getColumnByQName(name);
+ } else {
+ return getColumnByName(name);
+ }
}
}
@@ -268,12 +295,46 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
return -1;
}
-
- public List<Column> getColumns() {
+
+ /**
+ * Get root columns, meaning all columns except for nested fields.
+ *
+ * @return A list of root columns
+ */
+ public List<Column> getRootColumns() {
return ImmutableList.copyOf(fields);
}
+ /**
+ * Get all columns, including all nested fields
+ *
+ * @return A list of all columns
+ */
+ public List<Column> getAllColumns() {
+ final List<Column> columnList = TUtil.newList();
+
+ SchemaUtil.visitSchema(this, new ColumnVisitor() {
+ @Override
+ public void visit(int depth, List<String> path, Column column) {
+ if (path.size() > 0) {
+ String parentPath = StringUtils.join(path, NestedPathUtil.PATH_DELIMITER);
+ String currentPath = parentPath + NestedPathUtil.PATH_DELIMITER + column.getSimpleName();
+ columnList.add(new Column(currentPath, column.getTypeDesc()));
+ } else {
+ columnList.add(column);
+ }
+ }
+ });
+
+ return columnList;
+ }
+
public boolean contains(String name) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(name)) {
+ return (getColumn(name) != null);
+ }
+
if (fieldsByQualifiedName.containsKey(name)) {
return true;
}
@@ -288,6 +349,11 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean contains(Column column) {
+ // TODO - It's a hack
+ if (NestedPathUtil.isPath(column.getQualifiedName())) {
+ return (getColumn(column.getQualifiedName()) != null);
+ }
+
if (column.hasQualifier()) {
return fieldsByQualifiedName.containsKey(column.getQualifiedName());
} else {
@@ -314,7 +380,24 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public boolean containsAll(Collection<Column> columns) {
- return fields.containsAll(columns);
+ boolean containFlag = true;
+
+ for (Column c :columns) {
+ if (NestedPathUtil.isPath(c.getSimpleName())) {
+ if (contains(c.getQualifiedName())) {
+ containFlag &= true;
+ } else {
+ String[] paths = c.getQualifiedName().split("/");
+ boolean existRootPath = contains(paths[0]);
+ boolean existLeafPath = getColumn(c.getSimpleName()) != null;
+ containFlag &= existRootPath && existLeafPath;
+ }
+ } else {
+ containFlag &= fields.contains(c);
+ }
+ }
+
+ return containFlag;
}
public synchronized Schema addColumn(String name, TypeDesc typeDesc) {
@@ -351,7 +434,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
public synchronized void addColumns(Schema schema) {
- for(Column column : schema.getColumns()) {
+ for(Column column : schema.getRootColumns()) {
addColumn(column);
}
}
@@ -396,7 +479,7 @@ public class Schema implements ProtoObject<SchemaProto>, Cloneable, GsonObject {
}
@Override
- public void visit(int depth, Column column) {
+ public void visit(int depth, List<String> path, Column column) {
if (column.getDataType().getType() == Type.RECORD) {
DataType.Builder updatedType = DataType.newBuilder(column.getDataType());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index f2bb71c..c6b2f69 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -18,6 +18,10 @@
package org.apache.tajo.catalog;
+import org.apache.tajo.util.TUtil;
+
+import java.util.List;
+
import static org.apache.tajo.common.TajoDataTypes.DataType;
import static org.apache.tajo.common.TajoDataTypes.Type;
@@ -34,12 +38,12 @@ public class SchemaUtil {
static int tmpColumnSeq = 0;
public static Schema merge(Schema left, Schema right) {
Schema merged = new Schema();
- for(Column col : left.getColumns()) {
+ for(Column col : left.getRootColumns()) {
if (!merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn(col);
}
}
- for(Column col : right.getColumns()) {
+ for(Column col : right.getRootColumns()) {
if (merged.containsByQualifiedName(col.getQualifiedName())) {
merged.addColumn("?fake" + (tmpColumnSeq++), col.getDataType());
} else {
@@ -59,7 +63,7 @@ public class SchemaUtil {
*/
public static Schema getNaturalJoinColumns(Schema left, Schema right) {
Schema common = new Schema();
- for (Column outer : left.getColumns()) {
+ for (Column outer : left.getRootColumns()) {
if (!common.containsByName(outer.getSimpleName()) && right.containsByName(outer.getSimpleName())) {
common.addColumn(new Column(outer.getSimpleName(), outer.getDataType()));
}
@@ -113,7 +117,7 @@ public class SchemaUtil {
* Column visitor interface
*/
public static interface ColumnVisitor {
- public void visit(int depth, Column column);
+ public void visit(int depth, List<String> path, Column column);
}
/**
@@ -122,8 +126,8 @@ public class SchemaUtil {
* @param function
*/
public static void visitSchema(Schema schema, ColumnVisitor function) {
- for(Column col : schema.getColumns()) {
- visitInDepthFirstOrder(0, function, col);
+ for(Column col : schema.getRootColumns()) {
+ visitInDepthFirstOrder(0, NestedPathUtil.ROOT_PATH, function, col);
}
}
@@ -134,14 +138,21 @@ public class SchemaUtil {
* @param function Visitor
* @param column Current visiting column
*/
- private static void visitInDepthFirstOrder(int depth, ColumnVisitor function, Column column) {
+ private static void visitInDepthFirstOrder(int depth,
+ final List<String> path,
+ ColumnVisitor function,
+ Column column) {
+
if (column.getDataType().getType() == Type.RECORD) {
- for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getColumns()) {
- visitInDepthFirstOrder(depth + 1, function, nestedColumn);
+ for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) {
+ List<String> newPath = TUtil.newList(path);
+ newPath.add(column.getQualifiedName());
+
+ visitInDepthFirstOrder(depth + 1, newPath, function, nestedColumn);
}
- function.visit(depth, column);
+ function.visit(depth, path, column);
} else {
- function.visit(depth, column);
+ function.visit(depth, path, column);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
index 5b1a996..6acdbd1 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
@@ -453,7 +453,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
}
// set column information
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(columns.size());
for (Column eachField : columns) {
@@ -465,7 +465,7 @@ public class HiveCatalogStore extends CatalogConstants implements CatalogStore {
// set partition keys
if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) {
List<FieldSchema> partitionKeys = new ArrayList<FieldSchema>();
- for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(),
HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), ""));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
index a3960e6..a5b53f2 100644
--- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
+++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java
@@ -27,7 +27,7 @@ import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.cli.tsql.TajoCli;
import org.apache.tajo.util.FileUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
import java.util.Map;
@@ -126,7 +126,7 @@ public class DescTableCommand extends TajoShellCommand {
sb.append("type:").append(partition.getPartitionType().name()).append("\n");
sb.append("columns:").append(":");
- sb.append(TUtil.arrayToString(partition.getExpressionSchema().toArray()));
+ sb.append(StringUtils.join(partition.getExpressionSchema().toArray()));
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 385f99c..6e16095 100644
--- a/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -36,7 +36,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
index aec784f..fce10a3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
@@ -41,8 +41,8 @@ public interface Tuple extends Cloneable {
void put(Datum[] values);
+
Datum get(int fieldId);
-
void setOffset(long offset);
long getOffset();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index d035e4a..0a16072 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -371,4 +371,74 @@ public class StringUtils {
return resultArray;
}
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects Iterable objects
+ * @param delimiter Delimiter string
+ * @return A joined string
+ */
+ public static String join(Iterable objects, String delimiter) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(Object object : objects) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(object.toString());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenate all objects' string with the delimiter ", "
+ *
+ * @param objects Iterable objects
+ * @return A joined string
+ */
+ public static String join(Object[] objects) {
+ return join(objects, ", ", 0, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex) {
+ return join(objects, delimiter, startIndex, objects.length);
+ }
+
+ /**
+ * Concatenate all objects' string with a delimiter string
+ *
+ * @param objects object array
+ * @param delimiter Delimiter string
+ * @param startIndex the begin index to join
+ * @param length the number of columns to be joined
+ * @return A joined string
+ */
+ public static String join(Object[] objects, String delimiter, int startIndex, int length) {
+ boolean first = true;
+ StringBuilder sb = new StringBuilder();
+ for(int i = startIndex; i + startIndex < length; i++) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(delimiter);
+ }
+
+ sb.append(objects[i].toString());
+ }
+
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
index 75f3e2a..2293ef5 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java
@@ -240,38 +240,6 @@ public class TUtil {
}
}
- public static String collectionToString(Collection objects, String delimiter) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(delimiter);
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
- public static String arrayToString(Object [] objects) {
- boolean first = true;
- StringBuilder sb = new StringBuilder();
- for(Object object : objects) {
- if (first) {
- first = false;
- } else {
- sb.append(", ");
- }
-
- sb.append(object.toString());
- }
-
- return sb.toString();
- }
-
public static <T> T [] toArray(Collection<T> collection, Class<T> type) {
T array = (T) Array.newInstance(type, collection.size());
return collection.toArray((T[]) array);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
index 1a63470..3ab11bd 100644
--- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
+++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4
@@ -1307,7 +1307,7 @@ set_qualifier
;
column_reference
- : ((db_name = identifier DOT)? (tb_name=identifier DOT))? name=identifier
+ : identifier (DOT identifier)*
;
as_clause
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
index 6d32fa5..7c99868 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java
@@ -1054,14 +1054,17 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> {
@Override
public ColumnReferenceExpr visitColumn_reference(SQLParser.Column_referenceContext ctx) {
- ColumnReferenceExpr column = new ColumnReferenceExpr(ctx.name.getText());
- if (checkIfExist(ctx.db_name)) {
- column.setQualifier(CatalogUtil.buildFQName(ctx.db_name.getText(), ctx.tb_name.getText()));
- } else if (ctx.tb_name != null) {
- column.setQualifier(ctx.tb_name.getText());
+ String columnReferenceName = ctx.getText();
+ // find the last dot (.) position to separate a name into both a qualifier and name
+ int lastDotIdx = columnReferenceName.lastIndexOf(".");
+
+ if (lastDotIdx > 0) { // if any qualifier is given
+ String qualifier = columnReferenceName.substring(0, lastDotIdx);
+ String name = columnReferenceName.substring(lastDotIdx + 1, columnReferenceName.length());
+ return new ColumnReferenceExpr(qualifier, name);
+ } else {
+ return new ColumnReferenceExpr(ctx.getText());
}
-
- return column;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index f132793..506b03e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -55,6 +55,7 @@ import org.apache.tajo.storage.fragment.Fragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.IndexUtil;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -275,7 +276,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.",
context.getTaskId().toString(),
(left ? "Left" : "Right"),
- TUtil.arrayToString(lineage),
+ StringUtils.join(lineage),
FileUtil.humanReadableByteCount(volume, false),
(inMemoryInnerJoinFlag ? "" : "not ")));
return inMemoryInnerJoinFlag;
@@ -398,18 +399,18 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
larger = right;
LOG.info(String.format("[%s] Left relations %s (%s) is smaller than Right relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false)));
} else {
smaller = right;
larger = left;
LOG.info(String.format("[%s] Right relations %s (%s) is smaller than Left relations %s (%s).",
context.getTaskId().toString(),
- TUtil.arrayToString(rightLineage),
+ StringUtils.join(rightLineage),
FileUtil.humanReadableByteCount(rightSize, false),
- TUtil.arrayToString(leftLineage),
+ StringUtils.join(leftLineage),
FileUtil.humanReadableByteCount(leftSize, false)));
}
@@ -858,7 +859,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
} else if (storeTableNode.getType() == NodeType.CREATE_TABLE) {
int i = 0;
for (int j = 0; j < partitionKeyColumns.length; j++) {
- int id = storeTableNode.getOutSchema().getColumns().size() + j;
+ int id = storeTableNode.getOutSchema().getRootColumns().size() + j;
Column column = storeTableNode.getInSchema().getColumn(id);
sortSpecs[i++] = new SortSpec(column, true, false);
}
@@ -1002,7 +1003,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
ExternalSortExec sortExec = new ExternalSortExec(ctx, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
return new SortAggregateExec(ctx, groupbyNode, sortExec);
}
@@ -1043,7 +1044,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
sortNode.setInSchema(subOp.getSchema());
sortNode.setOutSchema(subOp.getSchema());
child = new ExternalSortExec(context, sortNode, subOp);
- LOG.info("The planner chooses [Sort Aggregation] in (" + TUtil.arrayToString(sortSpecs) + ")");
+ LOG.info("The planner chooses [Sort Aggregation] in (" + StringUtils.join(sortSpecs) + ")");
}
return new WindowAggExec(context, windowAggNode, child);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
index 11548d3..ba1b0bf 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java
@@ -23,7 +23,7 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import static org.apache.tajo.ipc.TajoWorkerProtocol.*;
@@ -193,7 +193,7 @@ public class DataChannel {
sb.append(" (type=").append(shuffleType);
if (hasShuffleKeys()) {
sb.append(", key=");
- sb.append(TUtil.arrayToString(shuffleKeys));
+ sb.append(StringUtils.join(shuffleKeys));
sb.append(", num=").append(numOutputs);
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index 36bdf21..ff1955e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -1146,11 +1146,11 @@ public class GlobalPlanner {
Column[] shuffleKeys = new Column[partitionMethod.getExpressionSchema().size()];
int i = 0, id = 0;
- for (Column column : partitionMethod.getExpressionSchema().getColumns()) {
+ for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) {
if (node.getType() == NodeType.INSERT) {
id = tableSchema.getColumnId(column.getQualifiedName());
} else {
- id = tableSchema.getColumns().size() + i;
+ id = tableSchema.getRootColumns().size() + i;
}
shuffleKeys[i++] = projectedSchema.getColumn(id);
}
@@ -1493,7 +1493,7 @@ public class GlobalPlanner {
addedTableSubQueries.add(copy);
//Find a SubQueryNode which contains all columns in InputSchema matched with Target and OutputSchema's column
- if (copy.getInSchema().containsAll(copy.getOutSchema().getColumns())) {
+ if (copy.getInSchema().containsAll(copy.getOutSchema().getRootColumns())) {
for (Target eachTarget : copy.getTargets()) {
Set<Column> columns = EvalTreeUtil.findUniqueColumns(eachTarget.getEvalTree());
if (copy.getInSchema().containsAll(columns)) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
index b5e9104..13ed99b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java
@@ -81,13 +81,13 @@ public class DistinctGroupbyBuilder {
// If there is not grouping column, we can't find column alias.
// Thus we should find the alias at Groupbynode output schema.
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
aggFunctions[i].setAlias(groupbyNode.getOutSchema().getColumn(i).getQualifiedName());
}
}
if (groupbyNode.getGroupingColumns().length == 0
- && aggFunctions.length == groupbyNode.getOutSchema().getColumns().size()) {
+ && aggFunctions.length == groupbyNode.getOutSchema().getRootColumns().size()) {
groupbyNode.setAggFunctions(aggFunctions);
}
@@ -672,7 +672,7 @@ public class DistinctGroupbyBuilder {
int index = 0;
for(GroupbyNode eachNode: secondStageDistinctNode.getSubPlans()) {
eachNode.setInSchema(firstStageDistinctNode.getOutSchema());
- for (Column column: eachNode.getOutSchema().getColumns()) {
+ for (Column column: eachNode.getOutSchema().getRootColumns()) {
if (secondStageInSchema.getColumn(column) == null) {
secondStageInSchema.addColumn(column);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
index e71976c..7c38d36 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
@@ -110,7 +110,7 @@ public class DistinctGroupbyThirdAggregationExec extends UnaryPhysicalExec {
}
int index = 0;
- for (Column eachOutputColumn: outSchema.getColumns()) {
+ for (Column eachOutputColumn: outSchema.getRootColumns()) {
// If column is avg aggregation function, outschema's column type is float
// but groupbyResultTupleIndex's column type is protobuf
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index 247b373..7694b2b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -77,7 +77,7 @@ public class PhysicalPlanUtil {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
index ff9477f..3d95068 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
@@ -138,6 +138,8 @@ public class SeqScanExec extends ScanExec {
public void init() throws IOException {
Schema projected;
+ // in the case where projected column or expression are given
+ // the target can be an empty list.
if (plan.hasTargets()) {
projected = new Schema();
Set<Column> columnSet = new HashSet<Column>();
@@ -150,12 +152,15 @@ public class SeqScanExec extends ScanExec {
columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree()));
}
- for (Column column : inSchema.getColumns()) {
+ for (Column column : inSchema.getAllColumns()) {
if (columnSet.contains(column)) {
projected.addColumn(column);
}
}
+
} else {
+ // no any projected columns, meaning that all columns should be projected.
+ // TODO - this implicit rule makes code readability bad. So, we should remove it later
projected = outSchema;
}
@@ -163,7 +168,11 @@ public class SeqScanExec extends ScanExec {
super.init();
if (plan.hasQual()) {
- qual.bind(context.getEvalContext(), inSchema);
+ if (scanner.isProjectable()) {
+ qual.bind(context.getEvalContext(), projected);
+ } else {
+ qual.bind(context.getEvalContext(), inSchema);
+ }
}
}
@@ -175,7 +184,7 @@ public class SeqScanExec extends ScanExec {
}
private void initScanner(Schema projected) throws IOException {
- this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+
TableMeta meta;
try {
meta = (TableMeta) plan.getTableDesc().getMeta().clone();
@@ -186,6 +195,7 @@ public class SeqScanExec extends ScanExec {
// set system default properties
PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta);
+ // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422
if (fragments != null) {
if (fragments.length > 1) {
this.scanner = new MergeScanner(context.getConf(), plan.getPhysicalSchema(), meta,
@@ -198,6 +208,17 @@ public class SeqScanExec extends ScanExec {
plan.getPhysicalSchema(), fragments[0], projected);
}
scanner.init();
+
+ // See Scanner.isProjectable() method Depending on the result of isProjectable(),
+ // the width of retrieved tuple is changed.
+ //
+ // If TRUE, the retrieved tuple will contain only projected fields.
+ // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected fields.
+ if (scanner.isProjectable()) {
+ this.projector = new Projector(context, projected, outSchema, plan.getTargets());
+ } else {
+ this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
index ba25172..0fd2fbe 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleCacheScanner.java
@@ -73,7 +73,7 @@ public class TupleCacheScanner implements Scanner {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
index 3a0a1c7..c01900a 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
@@ -89,7 +89,7 @@ public class TupleUtil {
}
int i = 0;
- for (Column col : sortSchema.getColumns()) {
+ for (Column col : sortSchema.getRootColumns()) {
ColumnStats columnStat = statMap.get(col);
if (columnStat == null) {
continue;
@@ -121,7 +121,7 @@ public class TupleUtil {
statSet.put(stat.getColumn(), stat);
}
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
Preconditions.checkState(statSet.containsKey(col),
"ERROR: Invalid Column Stats (column stats: " + colStats + ", there exists not target " + col);
}
@@ -134,7 +134,7 @@ public class TupleUtil {
// In outer join, empty table could be searched.
// As a result, min value and max value would be null.
// So, we should put NullDatum for this case.
- for (Column col : target.getColumns()) {
+ for (Column col : target.getRootColumns()) {
if (sortSpecs[sortSpecIndex].isAscending()) {
if (statSet.get(col).getMinValue() != null)
startTuple.put(i, statSet.get(col).getMinValue());
@@ -169,7 +169,7 @@ public class TupleUtil {
else
endTuple.put(i, DatumFactory.createNullDatum());
}
- if (target.getColumns().size() == sortSpecs.length) {
+ if (target.getRootColumns().size() == sortSpecs.length) {
// Not composite column sort
sortSpecIndex++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
index 804821b..9c0bd48 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java
@@ -89,7 +89,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc
StringBuffer path = new StringBuffer();
int depth = 0;
if (tableDesc.hasPartition()) {
- for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) {
String partitionValue = EvalTreeUtil.getPartitionValue(scanNode.getQual(), c.getSimpleName());
if (partitionValue == null)
break;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
index 958c252..6c9b485 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
@@ -144,7 +144,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTablespaces(Schema outSchema) {
List<TablespaceProto> tablespaces = masterContext.getCatalog().getAllTablespaces();
List<Tuple> tuples = new ArrayList<Tuple>(tablespaces.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablespaceProto tablespace: tablespaces) {
@@ -179,7 +179,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getDatabases(Schema outSchema) {
List<DatabaseProto> databases = masterContext.getCatalog().getAllDatabases();
List<Tuple> tuples = new ArrayList<Tuple>(databases.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (DatabaseProto database: databases) {
@@ -209,7 +209,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getTables(Schema outSchema) {
List<TableDescriptorProto> tables = masterContext.getCatalog().getAllTables();
List<Tuple> tuples = new ArrayList<Tuple>(tables.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableDescriptorProto table: tables) {
@@ -245,7 +245,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getColumns(Schema outSchema) {
List<ColumnProto> columnsList = masterContext.getCatalog().getAllColumns();
List<Tuple> tuples = new ArrayList<Tuple>(columnsList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
int columnId = 1, prevtid = -1, tid = 0;
@@ -293,7 +293,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getIndexes(Schema outSchema) {
List<IndexProto> indexList = masterContext.getCatalog().getAllIndexes();
List<Tuple> tuples = new ArrayList<Tuple>(indexList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (IndexProto index: indexList) {
@@ -332,7 +332,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableOptions(Schema outSchema) {
List<TableOptionProto> optionList = masterContext.getCatalog().getAllTableOptions();
List<Tuple> tuples = new ArrayList<Tuple>(optionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableOptionProto option: optionList) {
@@ -359,7 +359,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllTableStats(Schema outSchema) {
List<TableStatsProto> statList = masterContext.getCatalog().getAllTableStats();
List<Tuple> tuples = new ArrayList<Tuple>(statList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TableStatsProto stat: statList) {
@@ -386,7 +386,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
private List<Tuple> getAllPartitions(Schema outSchema) {
List<TablePartitionProto> partitionList = masterContext.getCatalog().getAllPartitions();
List<Tuple> tuples = new ArrayList<Tuple>(partitionList.size());
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple;
for (TablePartitionProto partition: partitionList) {
@@ -417,7 +417,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getQueryMasterTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
@@ -463,7 +463,7 @@ public class NonForwardQueryResultSystemScanner implements NonForwardQueryResult
}
private Tuple getWorkerTuple(Schema outSchema, Worker aWorker) {
- List<Column> columns = outSchema.getColumns();
+ List<Column> columns = outSchema.getRootColumns();
Tuple aTuple = new VTuple(outSchema.size());
WorkerResource aResource = aWorker.getResource();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
index 1ff81a6..e014379 100644
--- a/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/catalogview.jsp
@@ -30,7 +30,6 @@
<%@ page import="java.util.Collection" %>
<%@ page import="java.util.List" %>
<%@ page import="java.util.Map" %>
-<%@ page import="org.apache.tajo.service.ServiceTracker" %>
<%
TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
CatalogService catalog = master.getCatalog();
@@ -143,7 +142,7 @@
<div style='margin-top:5px'>
<%
if(tableDesc != null) {
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
out.write("<table border='1' class='border_table'><tr><th>No</th><th>Column name</th><th>Type</th></tr>");
int columnIndex = 1;
for(Column eachColumn: columns) {
@@ -155,7 +154,7 @@
if (tableDesc.getPartitionMethod() != null) {
PartitionMethodDesc partition = tableDesc.getPartitionMethod();
- List<Column> partitionColumns = partition.getExpressionSchema().getColumns();
+ List<Column> partitionColumns = partition.getExpressionSchema().getRootColumns();
String partitionColumnStr = "";
String prefix = "";
for (Column eachColumn: partitionColumns) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
index 32b98dd..cf34c33 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestFunctionLoader.java
@@ -20,8 +20,7 @@ package org.apache.tajo.engine.function;
import com.google.common.collect.Lists;
import org.apache.tajo.catalog.FunctionDesc;
-import org.apache.tajo.function.FunctionSignature;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import org.junit.Test;
import java.io.IOException;
@@ -38,7 +37,7 @@ public class TestFunctionLoader {
public void testFindScalarFunctions() throws IOException {
List<FunctionDesc> collections = Lists.newArrayList(FunctionLoader.findScalarFunctions());
Collections.sort(collections);
- String functionList = TUtil.collectionToString(collections, "\n");
+ String functionList = StringUtils.join(collections, "\n");
String result = getResultText(TestFunctionLoader.class, "testFindScalarFunctions.result");
assertEquals(result.trim(), functionList.trim());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index af0aa6a..dfac53d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -871,7 +871,7 @@ public class TestLogicalPlanner {
assertEquals(NodeType.EXPRS, root.getChild().getType());
Schema out = root.getOutSchema();
- Iterator<Column> it = out.getColumns().iterator();
+ Iterator<Column> it = out.getRootColumns().iterator();
Column col = it.next();
assertEquals("res1", col.getSimpleName());
col = it.next();
@@ -920,7 +920,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("deptname", col.getSimpleName());
col = it.next();
@@ -931,7 +931,7 @@ public class TestLogicalPlanner {
root = (LogicalRootNode) plan;
finalSchema = root.getOutSchema();
- it = finalSchema.getColumns().iterator();
+ it = finalSchema.getRootColumns().iterator();
col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
@@ -948,7 +948,7 @@ public class TestLogicalPlanner {
testJsonSerDerObject(root);
Schema finalSchema = root.getOutSchema();
- Iterator<Column> it = finalSchema.getColumns().iterator();
+ Iterator<Column> it = finalSchema.getRootColumns().iterator();
Column col = it.next();
assertEquals("id", col.getSimpleName());
col = it.next();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index f79f703..727b1a2 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -68,7 +68,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1"));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
Path path = new Path(desc.getPath());
@@ -111,7 +111,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
- assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
+ assertEquals("key", partitionDesc.getExpressionSchema().getRootColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(cluster.getConfiguration());
Path path = new Path(desc.getPath());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
index 48aea26..a1eceea 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
@@ -707,7 +707,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
"select a.o_orderdate, a.o_orderstatus, a.o_orderkey_mod, a.o_totalprice " +
"from " + tableName +
" a join "+ tableName + " b on a.o_orderkey = b.o_orderkey " +
- "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and o_orderkey_mod = 1 " +
+ "where a.o_orderdate = '1993-10-14' and a.o_orderstatus = 'F' and a.o_orderkey_mod = 1 " +
" order by a.o_orderkey"
);
String resultSetData = resultSetToString(res);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
new file mode 100644
index 0000000..9f8a5fd
--- /dev/null
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.engine.query;
+
+import org.apache.tajo.QueryTestCaseBase;
+import org.apache.tajo.util.TUtil;
+import org.junit.Test;
+
+import java.sql.ResultSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestSelectNestedRecord extends QueryTestCaseBase {
+
+ @Test
+ public final void testSelect1() throws Exception {
+ List<String> tables = executeDDL("sample1_ddl.sql", "sample1", "sample1");
+ assertEquals(TUtil.newList("sample1"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testSelect2() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsGroupbyKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
+ public final void testNestedFieldAsJoinKey1() throws Exception {
+ List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
+ assertEquals(TUtil.newList("tweets"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index 0dbb8e5..c8c24cd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -262,7 +262,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
TableDesc tableDesc = client.getTableDesc(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName));
assertNotNull(tableDesc);
- List<Column> columns = tableDesc.getSchema().getColumns();
+ List<Column> columns = tableDesc.getSchema().getRootColumns();
while (rs.next()) {
assertEquals(tableName, rs.getString("TABLE_NAME"));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
index 6fc4ea1..1b23966 100644
--- a/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
+++ b/tajo-core/src/test/java/org/apache/tajo/ws/rs/resources/TestQueryResultResource.java
@@ -140,7 +140,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -174,7 +174,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
@@ -236,7 +236,7 @@ public class TestQueryResultResource extends QueryTestCaseBase {
assertNotNull(response.getResultCode());
assertEquals(ResultCode.OK, response.getResultCode());
assertNotNull(response.getSchema());
- assertEquals(16, response.getSchema().getColumns().size());
+ assertEquals(16, response.getSchema().getRootColumns().size());
assertNotNull(response.getResultset());
assertTrue(response.getResultset().getId() != 0);
assertNotNull(response.getResultset().getLink());
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
new file mode 100644
index 0000000..db3ad6c
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample1/table.json
@@ -0,0 +1,3 @@
+{ "title" : "Hand of the King", "name" : { "first_name": "Eddard", "last_name": "Stark"}}
+{ "title" : "Assassin", "name" : { "first_name": "Arya", "last_name": "Stark"}}
+{ "title" : "Dancing Master", "name" : { "first_name": "Syrio", "last_name": "Forel"}}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
new file mode 100644
index 0000000..78a9071
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/tweets/sample1.json
@@ -0,0 +1,4 @@
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Mon Sep 24 03:35:21 +0000 2012","id_str":"250075927172759552","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Aggressive Ponytail #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":1,"in_reply_to_status_id_str":null,"id":250075927172759552,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDEEF6","profile_sidebar_border_color":"C0DEED","profile_background_tile":false,"name":"Sean Cummings","profile_image_url":"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","created_at":"Mon Apr 26 06:01:55 +0000 2010","location":"LA, CA","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"137238150","entities":{"url":{"urls":[{"expanded_url":null,"url":"","i
ndices":[0,0]}]},"description":{"urls":[]}},"default_profile":true,"contributors_enabled":false,"favourites_count":0,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg","utc_offset":-28800,"id":137238150,"profile_use_background_image":true,"listed_count":2,"profile_text_color":"333333","lang":"en","followers_count":70,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme1/bg.png","profile_background_color":"C0DEED","verified":false,"geo_enabled":true,"time_zone":"Pacific Time (US & Canada)","description":"Born 330 Live 310","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme1/bg.png","statuses_count":579,"friends_count":110,"following":null,"show_all_inline_media":false,"screen_name":"sean_cummings"},"in_reply_to_screen_name":null,"source":"<a>Twitter for Mac<\/a>","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:40:54 +0000 2012","id_str":"249292149810667520","entities":{"urls":[],"hashtags":[{"text":"FreeBandNames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Thee Namaste Nerdz. #FreeBandNames","metadata":{"iso_language_code":"pl","result_type":"recent"},"retweet_count":2,"in_reply_to_status_id_str":null,"id":249292149810667520,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"DDFFCC","profile_sidebar_border_color":"BDDCAD","profile_background_tile":true,"name":"Chaz Martenstein","profile_image_url":"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","created_at":"Tue Apr 07 19:05:07 +0000 2009","location":"Durham, NC","follow_request_sent":null,"profile_link_color":"0084B4","is_translator":false,"id_str":"29516238","entities":{"url":{"urls":[{"expanded_url":null,"url":"http://bu
llcityrecords.com/wnng/","indices":[0,32]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":8,"url":"http://bullcityrecords.com/wnng/","profile_image_url_https":"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg","utc_offset":-18000,"id":29516238,"profile_use_background_image":true,"listed_count":118,"profile_text_color":"333333","lang":"en","followers_count":2052,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp","profile_background_color":"9AE4E8","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp","statuses_c
ount":7579,"friends_count":348,"following":null,"show_all_inline_media":true,"screen_name":"bullcityrecords"},"in_reply_to_screen_name":null,"source":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 23:30:20 +0000 2012","id_str":"249289491129438208","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[29,43]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"Mexican Heaven, Mexican Hell #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":3,"in_reply_to_status_id_str":null,"id":249289491129438208,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"99CC33","profile_sidebar_border_color":"829D5E","profile_background_tile":false,"name":"Thomas John Wakeman","profile_image_url":"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","created_at":"Tue Sep 01 21:21:35 +0000 2009","location":"Kingston New York","follow_request_sent":null,"profile_link_color":"D02B55","is_translator":false,"id_str":"70789458","entities":{"url":{"urls":[{"expanded_url":n
ull,"url":"","indices":[0,0]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":19,"url":null,"profile_image_url_https":"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png","utc_offset":-18000,"id":70789458,"profile_use_background_image":true,"listed_count":1,"profile_text_color":"3E4415","lang":"en","followers_count":63,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/images/themes/theme5/bg.gif","profile_background_color":"352726","verified":false,"geo_enabled":false,"time_zone":"Eastern Time (US & Canada)","description":"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/images/themes/theme5/bg.gif","statuses_count":1048,"friends_count":63,"following":null,"show_all_inline_media":false,"screen_name":"MonkiesFist"},"in_reply_to_screen_name":null,"sour
ce":"web","in_reply_to_status_id":null}
+{"coordinates":null,"favorited":false,"truncated":false,"created_at":"Fri Sep 21 22:51:18 +0000 2012","id_str":"249279667666817024","entities":{"urls":[],"hashtags":[{"text":"freebandnames","indices":[20,34]}],"user_mentions":[]},"in_reply_to_user_id_str":null,"contributors":null,"text":"The Foolish Mortals #freebandnames","metadata":{"iso_language_code":"en","result_type":"recent"},"retweet_count":4,"in_reply_to_status_id_str":null,"id":249279667666817024,"geo":null,"retweeted":false,"in_reply_to_user_id":null,"place":null,"user":{"profile_sidebar_fill_color":"BFAC83","profile_sidebar_border_color":"615A44","profile_background_tile":true,"name":"Marty Elmer","profile_image_url":"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","created_at":"Mon May 04 00:05:00 +0000 2009","location":"Wisconsin, USA","follow_request_sent":null,"profile_link_color":"3B2A26","is_translator":false,"id_str":"37539828","entities":{"url":{"urls":[{"expanded_url":null,"url":"ht
tp://www.omnitarian.me","indices":[0,24]}]},"description":{"urls":[]}},"default_profile":false,"contributors_enabled":false,"favourites_count":647,"url":"http://www.omnitarian.me","profile_image_url_https":"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png","utc_offset":-21600,"id":37539828,"profile_use_background_image":true,"listed_count":52,"profile_text_color":"000000","lang":"en","followers_count":608,"protected":false,"notifications":null,"profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png","profile_background_color":"EEE3C4","verified":false,"geo_enabled":false,"time_zone":"Central Time (US & Canada)","description":"Cartoonist, Illustrator, and T-Shirt connoisseur","default_profile_image":false,"profile_background_image_url":"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png","statuses_count":3575,"friends_count":249,"following":null,"show_all_inline_media":true,"scree
n_name":"Omnitarian"},"in_reply_to_screen_name":null,"source":"<a>Twitter for iPhone<\/a>","in_reply_to_status_id":null}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
new file mode 100644
index 0000000..9ba5f8c
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample1_ddl.sql
@@ -0,0 +1,7 @@
+CREATE EXTERNAL TABLE ${0} (
+ title TEXT,
+ name RECORD (
+ first_name TEXT,
+ last_name TEXT
+ )
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
new file mode 100644
index 0000000..9537c3e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
@@ -0,0 +1,19 @@
+CREATE EXTERNAL TABLE ${0} (
+ glossary RECORD (
+ title TEXT,
+ "GlossDiv" RECORD (
+ "GlossEntry" RECORD (
+ "ID" TEXT,
+ "SortAs" TEXT,
+ "GlossTerm" TEXT,
+ "Acronym" TEXT,
+ "Abbrev" TEXT,
+ "GlossDef" RECORD (
+ para TEXT
+ ),
+
+ "GlossSee" TEXT
+ )
+ )
+ )
+) USING JSON LOCATION ${path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
new file mode 100644
index 0000000..057ba05
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ user.name,
+ sum(retweet_count) as total_retweet
+FROM
+ tweets
+GROUP BY
+ user.name;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
new file mode 100644
index 0000000..336840e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsJoinKey1.sql
@@ -0,0 +1,7 @@
+SELECT
+ t1.user.id,
+ t1.user.name,
+ t2.user.id,
+ t2.user.name
+FROM
+ tweets t1 join tweets t2 ON t1.user.id = t2.user.id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
new file mode 100644
index 0000000..b099e77
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect1.sql
@@ -0,0 +1 @@
+SELECT title, (name.first_name || ' ' || name.last_name) as full_name FROM sample1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
new file mode 100644
index 0000000..9993a4e
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect2.sql
@@ -0,0 +1,61 @@
+SELECT
+ coordinates,
+ favorited,
+ truncated,
+ created_at,
+ id_str,
+ in_reply_to_user_id_str,
+ contributors,
+ "text",
+ metadata.iso_language_code,
+ metadata.result_type,
+ retweet_count,
+ in_reply_to_status_id_str,
+ id,
+ geo,
+ retweeted,
+ in_reply_to_user_id,
+ place,
+ user.profile_sidebar_fill_color,
+ user.profile_sidebar_border_color,
+ user.profile_background_tile,
+ user.name,
+ user.profile_image_url,
+ user.created_at,
+ user.location,
+ user.follow_request_sent,
+ user.profile_link_color,
+ user.is_translator,
+ user.id_str,
+ user.default_profile,
+ user.contributors_enabled,
+ user.favourites_count,
+ user.url,
+ user.profile_image_url_https,
+ user.utc_offset,
+ user.id,
+ user.profile_use_background_image,
+ user.listed_count,
+ user.profile_text_color,
+ user.lang,
+ user.followers_count,
+ user.protected,
+ user.notifications,
+ user.profile_background_image_url_https,
+ user.profile_background_color,
+ user.verified,
+ user.geo_enabled,
+ user.time_zone,
+ user.description,
+ user.default_profile_image,
+ user.profile_background_image_url,
+ user.statuses_count,
+ user.friends_count,
+ user.following,
+ user.show_all_inline_media,
+ user.screen_name,
+ in_reply_to_screen_name,
+ source,
+ in_reply_to_status_id
+FROM
+ tweets;
\ No newline at end of file