You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/05/12 05:14:10 UTC
[11/14] tajo git commit: TAJO-1359: Add nested field projector and
language extension to project nested record. (hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
new file mode 100644
index 0000000..e750095
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/tweets_ddl.sql
@@ -0,0 +1,74 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS ${0} (
+ coordinates TEXT,
+ favorited BOOL,
+ truncated BOOL,
+ created_at TIMESTAMP,
+ id_str TEXT,
+ /*entrities RECORD ( -- when we support array, we should remove this comment.
+ urls ARRAY<TEXT>
+ )*/
+ in_reply_to_user_id_str TEXT,
+ contributors TEXT,
+ text TEXT,
+ metadata RECORD (
+ iso_language_code TEXT,
+ result_type TEXT
+ ),
+ retweet_count INTEGER,
+ in_reply_to_status_id_str TEXT,
+ id TEXT,
+ geo TEXT,
+ retweeted BOOL,
+ in_reply_to_user_id TEXT,
+ place TEXT,
+ user RECORD (
+ profile_sidebar_fill_color TEXT,
+ profile_sidebar_border_color TEXT,
+ profile_background_tile TEXT,
+ name TEXT,
+ profile_image_url TEXT,
+ created_at TIMESTAMP,
+ location TEXT,
+ follow_request_sent TEXT,
+ profile_link_color TEXT,
+ is_translator BOOL,
+ id_str TEXT,
+ /* -- when we support array, we should fill the following comments.
+ entities RECORD (
+ url RECORD (
+ ),
+ description RECORD (
+ )
+ ), */
+ default_profile BOOL,
+ contributors_enabled BOOL,
+ favourites_count INTEGER,
+ url TEXT,
+ profile_image_url_https TEXT,
+ utc_offset INTEGER,
+ id BIGINT,
+ profile_use_background_image BOOL,
+ listed_count INTEGER,
+ profile_text_color TEXT,
+ lang TEXT,
+ followers_count INTEGER,
+ protected BOOL,
+ notifications TEXT,
+ profile_background_image_url_https TEXT,
+ profile_background_color TEXT,
+ verified TEXT,
+ geo_enabled TEXT,
+ time_zone TEXT,
+ description TEXT,
+ default_profile_image TEXT,
+ profile_background_image_url TEXT,
+ statuses_count INTEGER,
+ friends_count INTEGER,
+ following TEXT,
+ show_all_inline_media BOOL,
+ screen_name TEXT
+ ),
+ in_reply_to_screen_name TEXT,
+ source TEXT,
+ in_reply_to_status_id TEXT
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
index 1760c99..e707a8c 100644
--- a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectWithParentheses2.sql
@@ -1 +1 @@
-(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n_nationkey = 1);
+(select n1.n_nationkey, n2.n_name from nation n1 join nation n2 on n1.n_nationkey = n2.n_nationkey where n1.n_nationkey = 1);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
new file mode 100644
index 0000000..debf06e
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
@@ -0,0 +1,6 @@
+user/name,total_retweet
+-------------------------------
+Chaz Martenstein,2
+Thomas John Wakeman,3
+Sean Cummings,1
+Marty Elmer,4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
new file mode 100644
index 0000000..1c57dc2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsJoinKey1.result
@@ -0,0 +1,6 @@
+user/id,user/name,user/id,user/name
+-------------------------------
+137238150,Sean Cummings,137238150,Sean Cummings
+29516238,Chaz Martenstein,29516238,Chaz Martenstein
+70789458,Thomas John Wakeman,70789458,Thomas John Wakeman
+37539828,Marty Elmer,37539828,Marty Elmer
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
new file mode 100644
index 0000000..f96fcc5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect1.result
@@ -0,0 +1,5 @@
+title,full_name
+-------------------------------
+Hand of the King,Eddard Stark
+Assassin,Arya Stark
+Dancing Master,Syrio Forel
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
new file mode 100644
index 0000000..90eadbd
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect2.result
@@ -0,0 +1,6 @@
+coordinates,favorited,truncated,created_at,id_str,in_reply_to_user_id_str,contributors,text,metadata/iso_language_code,metadata/result_type,retweet_count,in_reply_to_status_id_str,id,geo,retweeted,in_reply_to_user_id,place,user/profile_sidebar_fill_color,user/profile_sidebar_border_color,user/profile_background_tile,user/name,user/profile_image_url,user/created_at,user/location,user/follow_request_sent,user/profile_link_color,user/is_translator,user/id_str,user/default_profile,user/contributors_enabled,user/favourites_count,user/url,user/profile_image_url_https,user/utc_offset,user/id,user/profile_use_background_image,user/listed_count,user/profile_text_color,user/lang,user/followers_count,user/protected,user/notifications,user/profile_background_image_url_https,user/profile_background_color,user/verified,user/geo_enabled,user/time_zone,user/description,user/default_profile_image,user/profile_background_image_url,user/statuses_count,user/friends_count,user/following,user/show_all_in
line_media,user/screen_name,in_reply_to_screen_name,source,in_reply_to_status_id
+-------------------------------
+null,false,false,2012-09-24 03:35:21,250075927172759552,null,null,Aggressive Ponytail #freebandnames,en,recent,1,null,250075927172759552,null,false,null,null,DDEEF6,C0DEED,false,Sean Cummings,http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,2010-04-26 06:01:55,LA, CA,null,0084B4,false,137238150,true,false,0,null,https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg,-28800,137238150,true,2,333333,en,70,false,null,https://si0.twimg.com/images/themes/theme1/bg.png,C0DEED,false,true,Pacific Time (US & Canada),Born 330 Live 310,false,http://a0.twimg.com/images/themes/theme1/bg.png,579,110,null,false,sean_cummings,null,<a>Twitter for Mac</a>,null
+null,false,false,2012-09-21 23:40:54,249292149810667520,null,null,Thee Namaste Nerdz. #FreeBandNames,pl,recent,2,null,249292149810667520,null,false,null,null,DDFFCC,BDDCAD,true,Chaz Martenstein,http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,2009-04-07 19:05:07,Durham, NC,null,0084B4,false,29516238,false,false,8,http://bullcityrecords.com/wnng/,https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg,-18000,29516238,true,118,333333,en,2052,false,null,https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp,9AE4E8,false,false,Eastern Time (US & Canada),You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.,false,http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp,7579,348,null,true,bullcityrecords,null,web,null
+null,false,false,2012-09-21 23:30:20,249289491129438208,null,null,Mexican Heaven, Mexican Hell #freebandnames,en,recent,3,null,249289491129438208,null,false,null,null,99CC33,829D5E,false,Thomas John Wakeman,http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,2009-09-01 21:21:35,Kingston New York,null,D02B55,false,70789458,false,false,19,null,https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png,-18000,70789458,true,1,3E4415,en,63,false,null,https://si0.twimg.com/images/themes/theme5/bg.gif,352726,false,false,Eastern Time (US & Canada),Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.,false,http://a0.twimg.com/images/themes/theme5/bg.gif,1048,63,null,false,MonkiesFist,null,web,null
+null,false,false,2012-09-21 22:51:18,249279667666817024,null,null,The Foolish Mortals #freebandnames,en,recent,4,null,249279667666817024,null,false,null,null,BFAC83,615A44,true,Marty Elmer,http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,2009-05-04 00:05:00,Wisconsin, USA,null,3B2A26,false,37539828,false,false,647,http://www.omnitarian.me,https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png,-21600,37539828,true,52,000000,en,608,false,null,https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png,EEE3C4,false,false,Central Time (US & Canada),Cartoonist, Illustrator, and T-Shirt connoisseur,false,http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png,3575,249,null,true,Omnitarian,null,<a>Twitter for iPhone</a>,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
index 2368082..d4ef55e 100644
--- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
+++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java
@@ -513,7 +513,7 @@ public class TajoDatabaseMetaData implements DatabaseMetaData {
CatalogUtil.buildFQName(databaseName, table));
int pos = 0;
- for (Column column: tableDesc.getLogicalSchema().getColumns()) {
+ for (Column column: tableDesc.getLogicalSchema().getRootColumns()) {
if (column.getSimpleName().matches(regcolumnNamePattern)) {
MetaDataTuple tuple = new MetaDataTuple(22);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
index 6262995..fbad76e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
@@ -114,7 +114,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
Schema schema = relationOp.getLogicalSchema();
Column[] resolvedColumns = new Column[schema.size()];
- return schema.getColumns().toArray(resolvedColumns);
+ return schema.getRootColumns().toArray(resolvedColumns);
} else { // if a column reference is not qualified
// columns of every relation should be resolved.
Iterator<RelationNode> iterator = block.getRelations().iterator();
@@ -124,7 +124,7 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor<LogicalPlanner.P
while (iterator.hasNext()) {
relationOp = iterator.next();
schema = relationOp.getLogicalSchema();
- resolvedColumns.addAll(schema.getColumns());
+ resolvedColumns.addAll(schema.getRootColumns());
}
if (resolvedColumns.size() == 0) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index e0b4f7e..21270e9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -52,6 +52,7 @@ import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -527,13 +528,24 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (projectable instanceof RelationNode) {
RelationNode relationNode = (RelationNode) projectable;
+ prohibitNestedRecordProjection((Projectable) relationNode);
verifyIfTargetsCanBeEvaluated(relationNode.getLogicalSchema(), (Projectable) relationNode);
} else {
+ prohibitNestedRecordProjection(projectable);
verifyIfTargetsCanBeEvaluated(projectable.getInSchema(), projectable);
}
}
+ public static void prohibitNestedRecordProjection(Projectable projectable)
+ throws PlanningException {
+ for (Target t : projectable.getTargets()) {
+ if (t.getEvalTree().getValueType().getType() == TajoDataTypes.Type.RECORD) {
+ throw new PlanningException("Projecting RECORD fields is not supported yet: " + t);
+ }
+ }
+ }
+
public static void verifyIfEvalNodesCanBeEvaluated(Projectable projectable, EvalNode[] evalNodes)
throws PlanningException {
for (EvalNode e : evalNodes) {
@@ -649,7 +661,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
if (block.namedExprsMgr.isEvaluated(sortKeyRefNames[i])) {
column = block.namedExprsMgr.getTarget(sortKeyRefNames[i]).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst());
}
@@ -866,7 +878,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
} else if (block.namedExprsMgr.isEvaluated(refName)) {
column = block.namedExprsMgr.getTarget(refName).getNamedColumn();
} else {
- throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs));
+ throw new IllegalStateException("Unexpected State: " + StringUtils.join(sortSpecs));
}
annotatedSortSpecs.add(new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()));
}
@@ -1175,13 +1187,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Schema joinSchema = new Schema();
Schema commons = SchemaUtil.getNaturalJoinColumns(left.getOutSchema(), right.getOutSchema());
joinSchema.addColumns(commons);
- for (Column c : left.getOutSchema().getColumns()) {
+ for (Column c : left.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
}
- for (Column c : right.getOutSchema().getColumns()) {
+ for (Column c : right.getOutSchema().getRootColumns()) {
if (!joinSchema.contains(c.getQualifiedName())) {
joinSchema.addColumn(c);
}
@@ -1199,7 +1211,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
Column leftJoinKey;
Column rightJoinKey;
- for (Column common : commons.getColumns()) {
+ for (Column common : commons.getRootColumns()) {
leftJoinKey = leftSchema.getColumn(common.getQualifiedName());
rightJoinKey = rightSchema.getColumn(common.getQualifiedName());
equiQual = new BinaryEval(EvalType.EQUAL,
@@ -1312,7 +1324,15 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
private static LinkedHashSet<Target> createFieldTargetsFromRelation(QueryBlock block, RelationNode relationNode,
Set<String> newlyEvaluatedRefNames) {
LinkedHashSet<Target> targets = Sets.newLinkedHashSet();
- for (Column column : relationNode.getLogicalSchema().getColumns()) {
+ for (Column column : relationNode.getLogicalSchema().getAllColumns()) {
+
+ // TODO - Currently, EvalNode has DataType as a return type. So, RECORD cannot be used for any target.
+ // The following line is a kind of hack, preventing RECORD to be used for target in the logical planning phase.
+ // This problem should be resolved after TAJO-1402.
+ if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) {
+ continue;
+ }
+
String aliasName = block.namedExprsMgr.checkAndGetIfAliasedColumn(column.getQualifiedName());
if (aliasName != null) {
targets.add(new Target(new FieldEval(column), aliasName));
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
index 153a150..eddb022 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.common.TajoDataTypes.DataType;
@@ -88,7 +89,7 @@ public class RowConstantEval extends EvalNode {
}
public String toString() {
- return TUtil.arrayToString(values);
+ return StringUtils.join(values);
}
public void preOrder(EvalNodeVisitor visitor) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
index 0b60d14..a39d303 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java
@@ -26,6 +26,7 @@ import org.apache.tajo.datum.Datum;
import org.apache.tajo.plan.function.FunctionContext;
import org.apache.tajo.plan.logical.WindowSpec;
import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.Arrays;
@@ -120,7 +121,7 @@ public class WindowFunctionEval extends AggregationFunctionCallEval implements C
sb.append(funcDesc.getFunctionName()).append("(").append(isDistinct() ? " distinct" : "").append(sb)
.append(")");
if (hasSortSpecs()) {
- sb.append("ORDER BY ").append(TUtil.arrayToString(sortSpecs));
+ sb.append("ORDER BY ").append(StringUtils.join(sortSpecs));
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
index 878553f..079fc3e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/python/PythonScriptEngine.java
@@ -478,7 +478,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
@@ -512,7 +512,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -532,7 +532,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -552,7 +552,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
outputHandler.getNext(functionContext);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -573,7 +573,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
return outputHandler.getPartialResultString();
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
}
@@ -595,7 +595,7 @@ public class PythonScriptEngine extends TajoScriptEngine {
try {
result = outputHandler.getNext().get(0);
} catch (Exception e) {
- throw new RuntimeException("Problem getting output", e);
+ throw new RuntimeException("Problem getting output: " + e.getMessage(), e);
}
return result;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
index e4d6122..862cb8a 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java
@@ -26,7 +26,7 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.catalog.SchemaUtil;
import org.apache.tajo.plan.expr.AlgebraicUtil;
import org.apache.tajo.plan.logical.*;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.*;
@@ -59,7 +59,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
for (LogicalNode relation : remainRelations) {
Collection <String> relationStrings = PlannerUtil.getRelationLineageWithinQueryBlock(plan, relation);
List<JoinEdge> joinEdges = new ArrayList<JoinEdge>();
- String relationCollection = TUtil.collectionToString(relationStrings, ",");
+ String relationCollection = StringUtils.join(relationStrings, ",");
List<JoinEdge> joinEdgesForGiven = joinGraph.getIncomingEdges(relationCollection);
if (joinEdgesForGiven != null) {
joinEdges.addAll(joinEdgesForGiven);
@@ -236,7 +236,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
// If outer is outer join, make edge key using all relation names in outer.
SortedSet<String> relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer));
- String outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ String outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String innerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner)) {
if (graph.hasEdge(outerEdgeKey, innerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, innerName);
@@ -257,7 +257,7 @@ public class GreedyHeuristicJoinOrderAlgorithm implements JoinOrderAlgorithm {
relationNames =
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, inner));
- outerEdgeKey = TUtil.collectionToString(relationNames, ", ");
+ outerEdgeKey = StringUtils.join(relationNames, ", ");
for (String outerName : PlannerUtil.getRelationLineageWithinQueryBlock(plan, outer)) {
if (graph.hasEdge(outerEdgeKey, outerName)) {
JoinEdge existJoinEdge = graph.getEdge(outerEdgeKey, outerName);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
index ce06748..fb4fae1 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java
@@ -22,7 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.Collections;
import java.util.Set;
@@ -70,6 +70,6 @@ public class JoinEdge {
}
public String toString() {
- return leftRelation + " " + joinType + " " + rightRelation + " ON " + TUtil.collectionToString(joinQual, ", ");
+ return leftRelation + " " + joinType + " " + rightRelation + " ON " + StringUtils.join(joinQual, ", ");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
index 72e9b1d..9ae5245 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.graph.SimpleUndirectedGraph;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.NamedExprsManager;
@@ -33,7 +34,6 @@ import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.expr.EvalTreeUtil;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.RelationNode;
-import org.apache.tajo.util.TUtil;
import java.util.*;
@@ -111,8 +111,8 @@ public class JoinGraph extends SimpleUndirectedGraph<String, JoinEdge> {
new TreeSet<String>(PlannerUtil.getRelationLineageWithinQueryBlock(plan, joinNode.getRightChild()));
addEdge(
- TUtil.collectionToString(leftNodeRelationName, ", "),
- TUtil.collectionToString(rightNodeRelationName, ", "),
+ StringUtils.join(leftNodeRelationName, ", "),
+ StringUtils.join(rightNodeRelationName, ", "),
edge);
Set<EvalNode> allInOneCnf = new HashSet<EvalNode>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
index 61ce789..fb19f10 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java
@@ -19,12 +19,12 @@
package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import java.util.ArrayList;
@@ -150,7 +150,7 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone
public String toString() {
StringBuilder sb = new StringBuilder("Distinct GroupBy (");
if (groupingColumns != null && groupingColumns.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingColumns));
+ sb.append("grouping set=").append(StringUtils.join(groupingColumns));
sb.append(", ");
}
for (GroupbyNode eachNode: subGroupbyPlan) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
index 3aaf5d0..0632f68 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java
@@ -28,6 +28,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class EvalExprNode extends LogicalNode implements Projectable {
@@ -69,7 +70,7 @@ public class EvalExprNode extends LogicalNode implements Projectable {
@Override
public String toString() {
- return "EvalExprNode (" + TUtil.arrayToString(exprs) + ")";
+ return "EvalExprNode (" + StringUtils.join(exprs) + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
index 730f705..23a9154 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java
@@ -18,18 +18,18 @@
package org.apache.tajo.plan.logical;
-import java.util.Arrays;
-
import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
-
import org.apache.tajo.catalog.Column;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
+import java.util.Arrays;
+
public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
/** Grouping key sets */
@Expose private Column [] groupingKeys = PlannerUtil.EMPTY_COLUMNS;
@@ -110,11 +110,11 @@ public class GroupbyNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("GroupBy (");
if (groupingKeys != null && groupingKeys.length > 0) {
- sb.append("grouping set=").append(TUtil.arrayToString(groupingKeys));
+ sb.append("grouping set=").append(StringUtils.join(groupingKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(aggrFunctions));
+ sb.append("funcs=").append(StringUtils.join(aggrFunctions));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
index 3c9d497..ef6734e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java
@@ -25,6 +25,7 @@ import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class ProjectionNode extends UnaryNode implements Projectable {
@@ -70,7 +71,7 @@ public class ProjectionNode extends UnaryNode implements Projectable {
public String toString() {
StringBuilder sb = new StringBuilder("Projection (distinct=").append(distinct);
if (targets != null) {
- sb.append(", exprs=").append(TUtil.arrayToString(targets)).append(")");
+ sb.append(", exprs=").append(StringUtils.join(targets)).append(")");
}
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
index 7e335b0..ced9a36 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java
@@ -43,7 +43,19 @@ public abstract class RelationNode extends LogicalNode {
public abstract String getTableName();
+ /**
+ * Return a full qualified table name (i.e., dbname.table_name)
+ *
+ * @return A full qualified table name
+ */
public abstract String getCanonicalName();
+ /**
+ * Return a logical schema, meaning physically stored columns and virtual columns.
+ * Since partition keys in the column partition are not physically stored to files or tables,
+ * we call the partition keys virtual columns.
+ *
+ * @return A logical schema
+ */
public abstract Schema getLogicalSchema();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
index 8215f51..7487009 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import com.google.gson.annotations.Expose;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
import static org.apache.tajo.plan.serder.PlanProto.ShuffleType;
@@ -107,7 +108,7 @@ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneab
}
sb.append(", part number=").append(numOutputs);
if (shuffleKeys != null) {
- sb.append(", keys: ").append(TUtil.arrayToString(shuffleKeys));
+ sb.append(", keys: ").append(StringUtils.join(shuffleKeys));
}
sb.append(")");
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
index 0166ef8..ef61d51 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java
@@ -20,7 +20,7 @@ package org.apache.tajo.plan.logical;
import com.google.gson.annotations.Expose;
import org.apache.tajo.plan.PlanString;
-import org.apache.tajo.util.TUtil;
+import org.apache.tajo.util.StringUtils;
import java.util.List;
@@ -52,7 +52,7 @@ public class TruncateTableNode extends LogicalNode {
@Override
public String toString() {
- return "TruncateTable (table=" + TUtil.collectionToString(tableNames, ", ") + ")";
+ return "TruncateTable (table=" + StringUtils.join(tableNames, ", ") + ")";
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
index 3f624f6..1bee34c 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java
@@ -26,6 +26,7 @@ import org.apache.tajo.plan.PlanString;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.Target;
import org.apache.tajo.plan.expr.WindowFunctionEval;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
@@ -114,14 +115,14 @@ public class WindowAggNode extends UnaryNode implements Projectable, Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("WinAgg (");
if (hasPartitionKeys()) {
- sb.append("partition keys=").append(TUtil.arrayToString(partitionKeys));
+ sb.append("partition keys=").append(StringUtils.join(partitionKeys));
sb.append(", ");
}
if (hasAggFunctions()) {
- sb.append("funcs=").append(TUtil.arrayToString(windowFuncs));
+ sb.append("funcs=").append(StringUtils.join(windowFuncs));
}
if (hasSortSpecs()) {
- sb.append("sort=").append(TUtil.arrayToString(sortSpecs));
+ sb.append("sort=").append(StringUtils.join(sortSpecs));
}
sb.append(")");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index 2c55c81..633b30e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -18,11 +18,14 @@
package org.apache.tajo.plan.nameresolver;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.tajo.algebra.ColumnReferenceExpr;
+import org.apache.tajo.algebra.Relation;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.exception.NoSuchColumnException;
import org.apache.tajo.plan.algebra.AmbiguousFieldException;
@@ -31,14 +34,30 @@ import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.verifier.VerifyException;
import org.apache.tajo.plan.logical.RelationNode;
import org.apache.tajo.util.Pair;
+import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
- * NameResolver utility
+ * Column name resolution utility. A SQL statement can include many kinds of column names,
+ * defined in different ways. Some column name indicates just a column in a relation.
+ * Another column name includes alias table name or alias column name, derived from some expression.
+ *
+ * This utility ensures that each column name is derived from valid and accessible column, and
+ * it also finds the exact data type of the column.
+ *
+ * Terminology:
+ * <ul>
+ * <li>Qualifier: database name, table name, or both included in a column name</li>
+ * <li>Simple name: just column name without any qualifier</li>
+ * <li>Alias name: another name to shortly specify a certain column</li>
+ * <li>Fully qualified name: a column name with database name and table name</li>
+ * <li>Canonical name: a fully qualified name, but its simple name is aliased name.</li>
+ * </ul>
*/
public abstract class NameResolver {
@@ -51,30 +70,44 @@ public abstract class NameResolver {
resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy());
}
+ public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
+ NameResolvingMode mode) throws PlanningException {
+ if (!resolverMap.containsKey(mode)) {
+ throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ }
+ return resolverMap.get(mode).resolve(plan, block, column);
+ }
+
abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)
throws PlanningException;
/**
- * Try to find the database name
+ * Guess a relation from a table name regardless of whether the given name is qualified or not.
*
* @param block the current block
- * @param tableName The table name
- * @return The found database name
+ * @param tableName The table name which can be either qualified or not.
+ * @return A corresponding relation
* @throws PlanningException
*/
- public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
- List<String> found = new ArrayList<String>();
+ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tableName) throws PlanningException {
+ List<RelationNode> found = TUtil.newList();
+
for (RelationNode relation : block.getRelations()) {
- // check alias name or table name
- if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
+
+ // if a table name is qualified
+ if (relation.getCanonicalName().equals(tableName) || relation.getTableName().equals(tableName)) {
+ found.add(relation);
+
+ // if a table name is not qualified
+ } else if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) ||
CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) {
- // obtain the database name
- found.add(CatalogUtil.extractQualifier(relation.getTableName()));
+ found.add(relation);
}
}
if (found.size() == 0) {
return null;
+
} else if (found.size() > 1) {
throw new PlanningException("Ambiguous table name \"" + tableName + "\"");
}
@@ -82,12 +115,26 @@ public abstract class NameResolver {
return found.get(0);
}
- public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column,
- NameResolvingMode mode) throws PlanningException {
- if (!resolverMap.containsKey(mode)) {
- throw new PlanningException("Unsupported name resolving level: " + mode.name());
+ /**
+ * Find relations such that its schema contains a given column
+ *
+ * @param block the current block
+ * @param columnName The column name to find relation
+ * @return relations including a given column
+ * @throws PlanningException
+ */
+ public static Collection<RelationNode> lookupTableByColumns(LogicalPlan.QueryBlock block, String columnName)
+ throws PlanningException {
+
+ Set<RelationNode> found = TUtil.newHashSet();
+
+ for (RelationNode rel : block.getRelations()) {
+ if (rel.getLogicalSchema().contains(columnName)) {
+ found.add(rel);
+ }
}
- return resolverMap.get(mode).resolve(plan, block, column);
+
+ return found;
}
/**
@@ -107,7 +154,7 @@ public abstract class NameResolver {
String canonicalName;
if (columnRef.hasQualifier()) {
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
canonicalName = normalized.getSecond();
@@ -121,8 +168,8 @@ public abstract class NameResolver {
// Please consider a query case:
// select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey;
//
- // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used.
- // The below code makes it available. Otherwise, it cannot find any match in the relation schema.
+ // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available.
+ // The below code makes it possible. Otherwise, it cannot find any match in the relation schema.
if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) {
canonicalName =
CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName));
@@ -133,7 +180,7 @@ public abstract class NameResolver {
return column;
} else {
- return resolveFromAllRelsInBlock(block, columnRef);
+ return lookupColumnFromAllRelsInBlock(block, columnRef.getName());
}
}
@@ -162,18 +209,22 @@ public abstract class NameResolver {
}
/**
- * It tries to find a full qualified column name from all relations in the current block.
+ * Lookup a column among all relations in the current block from a column name.
+ *
+ * It assumes that <code>columnName</code> is not any qualified name.
*
* @param block The current query block
- * @param columnRef The column reference to be found
+ * @param columnName The column reference to be found
* @return The found column
*/
- static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef) throws VerifyException {
+ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block,
+ String columnName) throws VerifyException {
+ Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName));
+
List<Column> candidates = TUtil.newList();
for (RelationNode rel : block.getRelations()) {
- Column found = rel.getLogicalSchema().getColumn(columnRef.getName());
+ Column found = rel.getLogicalSchema().getColumn(columnName);
if (found != null) {
candidates.add(found);
}
@@ -240,39 +291,100 @@ public abstract class NameResolver {
}
/**
- * It returns a pair of names, which the first value is ${database}.${table} and the second value
- * is a simple column name.
+ * Lookup a qualifier and a canonical name of column.
+ *
+ * It returns a pair of names, which the first value is the qualifier ${database}.${table} and
+ * the second value is column's simple name.
*
* @param block The current block
* @param columnRef The column name
* @return A pair of normalized qualifier and column name
* @throws PlanningException
*/
- static Pair<String, String> normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
- ColumnReferenceExpr columnRef)
+ static Pair<String, String> lookupQualifierAndCanonicalName(LogicalPlan.QueryBlock block,
+ ColumnReferenceExpr columnRef)
throws PlanningException {
- String qualifier;
- String canonicalName;
+ Preconditions.checkArgument(columnRef.hasQualifier(), "ColumnReferenceExpr must be qualified.");
+
+ String [] qualifierParts = columnRef.getQualifier().split("\\.");
- if (CatalogUtil.isFQTableName(columnRef.getQualifier())) {
- qualifier = columnRef.getQualifier();
- canonicalName = columnRef.getCanonicalName();
+ // This method assumes that column name consists of two or more dot chained names.
+ // In this case, there must be three cases as follows:
+ //
+ // - dbname.tbname.column_name.nested_field...
+ // - tbname.column_name.nested_field...
+ // - column.nested_fieldX...
+
+ Set<RelationNode> guessedRelations = TUtil.newHashSet();
+
+ // this position indicates the index of column name in qualifierParts;
+ // It must be 0 or more because a qualified column is always passed to lookupQualifierAndCanonicalName().
+ int columnNamePosition = -1;
+
+ // check for dbname.tbname.column_name.nested_field
+ if (qualifierParts.length >= 2) {
+ RelationNode rel = lookupTable(block, CatalogUtil.buildFQName(qualifierParts[0], qualifierParts[1]));
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 2;
+ }
+ }
+
+ // check for tbname.column_name.nested_field
+ if (qualifierParts.length >= 1) {
+ RelationNode rel = lookupTable(block, qualifierParts[0]);
+ if (rel != null) {
+ guessedRelations.add(rel);
+ columnNamePosition = 1;
+ }
+ }
+
+ // column.nested_fieldX...
+ if (guessedRelations.size() == 0 && qualifierParts.length == 1) {
+ Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]);
+
+ if (rels.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ if (rels.size() == 1) {
+ guessedRelations.addAll(rels);
+ columnNamePosition = 0;
+ }
+ }
+
+ // throw exception if no column cannot be founded or two or more than columns are founded
+ if (guessedRelations.size() == 0) {
+ throw new NoSuchColumnException(columnRef.getQualifier());
+ } else if (guessedRelations.size() > 1) {
+ throw new AmbiguousFieldException(columnRef.getCanonicalName());
+ }
+
+ String qualifier = guessedRelations.iterator().next().getCanonicalName();
+ String columnName = "";
+
+ if (columnNamePosition >= qualifierParts.length) { // if there is no column in qualifierParts
+ columnName = columnRef.getName();
} else {
- String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier());
- if (resolvedDatabaseName == null) {
- throw new NoSuchColumnException(columnRef.getQualifier());
+ // join a column name and its nested field names
+ columnName = qualifierParts[columnNamePosition];
+
+ // if qualifierParts include nested field names
+ if (qualifierParts.length > columnNamePosition) {
+ columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length);
}
- qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier());
- canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
+
+ // columnRef always has a leaf field name.
+ columnName += "/" + columnRef.getName();
}
- return new Pair<String, String>(qualifier, canonicalName);
+ return new Pair<String, String>(qualifier, columnName);
}
static Column ensureUniqueColumn(List<Column> candidates) throws VerifyException {
if (candidates.size() == 1) {
return candidates.get(0);
- } else if (candidates.size() > 2) {
+ } else if (candidates.size() > 1) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (Column column : candidates) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
index 19f39dd..0a665ab 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java
@@ -48,12 +48,10 @@ public class ResolverByLegacy extends NameResolver {
private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef) throws PlanningException {
final String qualifier;
- String canonicalName;
final String qualifiedName;
- Pair<String, String> normalized = normalizeQualifierAndCanonicalName(block, columnRef);
+ Pair<String, String> normalized = lookupQualifierAndCanonicalName(block, columnRef);
qualifier = normalized.getFirst();
- canonicalName = normalized.getSecond();
qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName());
Column found = resolveFromRelsWithinBlock(plan, block, columnRef);
@@ -84,7 +82,7 @@ public class ResolverByLegacy extends NameResolver {
&& currentNode.getType() != NodeType.TABLE_SUBQUERY) {
List<Column> candidates = TUtil.newList();
if (block.getNamedExprsManager().isAliased(qualifiedName)) {
- String alias = block.getNamedExprsManager().getAlias(canonicalName);
+ String alias = block.getNamedExprsManager().getAlias(qualifiedName);
found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY);
if (found != null) {
candidates.add(found);
@@ -101,7 +99,7 @@ public class ResolverByLegacy extends NameResolver {
static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block,
ColumnReferenceExpr columnRef)throws PlanningException {
- Column found = resolveFromAllRelsInBlock(block, columnRef);
+ Column found = lookupColumnFromAllRelsInBlock(block, columnRef.getName());
if (found != null) {
return found;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index dc6b8ef..cc54a22 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -726,7 +726,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
// find aggregation column
Set<Column> groupingColumns = TUtil.newHashSet(groupByNode.getGroupingColumns());
Set<String> aggrFunctionOutColumns = TUtil.newHashSet();
- for (Column column : groupByNode.getOutSchema().getColumns()) {
+ for (Column column : groupByNode.getOutSchema().getRootColumns()) {
if (!groupingColumns.contains(column)) {
aggrFunctionOutColumns.add(column.getQualifiedName());
}
@@ -842,7 +842,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
TableDesc table = scanNode.getTableDesc();
boolean hasQualifiedName = false;
if (table.hasPartition()) {
- for (Column c: table.getPartitionMethod().getExpressionSchema().getColumns()) {
+ for (Column c: table.getPartitionMethod().getExpressionSchema().getRootColumns()) {
partitionColumns.add(c.getQualifiedName());
hasQualifiedName = c.hasQualifier();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index d8b6380..4b9fd48 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -224,7 +224,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod();
Schema paritionValuesSchema = new Schema();
- for (Column column : partitionDesc.getExpressionSchema().getColumns()) {
+ for (Column column : partitionDesc.getExpressionSchema().getRootColumns()) {
paritionValuesSchema.addColumn(column);
}
@@ -237,7 +237,7 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
// add qualifier to schema for qual
paritionValuesSchema.setQualifier(scanNode.getCanonicalName());
- for (Column column : paritionValuesSchema.getColumns()) {
+ for (Column column : paritionValuesSchema.getRootColumns()) {
for (EvalNode simpleExpr : conjunctiveForms) {
if (checkIfIndexablePredicateOnTargetColumn(simpleExpr, column)) {
indexablePredicateSet.add(simpleExpr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 763f938..3b83ded 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -146,7 +146,7 @@ public class PlannerUtil {
if (EvalTreeUtil.checkIfPartitionSelection(node, partSchema)) {
prefixPartitionWhere = true;
boolean isPrefix = true;
- for (Column c : partSchema.getColumns()) {
+ for (Column c : partSchema.getRootColumns()) {
String value = EvalTreeUtil.getPartitionValue(node, c.getSimpleName());
if (isPrefix && value == null)
isPrefix = false;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
index 22b3351..c5ef8ef 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java
@@ -23,6 +23,8 @@ import org.apache.tajo.OverridableConf;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.util.PlannerUtil;
@@ -55,7 +57,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
/**
* It checks if an output schema of a projectable node and target's output data types are equivalent to each other.
*/
- private static void verifyProjectableOutputSchema(Projectable node) throws PlanningException {
+ private static void verifyProjectableOutputSchema(Context context, Projectable node) throws PlanningException {
Schema outputSchema = node.getOutSchema();
Schema targetSchema = PlannerUtil.targetToSchema(node.getTargets());
@@ -66,9 +68,15 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
for (int i = 0; i < outputSchema.size(); i++) {
- if (!outputSchema.getColumn(i).getDataType().equals(targetSchema.getColumn(i).getDataType())) {
+ Column outputColumn = outputSchema.getColumn(i);
+
+ if (outputColumn.getDataType().getType() == Type.RECORD) {
+ context.state.addVerification("Projecting RECORD fields is not supported yet.");
+ }
+
+ if (!outputColumn.getDataType().equals(targetSchema.getColumn(i).getDataType())) {
Column targetColumn = targetSchema.getColumn(i);
- Column insertColumn = outputSchema.getColumn(i);
+ Column insertColumn = outputColumn;
throw new PlanningException("ERROR: " +
insertColumn.getSimpleName() + " is of type " + insertColumn.getDataType().getType().name() +
", but target column '" + targetColumn.getSimpleName() + "' is of type " +
@@ -86,7 +94,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(state.state, node, target.getEvalTree());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(state, node);
return node;
}
@@ -108,7 +116,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
GroupbyNode node, Stack<LogicalNode> stack) throws PlanningException {
super.visitGroupBy(context, plan, block, node, stack);
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -130,7 +138,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getJoinQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -192,7 +200,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
}
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
@@ -209,7 +217,7 @@ public class LogicalPlanVerifier extends BasicLogicalPlanVisitor<LogicalPlanVeri
ExprsVerifier.verify(context.state, node, node.getQual());
}
- verifyProjectableOutputSchema(node);
+ verifyProjectableOutputSchema(context, node);
return node;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index 275ab3a..8717bda 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -83,7 +83,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
super.visitProjection(context, stack, expr);
Set<String> names = TUtil.newHashSet();
-
+
for (NamedExpr namedExpr : expr.getNamedExprs()) {
if (namedExpr.hasAlias()) {
@@ -302,8 +302,8 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
return null;
}
if (table.hasPartition()) {
- int columnSize = table.getSchema().getColumns().size();
- columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size();
+ int columnSize = table.getSchema().getRootColumns().size();
+ columnSize += table.getPartitionMethod().getExpressionSchema().getRootColumns().size();
if (projectColumnNum < columnSize) {
context.state.addVerification("INSERT has smaller expressions than target columns");
} else if (projectColumnNum > columnSize) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
index 66b3667..5423fd7 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -84,7 +84,7 @@ public class MergeScanner implements Scanner {
tableStats.setNumBytes(numBytes);
tableStats.setNumBlocks(fragments.size());
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
index 33db798..9d69423 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
@@ -35,7 +35,7 @@ public class RowStoreUtil {
public static int[] getTargetIds(Schema inSchema, Schema outSchema) {
int[] targetIds = new int[outSchema.size()];
int i = 0;
- for (Column target : outSchema.getColumns()) {
+ for (Column target : outSchema.getRootColumns()) {
targetIds[i] = inSchema.getColumnId(target.getQualifiedName());
i++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
index 0356b19..7af8247 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Scanner.java
@@ -61,6 +61,15 @@ public interface Scanner extends SchemaObject, Closeable {
/**
* It returns if the projection is executed in the underlying scanner layer.
*
+ * If TRUE, the upper layers (i.e., SeqScanExec) assume that next()
+ * will return a tuple which contains only projected fields. In other words,
+ * the field number of a retrieved tuple is equivalent tothe number of targets.
+ *
+ * If FALSE, the upper layers assume that next() will return a tuple which
+ * contains projected fields and non-projected fields, padded by NullDatum.
+ * In other words, the width of tuple is equivalent to the field number
+ * of the table schema.
+ *
* @return true if this scanner can project the given columns.
*/
boolean isProjectable();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
index ce963c8..b8438e9 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -409,9 +409,7 @@ public abstract class StorageManager {
Class<? extends Scanner> scannerClass = getScannerClass(meta.getStoreType());
scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment);
- if (scanner.isProjectable()) {
- scanner.setTarget(target.toArray());
- }
+ scanner.setTarget(target.toArray());
return scanner;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
index 4836352..0e3441b 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java
@@ -60,7 +60,6 @@ public abstract class AbstractHBaseAppender implements Appender {
protected boolean[] isRowKeyMappings;
protected boolean[] isColumnKeys;
protected boolean[] isColumnValues;
- protected int[] rowKeyFieldIndexes;
protected int[] rowkeyColumnIndexes;
protected char rowKeyDelimiter;
@@ -107,7 +106,6 @@ public abstract class AbstractHBaseAppender implements Appender {
isColumnKeys = columnMapping.getIsColumnKeys();
isColumnValues = columnMapping.getIsColumnValues();
rowKeyDelimiter = columnMapping.getRowKeyDelimiter();
- rowKeyFieldIndexes = columnMapping.getRowKeyFieldIndexes();
this.columnNum = schema.size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
index c3094fd..e66a707 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java
@@ -77,7 +77,7 @@ public class ColumnMapping {
String[] columnMappingTokens = columnMapping.split(",");
- if (columnMappingTokens.length != schema.getColumns().size()) {
+ if (columnMappingTokens.length != schema.getRootColumns().size()) {
throw new IOException("The number of mapped HBase columns is great than the number of Tajo table columns");
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
index c05c5bb..53ff9dc 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseBinarySerializerDeserializer.java
@@ -39,10 +39,14 @@ public class HBaseBinarySerializerDeserializer {
datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt4(Bytes.toInt(bytes));
break;
case INT8:
- if (bytes.length == 4) {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toInt(bytes));
+ if (bytes == null) {
+ datum = NullDatum.get();
} else {
- datum = bytes == null || bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ if (bytes.length == 4) {
+ datum = DatumFactory.createInt8(Bytes.toInt(bytes));
+ } else {
+ datum = bytes.length == 0 ? NullDatum.get() : DatumFactory.createInt8(Bytes.toLong(bytes));
+ }
}
break;
case FLOAT4:
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
index ab56252..9eb1d86 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.hbase;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -87,7 +88,13 @@ public class HBaseScanner implements Scanner {
private char rowKeyDelimiter;
public HBaseScanner (Configuration conf, Schema schema, TableMeta meta, Fragment fragment) throws IOException {
- this.conf = (TajoConf)conf;
+ Preconditions.checkNotNull(conf);
+ Preconditions.checkNotNull(schema);
+ Preconditions.checkNotNull(meta);
+ Preconditions.checkNotNull(fragment);
+ Preconditions.checkArgument(conf instanceof TajoConf);
+
+ this.conf = (TajoConf) conf;
this.schema = schema;
this.meta = meta;
this.fragment = (HBaseFragment)fragment;
@@ -102,11 +109,10 @@ public class HBaseScanner implements Scanner {
tableStats.setNumBytes(0);
tableStats.setNumBlocks(1);
}
- if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
- ColumnStats columnStats = new ColumnStats(eachColumn);
- tableStats.addColumnStat(columnStats);
- }
+
+ for (Column eachColumn : schema.getRootColumns()) {
+ ColumnStats columnStats = new ColumnStats(eachColumn);
+ tableStats.addColumnStat(columnStats);
}
scanFetchSize = Integer.parseInt(
@@ -203,9 +209,9 @@ public class HBaseScanner implements Scanner {
}
Result result = scanResults[scanResultIndex++];
- Tuple resultTuple = new VTuple(schema.size());
+ Tuple resultTuple = new VTuple(targetIndexes.length);
for (int i = 0; i < targetIndexes.length; i++) {
- resultTuple.put(targetIndexes[i], getDatum(result, targetIndexes[i]));
+ resultTuple.put(i, getDatum(result, targetIndexes[i]));
}
numRows++;
return resultTuple;
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
index a9e5bde..f74245e 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseStorageManager.java
@@ -355,7 +355,7 @@ public class HBaseStorageManager extends StorageManager {
Collection<String> columnFamilies = columnMapping.getColumnFamilyNames();
//If 'columns' attribute is empty, Tajo table columns are mapped to all HBase table column.
if (columnFamilies.isEmpty()) {
- for (Column eachColumn: schema.getColumns()) {
+ for (Column eachColumn: schema.getRootColumns()) {
columnFamilies.add(eachColumn.getSimpleName());
}
}
@@ -735,7 +735,7 @@ public class HBaseStorageManager extends StorageManager {
for (String property : CONNECTION_PROPERTIES) {
String thisValue = this.properties.get(property);
String thatValue = that.properties.get(property);
- //noinspection StringEquality
+ // noinspection StringEquality
if (thisValue == thatValue) {
continue;
}
@@ -1121,7 +1121,7 @@ public class HBaseStorageManager extends StorageManager {
try {
HTableDescriptor hTableDesc = parseHTableDescriptor(tableMeta, cNode.getTableSchema());
- LOG.info("Delete table cause query failed:" + hTableDesc.getName());
+ LOG.info("Delete table cause query failed:" + new String(hTableDesc.getName()));
hAdmin.disableTable(hTableDesc.getName());
hAdmin.deleteTable(hTableDesc.getName());
} finally {
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
index c1047d9..5fc96f1 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -528,7 +528,7 @@ public class CSVFile {
@Override
public boolean isProjectable() {
- return true;
+ return false;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
index 038f0f4..0726125 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileScanner.java
@@ -69,7 +69,7 @@ public abstract class FileScanner implements Scanner {
}
if (schema != null) {
- for(Column eachColumn: schema.getColumns()) {
+ for(Column eachColumn: schema.getRootColumns()) {
ColumnStats columnStats = new ColumnStats(eachColumn);
tableStats.addColumnStat(columnStats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index 8d425b4..439bee0 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -732,7 +732,7 @@ public class FileStorageManager extends StorageManager {
//In the case of partitioned table, we should return same partition key data files.
int partitionDepth = 0;
if (tableDesc.hasPartition()) {
- partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size();
}
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
http://git-wip-us.apache.org/repos/asf/tajo/blob/0d1bf41f/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
index 51594df..26083a5 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
@@ -80,8 +80,7 @@ public class AvroScanner extends FileScanner {
avroSchema = AvroUtil.getAvroSchema(meta, conf);
avroFields = avroSchema.getFields();
- DatumReader<GenericRecord> datumReader =
- new GenericDatumReader<GenericRecord>(avroSchema);
+ DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(avroSchema);
SeekableInput input = new FsInput(fragment.getPath(), conf);
dataFileReader = new DataFileReader<GenericRecord>(input, datumReader);
super.init();
@@ -175,13 +174,13 @@ public class AvroScanner extends FileScanner {
return null;
}
- Tuple tuple = new VTuple(schema.size());
+ Tuple tuple = new VTuple(projectionMap.length);
GenericRecord record = dataFileReader.next();
for (int i = 0; i < projectionMap.length; ++i) {
int columnIndex = projectionMap[i];
Object value = record.get(columnIndex);
if (value == null) {
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
continue;
}
@@ -196,28 +195,28 @@ public class AvroScanner extends FileScanner {
TajoDataTypes.Type tajoType = dataType.getType();
switch (avroType) {
case NULL:
- tuple.put(columnIndex, NullDatum.get());
+ tuple.put(i, NullDatum.get());
break;
case BOOLEAN:
- tuple.put(columnIndex, DatumFactory.createBool((Boolean)value));
+ tuple.put(i, DatumFactory.createBool((Boolean)value));
break;
case INT:
- tuple.put(columnIndex, convertInt(value, tajoType));
+ tuple.put(i, convertInt(value, tajoType));
break;
case LONG:
- tuple.put(columnIndex, DatumFactory.createInt8((Long)value));
+ tuple.put(i, DatumFactory.createInt8((Long)value));
break;
case FLOAT:
- tuple.put(columnIndex, DatumFactory.createFloat4((Float)value));
+ tuple.put(i, DatumFactory.createFloat4((Float)value));
break;
case DOUBLE:
- tuple.put(columnIndex, DatumFactory.createFloat8((Double)value));
+ tuple.put(i, DatumFactory.createFloat8((Double)value));
break;
case BYTES:
- tuple.put(columnIndex, convertBytes(value, tajoType, dataType));
+ tuple.put(i, convertBytes(value, tajoType, dataType));
break;
case STRING:
- tuple.put(columnIndex, convertString(value, tajoType));
+ tuple.put(i, convertString(value, tajoType));
break;
case RECORD:
throw new RuntimeException("Avro RECORD not supported.");
@@ -228,7 +227,7 @@ public class AvroScanner extends FileScanner {
case UNION:
throw new RuntimeException("Avro UNION not supported.");
case FIXED:
- tuple.put(columnIndex, new BlobDatum(((GenericFixed)value).bytes()));
+ tuple.put(i, new BlobDatum(((GenericFixed)value).bytes()));
break;
default:
throw new RuntimeException("Unknown type.");