You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/08/12 11:38:12 UTC
tajo git commit: TAJO-1752: NameResolver cannot find nested records
properly.
Repository: tajo
Updated Branches:
refs/heads/master 5982cf1ff -> fd6a95180
TAJO-1752: NameResolver cannot find nested records properly.
Closes #686
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd6a9518
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd6a9518
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd6a9518
Branch: refs/heads/master
Commit: fd6a95180ecfa05df7cd0b1a8a957057c9788ad6
Parents: 5982cf1
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Aug 12 18:37:57 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Aug 12 18:37:57 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../java/org/apache/tajo/catalog/NestedPathUtil.java | 2 +-
.../main/java/org/apache/tajo/util/StringUtils.java | 3 ++-
.../tajo/engine/query/TestSelectNestedRecord.java | 10 ++++++++++
.../TestSelectNestedRecord/sample2/sample2.json | 1 +
.../queries/TestSelectNestedRecord/sample2_ddl.sql | 10 +++++++---
.../queries/TestSelectNestedRecord/testSelect3.sql | 1 +
.../results/TestSelectNestedRecord/testSelect3.result | 3 +++
.../apache/tajo/plan/nameresolver/NameResolver.java | 13 ++++++++-----
9 files changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f572ebc..c7a8fde 100644
--- a/CHANGES
+++ b/CHANGES
@@ -216,6 +216,8 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1752: NameResolver cannot find nested records properly. (jihoon)
+
TAJO-1732: During filter push down phase, join conditions are not set properly.
(jihoon)
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
index dec9a55..9c750e8 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java
@@ -79,7 +79,7 @@ public class NestedPathUtil {
}
if (found != null) {
- if (found.getDataType().getType() == Type.RECORD) {
+ if (found.getDataType().getType() == Type.RECORD && paths.length > depth + 1) {
return lookupColumnInternal(found, paths, depth + 1);
} else {
return found;
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
index 61dc855..7a9f0c3 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java
@@ -432,7 +432,8 @@ public class StringUtils {
public static String join(Object[] objects, String delimiter, int startIndex, int length) {
boolean first = true;
StringBuilder sb = new StringBuilder();
- for(int i = startIndex; i + startIndex < length; i++) {
+ int endIndex = startIndex + length;
+ for(int i = startIndex; i + startIndex < endIndex; i++) {
if (first) {
first = false;
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
index ffd6f08..41df0fa 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java
@@ -60,6 +60,16 @@ public class TestSelectNestedRecord extends QueryTestCaseBase {
}
@Test
+ public final void testSelect3() throws Exception {
+ List<String> tables = executeDDL("sample2_ddl.sql", "sample2", "sample5");
+ assertEquals(TUtil.newList("sample5"), tables);
+
+ ResultSet res = executeQuery();
+ assertResultSet(res);
+ cleanupQuery(res);
+ }
+
+ @Test
public final void testNestedFieldAsGroupbyKey1() throws Exception {
List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets");
assertEquals(TUtil.newList("tweets"), tables);
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
new file mode 100644
index 0000000..20bab62
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json
@@ -0,0 +1 @@
+{ "glossary": { "title": "example glossary", "GlossDiv": { "title": "S", "GlossList": { "GlossEntry": { "ID": "SGML", "SortAs": "SGML", "GlossTerm": "Standard Generalized Markup Language", "Acronym": "SGML", "Abbrev": "ISO 8879:1986", "GlossDef": { "para": "A meta-markup language, used to create markup languages such as DocBook.", "GlossSeeAlso": ["GML", "XML"] }, "GlossSee": "markup" } } } } }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
index 9537c3e..bc2d6e2 100644
--- a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql
@@ -2,18 +2,22 @@ CREATE EXTERNAL TABLE ${0} (
glossary RECORD (
title TEXT,
"GlossDiv" RECORD (
- "GlossEntry" RECORD (
+ title TEXT,
+ "GlossList" RECORD (
+ "GlossEntry" RECORD (
"ID" TEXT,
"SortAs" TEXT,
"GlossTerm" TEXT,
"Acronym" TEXT,
"Abbrev" TEXT,
"GlossDef" RECORD (
- para TEXT
+ para TEXT,
+ "GlossSeeAlso" TEXT
),
"GlossSee" TEXT
+ )
)
)
)
-) USING JSON LOCATION ${path};
\ No newline at end of file
+) USING JSON LOCATION ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
new file mode 100644
index 0000000..dfc79e9
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql
@@ -0,0 +1 @@
+select glossary.title, glossary."GlossDiv"."title" from sample5;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
new file mode 100644
index 0000000..47e3fe5
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result
@@ -0,0 +1,3 @@
+glossary/title,glossary/GlossDiv/title
+-------------------------------
+example glossary,S
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
index a6f3d35..c346ce9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java
@@ -24,6 +24,7 @@ import com.google.common.collect.Maps;
import org.apache.tajo.algebra.ColumnReferenceExpr;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.NestedPathUtil;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.exception.AmbiguousTableException;
import org.apache.tajo.catalog.exception.UndefinedColumnException;
@@ -346,8 +347,9 @@ public abstract class NameResolver {
}
// column.nested_fieldX...
- if (guessedRelations.size() == 0 && qualifierParts.length == 1) {
- Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]);
+ if (guessedRelations.size() == 0 && qualifierParts.length > 0) {
+ Collection<RelationNode> rels = lookupTableByColumns(block, StringUtils.join(qualifierParts,
+ NestedPathUtil.PATH_DELIMITER, 0));
if (rels.size() > 1) {
throw new AmbiguousColumnException(columnRef.getCanonicalName());
@@ -376,12 +378,13 @@ public abstract class NameResolver {
columnName = qualifierParts[columnNamePosition];
// if qualifierParts include nested field names
- if (qualifierParts.length > columnNamePosition) {
- columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length);
+ if (qualifierParts.length > columnNamePosition + 1) {
+ columnName += NestedPathUtil.PATH_DELIMITER + StringUtils.join(qualifierParts, NestedPathUtil.PATH_DELIMITER,
+ columnNamePosition + 1, qualifierParts.length);
}
// columnRef always has a leaf field name.
- columnName += "/" + columnRef.getName();
+ columnName += NestedPathUtil.PATH_DELIMITER + columnRef.getName();
}
return new Pair<String, String>(qualifier, columnName);