You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/31 06:36:04 UTC
incubator-lens git commit: LENS-342 : Fail query when a dimension is
accessed with chain and without chain in the same query (amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master 4a5c02a02 -> f57a9a149
LENS-342 : Fail query when a dimension is accessed with chain and without chain in the same query (amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/f57a9a14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/f57a9a14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/f57a9a14
Branch: refs/heads/master
Commit: f57a9a149f0e670f47b586080f14113ccd533ed8
Parents: 4a5c02a
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Mar 31 10:05:56 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Mar 31 10:05:56 2015 +0530
----------------------------------------------------------------------
.../cube/metadata/ReferencedDimAtrribute.java | 2 +-
.../cube/parse/DenormalizationResolver.java | 5 +-
.../apache/lens/cube/parse/JoinResolver.java | 5 +
.../apache/lens/cube/parse/CubeTestSetup.java | 9 +-
.../lens/cube/parse/TestBaseCubeQueries.java | 6 +-
.../lens/cube/parse/TestJoinResolver.java | 152 +++++++++++--------
6 files changed, 106 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
index 859b344..a8ece2d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
@@ -28,7 +28,7 @@ import lombok.Getter;
import lombok.ToString;
@EqualsAndHashCode(callSuper = true)
-@ToString
+@ToString(callSuper = true)
public class ReferencedDimAtrribute extends BaseDimAttribute {
@Getter
private final List<TableReference> references = new ArrayList<TableReference>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 2d239e1..f5d2115 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -232,7 +232,7 @@ public class DenormalizationResolver implements ContextRewriter {
return false;
}
- private void pickColumnsForTable(String tbl) {
+ private void pickColumnsForTable(String tbl) throws SemanticException {
if (tableToRefCols.containsKey(tbl)) {
for (ReferencedQueriedColumn refered : tableToRefCols.get(tbl)) {
if (!refered.col.isChainedColumn()) {
@@ -245,6 +245,9 @@ public class DenormalizationResolver implements ContextRewriter {
iter.remove();
}
}
+ if (refered.references.isEmpty()) {
+ throw new SemanticException("No reference column available for " + refered);
+ }
PickedReference picked = new PickedReference(refered.references.iterator().next(),
cubeql.getAliasForTabName(refered.srcTable.getName()), tbl);
addPickedReference(refered.col.getName(), picked);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index 32a4b85..bf57907 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -1082,6 +1082,11 @@ class JoinResolver implements ContextRewriter {
throw new SemanticException("Table " + joinee.getName() + " is getting accessed via two different names: "
+ "[" + dimensionInJoinChain.get(joinee).get(0).getName() + ", " + joinee.getName() + "]");
}
+ // table is accessed with chain and no chain
+ if (cubeql.getNonChainedDimensions().contains(joinee)) {
+ throw new SemanticException("Table " + joinee.getName() + " is getting accessed via joinchain: "
+ + dimensionInJoinChain.get(joinee).get(0).getName() + " and no chain at all");
+ }
}
}
// populate paths from joinchains
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 7d08212..b2ae9b5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1362,6 +1362,7 @@ public class CubeTestSetup {
dimColumns = new ArrayList<FieldSchema>();
dimColumns.add(new FieldSchema("id", "int", "code"));
+ dimColumns.add(new FieldSchema("name", "string", "name"));
client.createCubeDimensionTable(cityDim.getName(), dimName, dimColumns, 0L, dumpPeriods, dimProps, storageTables);
@@ -1402,7 +1403,10 @@ public class CubeTestSetup {
dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "name")));
dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("testDim3id", "string", "f-key to testdim3"), "Dim3 refer",
new TableReference("testdim3", "id")));
- dimAttrs.add(new BaseDimAttribute(new FieldSchema("cityId ", "string", "name")));
+ dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("cityId", "string", "f-key to citydim"), "cityid",
+ new TableReference("citydim", "id")));
+ dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("cityname", "string", "name"), "cityid",
+ new TableReference("citydim", "name"), null, null, 0.0, false));
// add ref dim through chain
dimAttrs.add(new ReferencedDimAtrribute(
@@ -1450,6 +1454,9 @@ public class CubeTestSetup {
dimColumns.add(new FieldSchema("id", "int", "code"));
dimColumns.add(new FieldSchema("bigid1", "int", "code"));
dimColumns.add(new FieldSchema("name", "string", "field1"));
+ dimColumns.add(new FieldSchema("cityId", "string", "f-key to cityDim"));
+ storageTables.put(c3, s1);
+ dumpPeriods.put(c3, UpdatePeriod.HOURLY);
client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 10L, dumpPeriods, dimProps, storageTables);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 1f03db6..632829f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -89,7 +89,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
ErrorMsg.EXPRESSION_NOT_IN_ANY_FACT.getErrorCode());
// no fact has the all the dimensions queried
- e = getSemanticExceptionInRewrite("select dim1, cityid, msr3, msr13 from basecube" + " where " + TWO_DAYS_RANGE,
+ e = getSemanticExceptionInRewrite("select dim1, stateid, msr3, msr13 from basecube" + " where " + TWO_DAYS_RANGE,
conf);
Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(),
ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
@@ -100,12 +100,12 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
Assert.assertTrue(matcher.matches());
Assert.assertEquals(matcher.groupCount(), 1);
String columnSetsStr = matcher.group(1);
- Assert.assertNotEquals(columnSetsStr.indexOf("cityid"), -1);
+ Assert.assertNotEquals(columnSetsStr.indexOf("stateid"), -1);
Assert.assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1);
Assert.assertEquals(pruneCauses.getDetails(),
new HashMap<String, List<CandidateTablePruneCause>>() {
{
- put("testfact3_base,testfact3_raw_base", Arrays.asList(CandidateTablePruneCause.columnNotFound("cityid")));
+ put("testfact3_base,testfact3_raw_base", Arrays.asList(CandidateTablePruneCause.columnNotFound("stateid")));
put("testfact2_raw_base,testfact2_base",
Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr13")));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f57a9a14/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
index 0e5978e..dfa178b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
@@ -118,13 +118,18 @@ public class TestJoinResolver extends TestQueryRewrite {
CubeInterface testCube = metastore.getCube("testcube");
Dimension zipDim = metastore.getDimension("zipdim");
Dimension cityDim = metastore.getDimension("citydim");
+ Dimension testDim2 = metastore.getDimension("testDim2");
SchemaGraph.GraphSearch search = new SchemaGraph.GraphSearch(zipDim, (AbstractCubeTable) testCube, schemaGraph);
List<SchemaGraph.JoinPath> paths = search.findAllPathsToTarget();
- Assert.assertEquals(2, paths.size());
+ Assert.assertEquals(6, paths.size());
validatePath(paths.get(0), zipDim, (AbstractCubeTable) testCube);
validatePath(paths.get(1), zipDim, cityDim, (AbstractCubeTable) testCube);
+ validatePath(paths.get(2), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube);
+ validatePath(paths.get(3), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube);
+ validatePath(paths.get(4), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube);
+ validatePath(paths.get(5), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube);
}
private void validatePath(SchemaGraph.JoinPath jp, AbstractCubeTable... tables) {
@@ -450,34 +455,6 @@ public class TestJoinResolver extends TestQueryRewrite {
);
TestCubeRewriter.compareQueries(expected, hqlQuery);
- // Single joinchain with two paths, intermediate dimension accessed separately by name.
- query = "select cityState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
- hqlQuery = rewrite(query, hconf);
- expected = getExpectedQuery("basecube",
- "select citystate.name, citydim.name, sum(basecube.msr2) FROM ",
- " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and "
- + "citydim.dt = 'latest'"
- + " join " + getDbName() + "c1_statetable citystate on citydim.stateid = citystate.id and "
- + "citystate.dt = 'latest'", null, "group by citystate.name,citydim.name", null,
- getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")
- );
- TestCubeRewriter.compareQueries(expected, hqlQuery);
-
- // Multi joinchains + a dimension part of one of the chains.
- query = "select cityState.name, cubeState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
- hqlQuery = rewrite(query, hconf);
- expected = getExpectedQuery("basecube",
- "select citystate.name, cubestate.name, citydim.name, sum(basecube.msr2) FROM ",
- " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and "
- + "citydim.dt = 'latest'"
- + " join " + getDbName() + "c1_statetable citystate on citydim.stateid = citystate.id and "
- + "citystate.dt = 'latest'"
- + " join " + getDbName() + "c1_statetable cubestate on basecube.stateid=cubestate.id and cubestate.dt='latest'"
- , null, "group by citystate.name,cubestate.name,citydim.name", null,
- getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")
- );
- TestCubeRewriter.compareQueries(expected, hqlQuery);
-
// Two joinchains, one accessed as refcol.
query = "select cubestate.name, cityStateCapital, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
hqlQuery = rewrite(query, hconf);
@@ -526,44 +503,6 @@ public class TestJoinResolver extends TestQueryRewrite {
);
TestCubeRewriter.compareQueries(expected, hqlQuery);
- // this test case should pass when default qualifiers for dimensions' chains are added
- // Two joinchains with same destination, and the destination table accessed separately
- query = "select cityState.name, cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
- try {
- rewrite(query, hconf);
- Assert.fail("Should have failed. "
- + "It's not possible to resolve which statedim is being asked for when cityState and cubeState both end at"
- + " statedim table.");
- } catch (SemanticException e) {
- Assert.assertNotNull(e.getCause());
- Assert.assertEquals(
- e.getCause().getMessage().indexOf("Table statedim has 2 different paths through joinchains"), 0);
- }
-
- // this test case should pass when default qualifiers for dimensions' chains are added
- // Two Single joinchain, And dest table accessed separately.
- query = "select cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
- try {
- rewrite(query, hconf);
- Assert.fail("Should have failed. "
- + "The table statedim is getting accessed as both cubeState and statedim ");
- } catch (SemanticException e) {
- Assert.assertNotNull(e.getCause());
- Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
- "Table statedim is getting accessed via two different names: [cubestate, statedim]".toLowerCase());
- }
- // this should pass when default qualifiers are added
- query = "select cityStateCapital, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
- try {
- rewrite(query, hconf);
- Assert.fail("Should have failed. "
- + "The table statedim is getting accessed as both cubeState and statedim ");
- } catch (SemanticException e) {
- Assert.assertNotNull(e.getCause());
- Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
- "Table statedim is getting accessed via two different names: [citystate, statedim]".toLowerCase());
- }
-
// Test 4 Dim only query with join chains
List<String> expectedClauses = new ArrayList<String>();
@@ -631,6 +570,85 @@ public class TestJoinResolver extends TestQueryRewrite {
}
@Test
+ public void testConflictingJoins() throws ParseException {
+ // Single joinchain with two paths, intermediate dimension accessed separately by name.
+ String query = "select cityState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(query, hconf);
+ Assert.fail("Should have failed. "
+ + "The table citydim is getting accessed as both chain and without chain ");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
+ "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase());
+ }
+
+ // Multi joinchains + a dimension part of one of the chains.
+ query = "select cityState.name, cubeState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(query, hconf);
+ Assert.fail("Should have failed. "
+ + "The table citydim is getting accessed as both chain and without chain ");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
+ "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase());
+ }
+
+ // this test case should pass when default qualifiers for dimensions' chains are added
+ // Two joinchains with same destination, and the destination table accessed separately
+ query = "select cityState.name, cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(query, hconf);
+ Assert.fail("Should have failed. "
+ + "It's not possible to resolve which statedim is being asked for when cityState and cubeState both end at"
+ + " statedim table.");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(
+ e.getCause().getMessage().indexOf("Table statedim has 2 different paths through joinchains"), 0);
+ }
+
+ // this test case should pass when default qualifiers for dimensions' chains are added
+ // Two Single joinchain, And dest table accessed separately.
+ query = "select cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(query, hconf);
+ Assert.fail("Should have failed. "
+ + "The table statedim is getting accessed as both cubeState and statedim ");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
+ "Table statedim is getting accessed via two different names: [cubestate, statedim]".toLowerCase());
+ }
+ // this should pass when default qualifiers are added
+ query = "select cityStateCapital, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(query, hconf);
+ Assert.fail("Should have failed. "
+ + "The table statedim is getting accessed as both cubeState and statedim ");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
+ "Table statedim is getting accessed via two different names: [citystate, statedim]".toLowerCase());
+ }
+
+ // table accessed through denorm column and chain column
+ Configuration conf = new Configuration(hconf);
+ conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3, C4");
+ String failingQuery = "select testDim2.cityname, testDim2.cityStateCapital FROM testDim2 where " + TWO_DAYS_RANGE;
+ try {
+ rewrite(failingQuery, conf);
+ Assert.fail("Should have failed. "
+ + "The table citydim is getting accessed as both chain and without chain ");
+ } catch (SemanticException e) {
+ Assert.assertNotNull(e.getCause());
+ Assert.assertEquals(e.getCause().getMessage().toLowerCase(),
+ "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase());
+ }
+ }
+
+ @Test
public void testMultiPaths() throws SemanticException, ParseException {
String query, hqlQuery, expected;