You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pu...@apache.org on 2017/02/21 11:02:53 UTC
lens git commit: Fix failing TestCases in
org.apache.lens.server.query.QueryAPIErrorResponseTest
Repository: lens
Updated Branches:
refs/heads/lens-1381 b26632a15 -> 4d493597e
Fix failing TestCases in org.apache.lens.server.query.QueryAPIErrorResponseTest
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4d493597
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4d493597
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4d493597
Branch: refs/heads/lens-1381
Commit: 4d493597e56248f0ed5d3d1d690d52212e2e9abe
Parents: b26632a
Author: Puneet Gupta <pu...@apache.org>
Authored: Tue Feb 21 16:32:17 2017 +0530
Committer: Puneet <pu...@inmobi.com>
Committed: Tue Feb 21 16:32:17 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CubeQueryRewriter.java | 49 ++++++++++++--------
.../lens/server/common/RestAPITestUtil.java | 11 +++++
.../lens/server/common/TestDataUtils.java | 4 ++
.../server/query/QueryAPIErrorResponseTest.java | 27 +++++++++++
4 files changed, 71 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index 4dd3d00..f052a2f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -138,61 +138,70 @@ public class CubeQueryRewriter {
rewriters.add(new ColumnResolver(conf));
// Rewrite base trees (groupby, having, orderby, limit) using aliases
rewriters.add(new AliasReplacer(conf));
+
ExpressionResolver exprResolver = new ExpressionResolver(conf);
DenormalizationResolver denormResolver = new DenormalizationResolver(conf);
CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf);
StorageTableResolver storageTableResolver = new StorageTableResolver(conf);
- // Resolve expressions
+
+ // Phase 1 of exprResolver: Resolve expressions
rewriters.add(exprResolver);
- // De-normalized columns resolved
+ // Phase 1 of denormResolver: De-normalized columns resolved
rewriters.add(denormResolver);
// Resolve time ranges
rewriters.add(new TimerangeResolver(conf));
- // Resolve candidate fact tables and dimension tables for columns queried
+ // Phase 1 of candidateTblResolver: Resolve candidate storages and dimension tables for columns queried
rewriters.add(candidateTblResolver);
// Resolve aggregations and generate base select tree
rewriters.add(new AggregateResolver());
rewriters.add(new GroupbyResolver(conf));
+ //validate fields queryability (in case of derived cubes setup)
rewriters.add(new FieldValidator());
- rewriters.add(storageTableResolver);
- //TODO union: Add CoveringSetResolver which creates UnionCandidates and JoinCandidates.
- //TODO union: Some code form candidateTblResolver(phase 2) to be moved to CoveringSetResolver
- //TODO union: AggregateResolver,GroupbyResolver,FieldValidator before CoveringSetResolver
// Resolve joins and generate base join tree
rewriters.add(new JoinResolver(conf));
- // Do col life validation
+ // Do col life validation for the time range(s) queried
rewriters.add(new TimeRangeChecker(conf));
- // Resolve candidate fact tables and dimension tables for columns included
+ // Phase 1 of storageTableResolver: Validate and prune candidate storages
+ rewriters.add(storageTableResolver);
+ // Phase 2 of candidateTblResolver: Resolve candidate storages and dimension tables for columns included
// in join and denorm resolvers
- //TODO union : this should be CoveringSetResolver now
rewriters.add(candidateTblResolver);
+ // Find Union and Join combinations over Storage Candidates that can answer the queried time range(s) and all
+ // queried measures
rewriters.add(new CandidateCoveringSetsResolver(conf));
- // Phase 1: resolve fact tables.
- //TODO union: This phase 1 of storageTableResolver should happen before CoveringSetResolver
+
+ // If lightest fact first option is enabled for this driver (via lens.cube.query.pick.lightest.fact.first = true),
+ // run LightestFactResolver and keep only the lighted combination(s) generated by CandidateCoveringSetsResolver
if (lightFactFirst) {
// Prune candidate tables for which denorm column references do not exist
- //TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver
rewriters.add(denormResolver);
- // Prune candidate facts without any valid expressions
+ // Phase 2 of exprResolver:Prune candidate facts without any valid expressions
rewriters.add(exprResolver);
+ // Pick the least cost combination(s) (and prune others) out of a set of combinations produced
+ // by CandidateCoveringSetsResolver
rewriters.add(new LightestFactResolver(conf));
}
- // Phase 2: resolve fact table partitions.
+
+ // Phase 2 of storageTableResolver: resolve storage table partitions.
rewriters.add(storageTableResolver);
+ // In case partial data is allowed (via lens.cube.query.fail.if.data.partial = false) and there are many
+ // combinations with partial data, pick the one that covers the maximum part of time ranges(s) queried
rewriters.add(new MaxCoveringFactResolver(conf));
- // Phase 3: resolve dimension tables and partitions.
+ // Phase 3 of storageTableResolver: resolve dimension tables and partitions.
rewriters.add(storageTableResolver);
// Prune candidate tables for which denorm column references do not exist
//TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver.. check if this makes sense
rewriters.add(denormResolver);
- // Prune candidate facts without any valid expressions
+ // Phase 2 of exprResolver : Prune candidate facts without any valid expressions
rewriters.add(exprResolver);
- // We can have LightestFactResolver before LeastPartitionResolver - that says
- // "if two facts have the same least weight, then the fact with least number of time partitions queried will be
- // picked". This will be useful, if users did not set fact weights.
+
if (!lightFactFirst) {
+ // Pick the least cost combination(s) (and prune others) out of a set of combinations produced
+ // by CandidateCoveringSetsResolver
rewriters.add(new LightestFactResolver(conf));
}
+ // if two combinations have the same least weight/cost, then the combination with least number of time partitions
+ // queried will be picked. Rest of the combinations will be pruned
rewriters.add(new LeastPartitionResolver(conf));
rewriters.add(new LightestDimensionResolver(conf));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
index 02e2f8b..df4e07a 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
@@ -39,6 +39,7 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.metastore.ObjectFactory;
import org.apache.lens.api.metastore.XCube;
import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.metastore.XStorage;
import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
@@ -166,6 +167,16 @@ public class RestAPITestUtil {
checkResponse(result);
}
+ public static void createStorageFailFast(final WebTarget target, final LensSessionHandle sessionId,
+ final XStorage storage, MediaType mt) {
+ APIResult result = target.path("metastore").path("storages").queryParam("sessionid", sessionId)
+ .request(mt).post(Entity.entity(
+ new GenericEntity<JAXBElement<XStorage>>(cubeObjectFactory.createXStorage(storage)) {
+ }, mt),
+ APIResult.class);
+ checkResponse(result);
+ }
+
public static APIResult setCurrentDatabase(final WebTarget target, final LensSessionHandle sessionId,
final String dbName, MediaType mt) {
http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
index 0400519..fbfd898 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
@@ -50,6 +50,10 @@ public class TestDataUtils {
return "db" + getRandomName();
}
+ public static String getRandomStorageName() {
+ return "storage" + getRandomName();
+ }
+
public static String getRandomCubeName() {
return "cube" + getRandomName();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 567c929..064da01 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -47,9 +47,11 @@ import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver;
import org.apache.lens.cube.error.ColUnAvailableInTimeRange;
+import org.apache.lens.cube.metadata.HDFSStorage;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensRequestLoggingFilter;
import org.apache.lens.server.common.ErrorResponseExpectedData;
+import org.apache.lens.server.common.RestAPITestUtil;
import org.apache.lens.server.error.GenericExceptionMapper;
import org.apache.lens.server.error.LensJAXBValidationExceptionMapper;
import org.apache.lens.server.metastore.MetastoreResource;
@@ -255,6 +257,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
final String testCube = getRandomCubeName();
final String testDimensionField = getRandomDimensionField();
final String testFact = getRandomFactName();
+ final String testStorage = getRandomStorageName();
/* Setup: Begin */
LensSessionHandle sessionId = openSession(target, "foo", "bar", new LensConf(), mt);
@@ -268,9 +271,21 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
XCube xcube = createXCubeWithDummyMeasure(testCube, Optional.of("dt"), testXDim);
createCubeFailFast(target, sessionId, xcube, mt);
+ /* Create Storage */
+ XStorage xs = new XStorage();
+ xs.setClassname(HDFSStorage.class.getCanonicalName());
+ xs.setName(testStorage);
+ RestAPITestUtil.createStorageFailFast(target, sessionId, xs, mt);
+
/* Create a fact with test dimension field */
XColumn xColumn = createXColumn(testDimensionField);
XFactTable xFactTable = createXFactTableWithColumns(testFact, testCube, xColumn);
+
+ //Create a StorageTable
+ XStorageTables tables = new XStorageTables();
+ tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY"));
+ xFactTable.setStorageTables(tables);
+
createFactFailFast(target, sessionId, xFactTable, mt);
/* Setup: End */
@@ -343,6 +358,18 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
} finally {
closeSessionFailFast(target(), sessionId, mt);
}
+ }
+ private XStorageTableElement createStorageTblElement(String storageName, String... updatePeriod) {
+ XStorageTableElement tbl = new XStorageTableElement();
+ tbl.setUpdatePeriods(new XUpdatePeriods());
+ tbl.setStorageName(storageName);
+ if (updatePeriod != null) {
+ for (String p : updatePeriod) {
+ tbl.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p));
+ }
+ }
+ tbl.setTableDesc(new XStorageTableDesc());
+ return tbl;
}
}