You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/07/13 09:13:54 UTC
[01/12] lens git commit: LENS-1441: CandidateTableResolver should not
add StorageCandidate if storage is not supported
Repository: lens
Updated Branches:
refs/heads/current-release-line 051412bec -> 6dca44661
LENS-1441: CandidateTableResolver should not add StorageCandidate if storage is not supported
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c2a9c931
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c2a9c931
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c2a9c931
Branch: refs/heads/current-release-line
Commit: c2a9c9316cc5d3a06b02a157722fa58f2757d47a
Parents: 051412b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Jun 20 16:08:00 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:48 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateTableResolver.java | 24 +++++++++++++++++++-
.../lens/cube/parse/CubeQueryRewriter.java | 2 +-
.../lens/cube/parse/ExpressionResolver.java | 5 +++-
.../lens/cube/parse/StorageTableResolver.java | 6 -----
.../cube/parse/TestDenormalizationResolver.java | 10 ++------
.../lens/cube/parse/TestTimeRangeResolver.java | 20 +++++++---------
.../server/query/QueryAPIErrorResponseTest.java | 1 +
7 files changed, 39 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index f530650..be3b474 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -29,6 +29,7 @@ import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -48,8 +49,15 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
class CandidateTableResolver implements ContextRewriter {
+ private final List<String> supportedStorages;
+ private final boolean allStoragesSupported;
private boolean checkForQueriedColumns = true;
+ public CandidateTableResolver(Configuration conf) {
+ this.supportedStorages = getSupportedStorages(conf);
+ this.allStoragesSupported = (supportedStorages == null);
+ }
+
@Override
public void rewriteContext(CubeQueryContext cubeql) throws LensException {
if (checkForQueriedColumns) {
@@ -81,6 +89,18 @@ class CandidateTableResolver implements ContextRewriter {
checkForQueriedColumns = true;
}
}
+ private List<String> getSupportedStorages(Configuration conf) {
+ String[] storages = conf.getStrings(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES);
+ if (storages != null) {
+ return Arrays.asList(storages);
+ }
+ return null;
+ }
+
+ private boolean isStorageSupportedOnDriver(String storage) {
+ return allStoragesSupported || supportedStorages.contains(storage);
+ }
+
private void populateCandidateTables(CubeQueryContext cubeql) throws LensException {
if (cubeql.getCube() != null) {
@@ -95,7 +115,9 @@ class CandidateTableResolver implements ContextRewriter {
} else {
for (String s : fact.getStorages()) {
StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, s, cubeql);
- cubeql.getCandidates().add(sc);
+ if (isStorageSupportedOnDriver(sc.getStorageName())) {
+ cubeql.getCandidates().add(sc);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index 0ef41f3..143b266 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -147,7 +147,7 @@ public class CubeQueryRewriter {
rewriters.add(new AliasReplacer());
ExpressionResolver exprResolver = new ExpressionResolver();
DenormalizationResolver denormResolver = new DenormalizationResolver();
- CandidateTableResolver candidateTblResolver = new CandidateTableResolver();
+ CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf);
StorageTableResolver storageTableResolver = new StorageTableResolver(conf);
LightestFactResolver lightestFactResolver = new LightestFactResolver();
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index b1654d1..2403576 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -556,7 +556,10 @@ class ExpressionResolver implements ContextRewriter {
log.info("Removing expression {} as all tables have non reachable fields", esc);
iterator.remove();
removedEsc.add(esc);
- break;
+ removed = true;
+ }
+ if (removed) {
+ continue;
}
//remove expressions which are not valid in the timerange queried
// If an expression is defined as
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 3acd754..d7da8cb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -252,12 +252,6 @@ class StorageTableResolver implements ContextRewriter {
if (c instanceof StorageCandidate) {
StorageCandidate sc = (StorageCandidate) c;
// first check: if the storage is supported on driver
- if (!isStorageSupportedOnDriver(sc.getStorageName())) {
- log.info("Skipping storage: {} as it is not supported", sc.getStorageName());
- cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE));
- it.remove();
- continue;
- }
String str = conf.get(CubeQueryConfUtil.getValidStorageTablesKey(sc.getFact().getName()));
List<String> validFactStorageTables =
StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index ffd0dec..9b217ae 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -200,16 +200,10 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
Map<Set<String>, List<CandidateTablePruneCause>> expected = Maps.newHashMap();
expected.put(newHashSet("c1_summary1", "c1_testfact", "c1_testfact2"),
newArrayList(columnNotFound("dim2big2")));
- expected.put(newHashSet("c2_summary2", "c2_summary3", "c1_testfact2_raw", ""
- + "c3_testfact2_raw", "c1_summary3", "c1_summary2"),
+ expected.put(newHashSet("c1_testfact2_raw", "c1_summary3", "c1_summary2"),
newArrayList(new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE)));
- expected.put(newHashSet("c0_b1b2fact1", "c0_testfact_continuous", "SEG[b1cube; b2cube]"),
+ expected.put(newHashSet("SEG[b1cube; b2cube]"),
newArrayList(columnNotFound("msr2", "msr3")));
- expected.put(newHashSet("c2_summary2", "c2_summary3", "c2_summary4", "c4_testfact", "c2_summary1",
- "c3_testfact", "c3_testfact2_raw", "c6_testfact", "c4_testfact2", "c5_testfact", "c99_cheapfact",
- "c2_testfact", "c0_cheapfact", "c2_testfactmonthly", "c0_testfact"),
- newArrayList(new CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE)));
-
Assert.assertEquals(enhanced, expected);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
index 929fb46..181608f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,9 +21,9 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.metadata.DateFactory.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.UNSUPPORTED_STORAGE;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
import java.util.*;
@@ -41,8 +41,6 @@ import org.testng.annotations.Test;
public class TestTimeRangeResolver extends TestQueryRewrite {
- private final String cubeName = CubeTestSetup.TEST_CUBE_NAME;
-
private Configuration conf;
@BeforeTest
@@ -62,7 +60,7 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
@Test
public void testFactValidity() throws ParseException, LensException, HiveException, ClassNotFoundException {
- String query = "select msr2 from " + cubeName + " where " + LAST_YEAR_RANGE;
+ String query = "select msr2 from " + CubeTestSetup.TEST_CUBE_NAME + " where " + LAST_YEAR_RANGE;
LensException e = getLensExceptionInRewrite(query, getConf());
assertEquals(e.getErrorInfo().getErrorName(), "NO_UNION_CANDIDATE_AVAILABLE");
}
@@ -74,13 +72,11 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
getConf());
List<CandidateTablePruneCause> causes = findPruningMessagesForStorage("c3_testfact_deprecated",
ctx.getStoragePruningMsgs());
- assertEquals(causes.size(), 1);
- assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE);
+ assertTrue(causes.isEmpty());
causes = findPruningMessagesForStorage("c4_testfact_deprecated",
ctx.getStoragePruningMsgs());
- assertEquals(causes.size(), 1);
- assertEquals(causes.get(0).getCause(), UNSUPPORTED_STORAGE);
+ assertTrue(causes.isEmpty());
// testfact_deprecated's validity should be in between of both ranges. So both ranges should be in the invalid list
// That would prove that parsing of properties has gone through successfully
@@ -109,8 +105,8 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
/**
*
* @param stoargeName storageName_factName
- * @param allStoragePruningMsgs
- * @return
+ * @param allStoragePruningMsgs all pruning messages
+ * @return pruning messages for storagetable
*/
private static List<CandidateTablePruneCause> findPruningMessagesForStorage(String stoargeName,
PruneCauses<Candidate> allStoragePruningMsgs) {
@@ -121,6 +117,6 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
}
}
}
- return new ArrayList<CandidateTablePruneCause>();
+ return new ArrayList<>();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c2a9c931/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 5409d21..82bb505 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -284,6 +284,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
//Create a StorageTable
XStorageTables tables = new XStorageTables();
tables.getStorageTable().add(createStorageTblElement(testStorage, "DAILY"));
+ tables.getStorageTable().add(createStorageTblElement("mydb", "DAILY")); // for jdbc
xFactTable.setStorageTables(tables);
createFactFailFast(target, sessionId, xFactTable, mt);
[12/12] lens git commit: LENS-1452: Optimize Time Union candidate
Algorithm
Posted by pr...@apache.org.
LENS-1452: Optimize Time Union candidate Algorithm
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6dca4466
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6dca4466
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6dca4466
Branch: refs/heads/current-release-line
Commit: 6dca44661bf604ca1436c6cd1d3998405d0333a4
Parents: 3769ef0
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Jul 10 16:52:54 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:43:03 2017 +0530
----------------------------------------------------------------------
.../parse/CandidateCoveringSetsResolver.java | 47 +++++++++++++++++---
1 file changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/6dca4466/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 8e07162..69d9562 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -26,6 +26,7 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.TimeRange;
import org.apache.lens.server.api.error.LensException;
+import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -124,16 +125,14 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
}
}
// Get all covering fact sets
- List<UnionCandidate> unionCoveringSet =
- getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
+// List<UnionCandidate> unionCoveringSet = getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
+ List<UnionCandidate> unionCoveringSet = getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
// Sort the Collection based on no of elements
unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
// prune candidate set which doesn't contain any common measure i
if (!queriedMsrs.isEmpty()) {
pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, queriedMsrs);
}
- // prune redundant covering sets
- pruneRedundantUnionCoveringSets(unionCoveringSet);
// pruing done in the previous steps, now create union candidates
candidateSet.addAll(unionCoveringSet);
updateQueriableMeasures(candidateSet, qpcList);
@@ -155,7 +154,7 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
}
}
}
-
+ @Deprecated
private void pruneRedundantUnionCoveringSets(List<UnionCandidate> candidates) {
for (int i = 0; i < candidates.size(); i++) {
UnionCandidate current = candidates.get(i);
@@ -168,7 +167,7 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
}
}
}
-
+ @Deprecated
private List<UnionCandidate> getCombinations(final List<Candidate> candidates, CubeQueryContext cubeql) {
List<UnionCandidate> combinations = new LinkedList<>();
int size = candidates.size();
@@ -193,6 +192,42 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
return combinations;
}
+ /**
+ * The following function is iterative rewrite of the following tail-recursive implementation:
+ * (ignoring cubeql for clarity)
+ * getCombinations(candidates) = getCombinationsTailRecursive(emptyList(), candidates)
+ *
+ * getCombinationsTailRecursive(incompleteCombinations: List[List[Candidate]], candidates: List[Candidate]) =
+ * head, tail = head and tail of linked List candidates
+ * add head to all elements of incompleteCombinations.
+ * complete = remove now complete combinations from incompleteCombinations
+ * return complete ++ getCombinationsTailRecursive(incompleteCombinations, tail)
+ * @param candidates
+ * @param cubeql
+ * @return
+ */
+ private List<UnionCandidate> getCombinationTailIterative(List<Candidate> candidates, CubeQueryContext cubeql) {
+ LinkedList<Candidate> candidateLinkedList = Lists.newLinkedList(candidates);
+ List<List<Candidate>> incompleteCombinations = Lists.newArrayList();
+ List<UnionCandidate> unionCandidates = Lists.newArrayList();
+
+ while(!candidateLinkedList.isEmpty()) {
+ Candidate candidate = candidateLinkedList.remove();
+ incompleteCombinations.add(Lists.newArrayList());
+ Iterator<List<Candidate>> iter = incompleteCombinations.iterator();
+ while(iter.hasNext()) {
+ List<Candidate> incompleteCombination = iter.next();
+ incompleteCombination.add(candidate);
+ UnionCandidate unionCandidate = new UnionCandidate(incompleteCombination, cubeql);
+ if (isCandidateCoveringTimeRanges(unionCandidate, cubeql.getTimeRanges())) {
+ unionCandidates.add(unionCandidate);
+ iter.remove();
+ }
+ }
+ }
+ return unionCandidates;
+ }
+
private List<List<Candidate>> resolveJoinCandidates(List<Candidate> candidates,
Set<QueriedPhraseContext> msrs) throws LensException {
List<List<Candidate>> msrCoveringSets = new ArrayList<>();
[10/12] lens git commit: LENS-1449 : lens.query.result.parent.dir to
be set at driver level.
Posted by pr...@apache.org.
LENS-1449 : lens.query.result.parent.dir to be set at driver level.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/41051ea4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/41051ea4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/41051ea4
Branch: refs/heads/current-release-line
Commit: 41051ea47f164f004f5fccaa7ea91765007811fc
Parents: 503a46d
Author: Raghavendra Singh <ra...@apache.org>
Authored: Thu Jul 6 14:03:59 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:59 2017 +0530
----------------------------------------------------------------------
.../api/query/DriverSelectorQueryContext.java | 4 +++-
.../lens/server/api/query/QueryContext.java | 5 +++++
.../lens/server/query/TestQueryService.java | 6 ++++--
.../lens/server/query/TestResultFormatting.java | 20 +++++++++++++++++++-
.../drivers/hive/hive1/hivedriver-site.xml | 5 +++++
.../drivers/hive/hive2/hivedriver-site.xml | 6 ++++++
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 5 +++++
lens-server/src/test/resources/lens-site.xml | 5 -----
8 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 8e431d1..41e533e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -155,7 +155,9 @@ public class DriverSelectorQueryContext {
private Configuration mergeConf(LensDriver driver, Configuration queryConf) {
Configuration conf = new Configuration(driver.getConf());
for (Map.Entry<String, String> entry : queryConf) {
- conf.set(entry.getKey(), entry.getValue());
+ if (!conf.getFinalParameters().contains(entry.getKey())) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
}
conf.setClassLoader(queryConf.getClassLoader());
return conf;
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 9923589..ccdef87 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -328,6 +328,11 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
}
public String getResultSetParentDir() {
+ if (getSelectedDriver() != null && getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) != null) {
+ log.info("Fetching Parent Dir from driver conf:- "
+ + getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR));
+ return getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR);
+ }
return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 1149696..b7cdb88 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -848,9 +848,11 @@ public class TestQueryService extends LensJerseyTest {
LensResultSet rs = queryService.getResultset(handle3);
//check persisted result path
String expectedPath =
- ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString()
+ ctx3.getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString()
+ ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN);
- assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath));
+ assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath)
+ , "Result Path " + ((PersistentResultSet) rs).getOutputPath()
+ + " does not contain expected path: " + expectedPath);
validateHttpEndPoint(target(), null, handle3, null);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
index 4f8b8ff..76d3720 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
@@ -157,6 +157,22 @@ public class TestResultFormatting extends LensJerseyTest {
}
/**
+ * Test result formatter hdfs persistent result location override from driver conf.
+ *
+ * @throws InterruptedException the interrupted exception
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ @Test
+ public void testResultDirOverrideFromDriverConf() throws InterruptedException, IOException {
+ LensConf conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true");
+ LensQuery lensQuery = testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false,
+ null, MediaType.APPLICATION_XML_TYPE);
+ assertEquals(lensQuery.getResultSetPath(), "file:"+System.getProperty("user.dir")+"/target/hive-lens-results/"
+ +lensQuery.getQueryHandle().getHandleIdString()+".csv");
+ }
+
+ /**
* Test persistent result with max size.
*
* @throws InterruptedException the interrupted exception
@@ -196,7 +212,7 @@ public class TestResultFormatting extends LensJerseyTest {
* @throws InterruptedException the interrupted exception
* @throws IOException Signals that an I/O exception has occurred.
*/
- private void testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt)
+ private LensQuery testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt)
throws InterruptedException, IOException {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
@@ -266,6 +282,8 @@ public class TestResultFormatting extends LensJerseyTest {
|| ctx.getStatus().getStatusMessage().equals(ResultFormatter.ERROR_MESSAGE));
assertEquals(ctx.getStatus().getErrorMessage(), "Class NonexistentSerde.class not found");
}
+
+ return ctx;
}
@AfterTest
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index 1f0ff43..83143e7 100644
--- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -82,4 +82,9 @@
</description>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/hive-lens-results</value>
+ <final>true</final>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
index eb902e0..4170e9b 100644
--- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
@@ -82,4 +82,10 @@
</description>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/hive-lens-results</value>
+ <final>true</final>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
index 5b8b43f..ad068f8 100644
--- a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
+++ b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
@@ -55,4 +55,9 @@
<name>lens.driver.test.drivername</name>
<value>jdbc/jdbc1</value>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/jdbc-lens-results</value>
+ <final>true</final>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 0060fa7..7e5f522 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -49,11 +49,6 @@
<name>lens.server.persist.location</name>
<value>target/persist-dir</value>
</property>
-
- <property>
- <name>lens.query.result.parent.dir</name>
- <value>target/lens-results</value>
- </property>
<property>
<name>hive.server2.authentication</name>
[04/12] lens git commit: LENS-1444: Optimize the algorithm of finding
all eligible union candidates
Posted by pr...@apache.org.
LENS-1444: Optimize the algorithm of finding all eligible union candidates
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7f407bc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7f407bc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7f407bc
Branch: refs/heads/current-release-line
Commit: a7f407bcb59ffa84c8ab6e830ba98aee81516085
Parents: c174583
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jun 23 16:39:37 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:52 2017 +0530
----------------------------------------------------------------------
.../cube/parse/CandidateCoveringSetsResolver.java | 17 ++++-------------
1 file changed, 4 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/a7f407bc/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 61c28c6..8e07162 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -106,16 +106,6 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
return true;
}
- private void pruneUnionCandidatesNotCoveringAllRanges(List<UnionCandidate> ucs, CubeQueryContext cubeql) {
- for (Iterator<UnionCandidate> itr = ucs.iterator(); itr.hasNext();) {
- UnionCandidate uc = itr.next();
- if (!isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
- itr.remove();
- cubeql.addCandidatePruningMsg(uc, CandidateTablePruneCause.storageNotAvailableInRange(cubeql.getTimeRanges()));
- }
- }
- }
-
private List<Candidate> resolveTimeRangeCoveringFactSet(CubeQueryContext cubeql,
Set<QueriedPhraseContext> queriedMsrs, List<QueriedPhraseContext> qpcList) throws LensException {
List<Candidate> candidateSet = new ArrayList<>();
@@ -138,8 +128,6 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
// Sort the Collection based on no of elements
unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
- // prune non covering sets
- pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql);
// prune candidate set which doesn't contain any common measure i
if (!queriedMsrs.isEmpty()) {
pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, queriedMsrs);
@@ -197,7 +185,10 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
clonedI = clonedI >>> 1;
--count;
}
- combinations.add(new UnionCandidate(individualCombinationList, cubeql));
+ UnionCandidate uc = new UnionCandidate(individualCombinationList, cubeql);
+ if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+ combinations.add(uc);
+ }
}
return combinations;
}
[06/12] lens git commit: LENS-1437 : Missing unit test cases for
virtual fact
Posted by pr...@apache.org.
LENS-1437 : Missing unit test cases for virtual fact
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fbad3507
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fbad3507
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fbad3507
Branch: refs/heads/current-release-line
Commit: fbad350798a342a4de5e5b8b566a6f055338fde3
Parents: d49f45a
Author: Rajitha R <ra...@gmail.com>
Authored: Tue Jun 27 16:12:56 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:54 2017 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CubeTestSetup.java | 4 +-
.../lens/cube/parse/TestCubeRewriter.java | 18 +--
.../lens/cube/parse/TestVirtualFactQueries.java | 123 +++++++++++++++++++
.../resources/schema/cubes/base/testcube2.xml | 31 +++++
.../resources/schema/cubes/base/virtualcube.xml | 23 +++-
.../resources/schema/facts/testfact7_base.xml | 67 ++++++++++
.../resources/schema/facts/testfact8_base.xml | 58 +++++++++
.../resources/schema/facts/testfact9_base.xml | 57 +++++++++
.../schema/facts/virtual/virtualfact.xml | 2 +-
9 files changed, 361 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 48869c2..0366e56 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -364,7 +364,7 @@ public class CubeTestSetup {
// storageName[0] is hourly
// storageName[1] is daily
// storageName[2] is monthly
- public static Map<String, String> getWhereForMonthlyDailyAndHourly2months(String... storageTables) {
+ public static Map<String, String> getWhereForMonthlyDailyAndHourly2months(String cubeName, String... storageTables) {
Map<String, String> storageTableToWhereClause = new LinkedHashMap<String, String>();
List<String> hourlyparts = new ArrayList<String>();
List<String> dailyparts = new ArrayList<String>();
@@ -411,7 +411,7 @@ public class CubeTestSetup {
tables.append(storageTables[0]);
}
Collections.sort(parts);
- storageTableToWhereClause.put(tables.toString(), StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
+ storageTableToWhereClause.put(tables.toString(), StorageUtil.getWherePartClause("dt", cubeName, parts));
return storageTableToWhereClause;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index d8c7335..1c75e6c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -145,22 +145,6 @@ public class TestCubeRewriter extends TestQueryRewrite {
// assertNotNull(rewrittenQuery.getNonExistingParts());
}
-
- @Test
- public void testVirtualFactCubeSimpleQuery() throws Exception {
- Configuration conf = getConf();
- conf.set(DRIVER_SUPPORTED_STORAGES, "C1");
- CubeQueryContext rewrittenQuery =
- rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1"));
- String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ",
- null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME,
- "C1_summary1"));
- String hql = rewrittenQuery.toHQL();
- compareQueries(hql, expected);
- System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts());
- }
-
-
@Test
public void testMaxCoveringFact() throws Exception {
Configuration conf = getConf();
@@ -954,7 +938,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
rewrite("select SUM(msr2) from testCube" + " where " + TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2"));
String expected =
getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2months("C2_testfact"));
+ getWhereForMonthlyDailyAndHourly2months(TEST_CUBE_NAME, "C2_testfact"));
compareQueries(hqlQuery, expected);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
new file mode 100644
index 0000000..4bcb376
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.parse;
+
+import static org.apache.lens.cube.metadata.DateFactory.*;
+import static org.apache.lens.cube.parse.CubeQueryConfUtil.*;
+import static org.apache.lens.cube.parse.CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER;
+import static org.apache.lens.cube.parse.CubeTestSetup.*;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.LensServerAPITestUtil;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class TestVirtualFactQueries extends TestQueryRewrite {
+
+ private Configuration conf;
+
+ @BeforeTest
+ public void setupDriver() throws Exception {
+ conf = LensServerAPITestUtil.getConfiguration(
+ DRIVER_SUPPORTED_STORAGES, "C1",
+ DISABLE_AUTO_JOINS, false,
+ ENABLE_SELECT_TO_GROUPBY, true,
+ ENABLE_GROUP_BY_TO_SELECT, true,
+ DISABLE_AGGREGATE_RESOLVER, false);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return new Configuration(conf);
+ }
+
+ @Test
+ public void testVirtualFactDayQuery() throws Exception {
+ CubeQueryContext rewrittenQuery =
+ rewriteCtx("select SUM(msr1) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1"));
+ String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr1) as `sum(msr1)` FROM ",
+ null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME,
+ "C1_testfact9_base"));
+ String hql = rewrittenQuery.toHQL();
+ compareQueries(hql, expected);
+ }
+
+ @Test
+ public void testVirtualFactMonthQuery() throws Exception {
+
+ CubeQueryContext rewrittenQuery =
+ rewriteCtx("select SUM(msr1) from virtualCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C1"));
+ String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr1) as `sum(msr1)` FROM ",
+ null, "AND ( dim1 = 10 )", getWhereForMonthlyDailyAndHourly2months("virtualcube",
+ "C1_testfact9_base"));
+ String hql = rewrittenQuery.toHQL();
+ compareQueries(hql, expected);
+ }
+
+ static void compareQueries(String actual, String expected) {
+ assertEquals(new TestQuery(actual), new TestQuery(expected));
+ }
+
+ @Test
+ public void testVirtualFactUnionQuery() throws Exception {
+
+ String expectedInnerSelect = getExpectedQuery("virtualcube", "SELECT (virtualcube.cityid) AS `alias0`,"
+ + " sum((virtualcube.msr2)) AS `alias1`,0.0 AS `alias2` FROM ",
+ null, null, "GROUP BY (virtualcube.cityid)", null,
+ getWhereForDailyAndHourly2days("virtualcube", "c1_testfact8_base"))
+ + " UNION ALL " + getExpectedQuery("virtualcube", "SELECT (virtualcube.cityid) AS `alias0`,"
+ + "0.0 AS `alias1`, sum((virtualcube.msr3)) AS `alias2` FROM ",
+ null, null, "GROUP BY (virtualcube.cityid)", null,
+ getWhereForDailyAndHourly2days("virtualcube", "c1_testfact7_base"));
+
+
+ String expected = "SELECT (virtualcube.alias0) AS `cityid`,"
+ + " sum((virtualcube.alias1)) AS `sum(msr2)`, sum((virtualcube.alias2)) AS `sum(msr3)`"
+ + " FROM (" + expectedInnerSelect + ") AS virtualcube GROUP BY (virtualcube.alias0)";
+
+ CubeQueryContext rewrittenQuery =
+ rewriteCtx("select cityid as `cityid`, SUM(msr2), SUM(msr3) from virtualcube where " + TWO_DAYS_RANGE,
+ getConfWithStorages("C1"));
+ String hql = rewrittenQuery.toHQL();
+ compareQueries(hql, expected);
+ }
+
+ @Test
+ public void testVirtualFactJoinQuery() throws Exception {
+ String query, hqlQuery, expected;
+
+ // Single joinchain with direct link
+ query = "select cubestate.name, sum(msr2) from virtualcube where " + TWO_DAYS_RANGE + " group by cubestate.name";
+ hqlQuery = rewrite(query, conf);
+ expected = getExpectedQuery("virtualcube", "SELECT (cubestate.name) as `name`, sum((virtualcube.msr2)) "
+ + "as `sum(msr2)` FROM ",
+ " join " + getDbName() + "c1_statetable cubestate ON virtualcube.stateid=cubeState.id and cubeState.dt= 'latest'",
+ null, "group by cubestate.name",
+ null, getWhereForDailyAndHourly2days("virtualcube", "c1_testfact8_base"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
new file mode 100644
index 0000000..9740e00
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="testcube2" xmlns="uri:lens:cube:0.1">
+ <properties>
+ <property name="cube.timedim.partition.d_time" value="dt"/>
+ <property name="cube.testcube.timed.dimensions.list" value="d_time"/>
+ </properties>
+ <measures>
+ <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr1" display_string="Measure1"
+ description="first measure"/>
+ </measures>
+</x_base_cube>
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml
index 2a5e0b0..187ac29 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml
@@ -25,12 +25,31 @@
<property name="cube.virtualcube.timed.dimensions.list" value="d_time"/>
</properties>
<measures>
+ <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr1" display_string="Measure1"
+ description="first measure"/>
<measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr2" display_string="Measure2"
description="second measure"/>
+ <measure _type="DOUBLE" default_aggr="SUM" unit="RS" name="msr3" display_string="Measure3"
+ description="third measure"/>
</measures>
<dim_attributes>
- <dim_attribute _type="string" name="dim1" description="basedim">
- </dim_attribute>
+ <dim_attribute _type="string" name="dim1" description="basedim"/>
+ <dim_attribute _type="int" name="cityid" description="basedim"/>
+ <dim_attribute name="stateid" _type="int" description="state id"/>
</dim_attributes>
+ <join_chains>
+ <join_chain dest_table="statedim" name="cubestate" display_string="cube-state" description="state thru cube">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="virtualcube" column="stateid" maps_to_many="false"/>
+ <to table="statedim" column="id" maps_to_many="false"/>
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ </join_chains>
</x_base_cube>
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/facts/testfact7_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact7_base.xml b/lens-cube/src/test/resources/schema/facts/testfact7_base.xml
new file mode 100644
index 0000000..c63b951
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact7_base.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table name="testfact7_base" cube_name="virtualcube" weight="5.0" xmlns="uri:lens:cube:0.1">
+ <columns>
+ <column name="msr3" _type="double" comment="msr3"/>
+ <column name="cityid" _type="int" comment="city id"/>
+ <column name="stateid" _type="int" comment="state id"/>
+ </columns>
+ <properties>
+ <property name="cube.fact.testfact.c5.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ <property name="cube.fact.testfact.cubename" value="testCube"/>
+ <property name="cube.fact.relative.start.time" value="now.year - 90 days"/>
+ <property name="cube.fact.testfact.c4.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ <property name="cube.table.testfact.weight" value="5.0"/>
+ <property name="cube.fact.testfact.c3.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ <property name="cube.fact.testfact.c2.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ <property name="cube.fact.testfact.storages" value="C3,C4,C5,C0,C1,C2"/>
+ <property name="cube.fact.testfact.c0.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ <property name="cube.fact.testfact.c1.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ <update_period>DAILY</update_period>
+ </update_periods>
+ <storage_name>C1</storage_name>
+ <table_desc external="false">
+ <part_cols>
+ <column name="dt" _type="string" comment="date partition"/>
+ </part_cols>
+ <table_parameters>
+ <property name="cube.storagetable.time.partcols" value="dt"/>
+ </table_parameters>
+ <serde_parameters>
+ <property name="serialization.format" value="1"/>
+ </serde_parameters>
+ <time_part_cols>dt</time_part_cols>
+ </table_desc>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/facts/testfact8_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact8_base.xml b/lens-cube/src/test/resources/schema/facts/testfact8_base.xml
new file mode 100644
index 0000000..5450b59
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact8_base.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table name="testfact8_base" cube_name="virtualcube" weight="5.0" xmlns="uri:lens:cube:0.1">
+ <columns>
+
+ <column name="msr2" _type="float" comment="second measure"/>
+ <column name="cityid" _type="int" comment="city id"/>
+ <column name="stateid" _type="int" comment="state id"/>
+ </columns>
+ <properties>
+ <property name="cube.fact.testfact.cubename" value="testCube"/>
+ <property name="cube.fact.relative.start.time" value="now.year - 90 days"/>
+ <property name="cube.table.testfact.weight" value="5.0"/>
+ <property name="cube.fact.testfact.storages" value="C1"/>
+ <property name="cube.fact.testfact.c1.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ <update_period>DAILY</update_period>
+ </update_periods>
+ <storage_name>C1</storage_name>
+ <table_desc external="false">
+ <part_cols>
+ <column name="dt" _type="string" comment="date partition"/>
+ </part_cols>
+ <table_parameters>
+ <property name="cube.storagetable.time.partcols" value="dt"/>
+ </table_parameters>
+ <serde_parameters>
+ <property name="serialization.format" value="1"/>
+ </serde_parameters>
+ <time_part_cols>dt</time_part_cols>
+ </table_desc>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact9_base.xml b/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
new file mode 100644
index 0000000..56e5746
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table name="testfact9_base" cube_name="testcube2" weight="5.0" xmlns="uri:lens:cube:0.1">
+ <columns>
+
+ <column name="msr1" _type="float" comment="second measure"/>
+ </columns>
+ <properties>
+ <property name="cube.fact.testfact.cubename" value="testCube"/>
+ <property name="cube.fact.relative.start.time" value="now.year - 90 days"/>
+ <property name="cube.table.testfact.weight" value="5.0"/>
+ <property name="cube.fact.testfact.storages" value="C1"/>
+ <property name="cube.fact.testfact.c1.updateperiods"
+ value="MONTHLY,HOURLY,YEARLY,CONTINUOUS,QUARTERLY,MINUTELY,DAILY"/>
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ <update_period>DAILY</update_period>
+ <update_period>MONTHLY</update_period>
+ </update_periods>
+ <storage_name>C1</storage_name>
+ <table_desc external="false">
+ <part_cols>
+ <column name="dt" _type="string" comment="date partition"/>
+ </part_cols>
+ <table_parameters>
+ <property name="cube.storagetable.time.partcols" value="dt"/>
+ </table_parameters>
+ <serde_parameters>
+ <property name="serialization.format" value="1"/>
+ </serde_parameters>
+ <time_part_cols>dt</time_part_cols>
+ </table_desc>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
index 860965f..13f6773 100644
--- a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
+++ b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
+<x_virtual_fact_table source_fact_name="testfact9_base" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<properties>
<property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
[05/12] lens git commit: LENS-1445: Expression having reference
column ends up rewriting wrong query
Posted by pr...@apache.org.
LENS-1445: Expression having reference column ends up rewriting wrong query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d49f45a0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d49f45a0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d49f45a0
Branch: refs/heads/current-release-line
Commit: d49f45a0f8c6665784a3770a534d6495c21fd1bc
Parents: a7f407b
Author: Sushil Mohanty <su...@gmail.com>
Authored: Fri Jun 23 16:40:25 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:53 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/UnionQueryWriter.java | 79 +++++++++++++-------
.../parse/TestCubeSegmentationRewriter.java | 3 +-
.../cube/parse/TestUnionAndJoinCandidates.java | 26 +++++++
.../resources/schema/cubes/base/basecube.xml | 11 +++
.../resources/schema/cubes/base/testcube.xml | 1 +
5 files changed, 92 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index cc0a2e5..f6c9ce1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -28,7 +28,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
import java.util.*;
-import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.ql.lib.Node;
@@ -117,7 +118,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
* @return ASTNode
* @throws LensException
*/
- private ASTNode processHavingAST(ASTNode innerAst, AliasDecider aliasDecider, StorageCandidate sc)
+ private ASTNode processHavingAST(ASTNode innerAst, AliasDecider aliasDecider, StorageCandidateHQLContext sc)
throws LensException {
if (cubeql.getHavingAST() != null) {
ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
@@ -251,25 +252,15 @@ public class UnionQueryWriter extends SimpleHQLContext {
for (StorageCandidateHQLContext sc : storageCandidates) {
node = (ASTNode) sc.getQueryAst().getSelectAST().getChild(position).getChild(0);
if (HQLParser.isAggregateAST(node) || HQLParser.hasAggregate(node)) {
- return MetastoreUtil.copyAST(node);
+ if (!node.getChild(1).toString().equals(DEFAULT_MEASURE)) {
+ return MetastoreUtil.copyAST(node);
+ }
}
}
return MetastoreUtil.copyAST(node);
}
/**
- * Check if ASTNode is answerable by StorageCandidate
- * @param sc
- * @param node
- * @return
- */
- private boolean isNodeNotAnswerableForStorageCandidate(StorageCandidate sc, ASTNode node) {
- Set<String> cols = new LinkedHashSet<>();
- getAllColumnsOfNode(node, cols);
- return !sc.getColumns().containsAll(cols);
- }
-
- /**
* Set the default value "0.0" in the non answerable aggreagte expressions.
* @param node
* @param sc
@@ -467,7 +458,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
// Iterate over the StorageCandidates and add non projected having columns in inner select ASTs
for (StorageCandidateHQLContext sc : storageCandidates) {
aliasDecider.setCounter(selectAliasCounter);
- processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc.getStorageCandidate());
+ processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc);
}
removeRedundantProjectedPhrases();
}
@@ -493,7 +484,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
ASTNode child = (ASTNode) selectAST.getChild(i);
ASTNode outerSelect = new ASTNode(child);
ASTNode selectExprAST = (ASTNode) child.getChild(0);
- ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, aliasDecider, sc.getStorageCandidate(), true,
+ ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, aliasDecider, sc, true,
cubeql.getBaseCube().getDimAttributeNames());
outerSelect.addChild(outerAST);
// has an alias? add it
@@ -529,13 +520,14 @@ public class UnionQueryWriter extends SimpleHQLContext {
5. If given ast is memorized as mentioned in the above cases, return the mapping.
*/
private ASTNode getOuterAST(ASTNode astNode, ASTNode innerSelectAST,
- AliasDecider aliasDecider, StorageCandidate sc, boolean isSelectAst, Set<String> dimensionSet)
+ AliasDecider aliasDecider, StorageCandidateHQLContext scContext, boolean isSelectAst, Set<String> dimensionSet)
throws LensException {
+ StorageCandidate sc = scContext.getStorageCandidate();
if (astNode == null) {
return null;
}
Set<String> msrCols = new HashSet<>();
- getAllColumnsOfNode(astNode, msrCols);
+ getAllColumnsOfNode(astNode, msrCols, scContext);
msrCols.removeAll(dimensionSet);
if (isAggregateAST(astNode) && sc.getColumns().containsAll(msrCols)) {
return processAggregate(astNode, innerSelectAST, aliasDecider, isSelectAst);
@@ -544,7 +536,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
ASTNode exprCopy = MetastoreUtil.copyAST(astNode);
setDefaultValueInExprForAggregateNodes(exprCopy, sc);
outerAST.addChild(getOuterAST(getSelectExpr(exprCopy, null, true),
- innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet));
+ innerSelectAST, aliasDecider, scContext, isSelectAst, dimensionSet));
return outerAST;
} else {
if (hasAggregate(astNode)) {
@@ -552,10 +544,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
for (Node child : astNode.getChildren()) {
ASTNode childAST = (ASTNode) child;
if (hasAggregate(childAST) && sc.getColumns().containsAll(msrCols)) {
- outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet));
+ outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider,
+ scContext, isSelectAst, dimensionSet));
} else if (hasAggregate(childAST) && !sc.getColumns().containsAll(msrCols)) {
childAST.replaceChildren(1, 1, getSelectExpr(null, null, true));
- outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider, sc, isSelectAst, dimensionSet));
+ outerAST.addChild(getOuterAST(childAST, innerSelectAST, aliasDecider,
+ scContext, isSelectAst, dimensionSet));
} else {
outerAST.addChild(childAST);
}
@@ -643,7 +637,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
private void processHavingExpression(ASTNode innerSelectAst, Set<ASTNode> havingAggASTs,
- AliasDecider aliasDecider, StorageCandidate sc) throws LensException {
+ AliasDecider aliasDecider, StorageCandidateHQLContext sc) throws LensException {
// iterate over all children of the ast and get outer ast corresponding to it.
for (ASTNode child : havingAggASTs) {
if (!innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(child))) {
@@ -677,18 +671,51 @@ public class UnionQueryWriter extends SimpleHQLContext {
* @param msrs
* @return
*/
- private Set<String> getAllColumnsOfNode(ASTNode node, Set<String> msrs) {
+ private Set<String> getAllColumnsOfNode(ASTNode node, Set<String> msrs, StorageCandidateHQLContext sc) {
if (node.getToken().getType() == HiveParser.DOT) {
- msrs.add(node.getChild(1).toString());
+ String col = node.getChild(1).toString();
+ msrs.addAll(getSourceColumnOfRefColumn(col, sc));
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
- getAllColumnsOfNode(child, msrs);
+ getAllColumnsOfNode(child, msrs, sc);
}
return msrs;
}
/**
+ * Returns the source column of the ref column
+ *
+ * @param refCol
+ * @return
+ */
+ private Set<String> getSourceColumnOfRefColumn(String refCol, StorageCandidateHQLContext sc) {
+ Set<String> sourceColumns = new HashSet<String>();
+ for (Map.Entry<String, Set<String>> entry : sc.getCubeQueryContext().getTblAliasToColumns().entrySet()) {
+ if (entry.getValue().contains(refCol)) {
+ String table = entry.getKey();
+
+ if (sc.getCubeQueryContext().getAutoJoinCtx() != null) {
+ for (Map.Entry<Aliased<Dimension>, List<JoinPath>> dimPaths
+ : sc.getCubeQueryContext().getAutoJoinCtx().getAllPaths().entrySet()) {
+
+ if (dimPaths.getKey().alias.equals(table)) {
+ List<JoinPath> joinPaths = dimPaths.getValue();
+ for (JoinPath path : joinPaths) {
+ sourceColumns.addAll(path.getColumnsForTable(sc.getCubeQueryContext().getBaseCube()));
+ }
+ }
+ }
+ }
+ }
+ }
+ if (sourceColumns.isEmpty()) {
+ sourceColumns.add(refCol);
+ }
+ return sourceColumns;
+ }
+
+ /**
* Gets from string of the ouer query, this is a union query of all
* StorageCandidates participated.
*
http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
index 9fa31dc..7e1714b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
@@ -234,8 +234,7 @@ public class TestCubeSegmentationRewriter extends TestQueryRewrite {
userQuery = "select cityid, segmsr1 from testcube where cityname='blah' and "
+ TWO_DAYS_RANGE + " having citysegmsr1 > 20";
String rewrittenQuery = rewrite(userQuery, getConf());
- assertTrue(rewrittenQuery.toLowerCase().endsWith("sum(case when ((cubecity.name) = 'foo') "
- + "then (testcube.segmsr1) end) > 20)"));
+ assertTrue(rewrittenQuery.toLowerCase().endsWith("(sum((testcube.alias2)) > 20)"));
// Order by on alias
userQuery = "select cityid as `city_id_alias`, segmsr1 from testcube where cityname='blah' and "
http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
index dc06ead..ccc3bf4 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
@@ -92,6 +92,32 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite {
}
@Test
+ public void testExpressionHavingRefcol() throws ParseException, LensException {
+ String colsSelected = " union_join_ctx_cityid, union_join_ctx_cityname_msr1_expr, "
+ + "union_join_ctx_cityname_msr2_expr ";
+ String whereCond = "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")";
+ String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf);
+ assertTrue(rewrittenQuery.contains("UNION ALL"));
+ String expectedInnerSelect1 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, sum(case "
+ + "when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr1) else 0 end) "
+ + "as `alias1`, 0.0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact1 basecube ";
+ String expectedInnerSelect2 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, "
+ + "sum(case when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr1) else 0 end) "
+ + "as `alias1`, 0.0 as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact2 basecube";
+ String expectedInnerSelect3 = "SELECT (basecube.union_join_ctx_cityid) as `alias0`, 0.0 as `alias1`, "
+ + "sum(case when ((cubecityjoinunionctx.name) = 'blr') then (basecube.union_join_ctx_msr2) else 0 end) "
+ + "as `alias2` FROM TestQueryRewrite.c1_union_join_ctx_fact3 basecube";
+ String outerSelect = "SELECT (basecube.alias0) as `union_join_ctx_cityid`, sum((basecube.alias1)) "
+ + "as `union_join_ctx_cityname_msr1_expr`, sum((basecube.alias2)) as `union_join_ctx_cityname_msr2_expr` FROM";
+ String outerGroupBy = "GROUP BY (basecube.alias0)";
+ compareContains(expectedInnerSelect1, rewrittenQuery);
+ compareContains(expectedInnerSelect2, rewrittenQuery);
+ compareContains(expectedInnerSelect3, rewrittenQuery);
+ compareContains(outerSelect, rewrittenQuery);
+ compareContains(outerGroupBy, rewrittenQuery);
+ }
+
+ @Test
public void testCustomExpressionForJoinCandidate() throws ParseException, LensException {
// Expr : (case when union_join_ctx_msr2_expr = 0 then 0 else
// union_join_ctx_msr4_expr * 100 / union_join_ctx_msr2_expr end) is completely answered by
http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
index bcea938..6bb5eb9 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
@@ -360,6 +360,17 @@
description="union_join_ctx_Not null cityid">
<expr_spec expr="case when union_join_ctx_cityid is null then 0 else union_join_ctx_cityid end"/>
</expression>
+
+ <expression _type="int" name="union_join_ctx_cityname_msr1_expr" display_string="union_join_ctx_cityname_msr1_expr"
+ description="union_join_ctx_cityname_msr1_expr">
+ <expr_spec expr="sum(case when union_join_ctx_cityname = 'blr' then union_join_ctx_msr1 else 0 end)"/>
+ </expression>
+
+ <expression _type="int" name="union_join_ctx_cityname_msr2_expr" display_string="union_join_ctx_cityname_msr2_expr"
+ description="union_join_ctx_cityname_msr2_expr">
+ <expr_spec expr="sum(case when union_join_ctx_cityname = 'blr' then union_join_ctx_msr2 else 0 end)"/>
+ </expression>
+
<expression _type="String" name="cityandstatenew" display_string="City and State"
description="city and state together">
<expr_spec expr="concat(cityname, ":", statename_cube)" end_time="$gregorian{now.month-2months}"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
index 088c8de..9d2bb02 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
@@ -206,6 +206,7 @@
<expression _type="int" name="notnullcityid" display_string="Not null cityid Expr" description="Not null cityid">
<expr_spec expr="case when cityid is null then 0 else cityid end"/>
</expression>
+
<expression _type="double" name="roundedmsr1" display_string="Rounded msr1" description="rounded measure1">
<expr_spec expr="round(msr1/1000)"/>
</expression>
[07/12] lens git commit: LENS-1448: Having clause expressions are not
resolved correctly for JoinCandidates
Posted by pr...@apache.org.
LENS-1448: Having clause expressions are not resolved correctly for JoinCandidates
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/54ab131a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/54ab131a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/54ab131a
Branch: refs/heads/current-release-line
Commit: 54ab131a2119a029dd8a470653d803fa0a7c35b6
Parents: fbad350
Author: Sushil Mohanty <su...@gmail.com>
Authored: Wed Jun 28 13:10:55 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:55 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/ExpressionResolver.java | 11 +++++------
.../lens/cube/parse/UnionQueryWriter.java | 19 +++++++++++--------
.../lens/cube/parse/TestBaseCubeQueries.java | 20 ++++++++++++++++++++
3 files changed, 36 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 2403576..66b043e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -424,20 +424,19 @@ class ExpressionResolver implements ContextRewriter {
}
replaceAST(cubeql, queryAST.getJoinAST());
replaceAST(cubeql, queryAST.getGroupByAST());
- // Having AST is resolved by each fact, so that all facts can expand their expressions.
- // Having ast is not copied now, it's maintained in cubeQueryContext, each fact processes that serially.
+ // Resolve having expression for StorageCandidate
if (queryAST.getHavingAST() != null) {
replaceAST(cubeql, queryAST.getHavingAST());
- } else if (cubeql.getHavingAST() != null && nonPickedExpressionsForCandidate.isEmpty()) {
- replaceAST(cubeql, cubeql.getHavingAST());
- queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
+ } else if (cubeql.getHavingAST() != null) {
+ ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+ replaceAST(cubeql, havingCopy);
+ queryAST.setHavingAST(havingCopy);
}
replaceAST(cubeql, queryAST.getOrderByAST());
}
private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws LensException {
if (node == null) {
-
return;
}
// Traverse the tree and resolve expression columns
http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f6c9ce1..9dc7ee6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -96,7 +96,6 @@ public class UnionQueryWriter extends SimpleHQLContext {
if (sc.getQueryAst().getHavingAST() != null) {
cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
}
- sc.getQueryAst().setHavingAST(null);
sc.getQueryAst().setOrderByAST(null);
sc.getQueryAst().setLimitValue(null);
}
@@ -120,8 +119,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
private ASTNode processHavingAST(ASTNode innerAst, AliasDecider aliasDecider, StorageCandidateHQLContext sc)
throws LensException {
- if (cubeql.getHavingAST() != null) {
- ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+ if (sc.getQueryAst().getHavingAST() == null
+ && cubeql.getHavingAST() != null) {
+ sc.getQueryAst().setHavingAST(cubeql.getHavingAST());
+ }
+ if (sc.getQueryAst().getHavingAST() != null) {
+ ASTNode havingCopy = MetastoreUtil.copyAST(sc.getQueryAst().getHavingAST());
Set<ASTNode> havingAggChildrenASTs = new LinkedHashSet<>();
getAggregateChildrenInNode(havingCopy, havingAggChildrenASTs);
processHavingExpression(innerAst, havingAggChildrenASTs, aliasDecider, sc);
@@ -283,8 +286,9 @@ public class UnionQueryWriter extends SimpleHQLContext {
}
private boolean isNodeDefault(ASTNode node) {
- if (HQLParser.isAggregateAST((ASTNode) node.getChild(0))) {
- if (HQLParser.getString((ASTNode) node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)) {
+ if (HQLParser.isAggregateAST((ASTNode) node.getChild(0)) || HQLParser.isAggregateAST(node)) {
+ if (HQLParser.getString((ASTNode) node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)
+ || HQLParser.getString((ASTNode) node.getChild(1)).equals(DEFAULT_MEASURE)) {
return true;
}
}
@@ -640,9 +644,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
AliasDecider aliasDecider, StorageCandidateHQLContext sc) throws LensException {
// iterate over all children of the ast and get outer ast corresponding to it.
for (ASTNode child : havingAggASTs) {
- if (!innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(child))) {
- getOuterAST(child, innerSelectAst, aliasDecider, sc, false, cubeql.getBaseCube().getDimAttributeNames());
- }
+ getOuterAST(child, innerSelectAst, aliasDecider, sc, false, cubeql.getBaseCube().getDimAttributeNames());
}
}
@@ -726,6 +728,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
List<String> hqlQueries = new ArrayList<>();
for (StorageCandidateHQLContext sc : storageCandidates) {
removeAggreagateFromDefaultColumns(sc.getQueryAst().getSelectAST());
+ sc.getQueryAst().setHavingAST(null);
hqlQueries.add(sc.toHQL());
}
return hqlQueries.stream().collect(joining(" UNION ALL ", "(", ") as " + cubeql.getBaseCube()));
http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index f87158c..cf29dff 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -882,6 +882,26 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) "
+ "and (round((sum((basecube.alias3)) / 1000)) > 0))"));
+ // Having clause with expression answerable from any one fact and not projected
+ hqlQuery = rewrite("select dim1, dim11, msr12 from basecube where " + TWO_DAYS_RANGE
+ + "having msr12 > 2 and roundedmsr2 > 0", conf);
+ expected1 = getExpectedQuery(cubeName,
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) "
+ + "as `alias2`, 0.0 as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
+ expected2 = getExpectedQuery(cubeName,
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, 0.0 as `alias2`, "
+ + "sum((basecube.msr2)) as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
+
+ compareContains(expected1, hqlQuery);
+ compareContains(expected2, hqlQuery);
+ assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) as `dim11`, "
+ + "sum((basecube.alias2)) as `msr12` from"),
+ hqlQuery);
+ assertTrue(hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and "
+ + "(round((sum((basecube.alias4)) / 1000)) > 0))"));
+
hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE
+ "having msr12+roundedmsr2 <= 1000 and msr12 > 2 and roundedmsr2 > 0", conf);
expected1 = getExpectedQuery(cubeName,
[09/12] lens git commit: LENS-1450: filters are getting replicated in
jdbc driver query
Posted by pr...@apache.org.
LENS-1450: filters are getting replicated in jdbc driver query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/503a46dc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/503a46dc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/503a46dc
Branch: refs/heads/current-release-line
Commit: 503a46dc6c89598cc2ea557f2311e4bd26c3d9a1
Parents: dd3b1bd
Author: Rajitha R <ra...@gmail.com>
Authored: Wed Jul 5 16:49:07 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:58 2017 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/503a46dc/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b5330a7..fb49b6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST,
}
}
} else if (node.getParent() == null
- && node.getToken().getType() != HiveParser.KW_AND) {
+ && node.getToken().getType() != HiveParser.KW_AND && node.getChildCount() == 0) {
// if node is the only child
allFilters.add(HQLParser.getString((ASTNode) node));
}
[08/12] lens git commit: LENS-1446: Cube latestdate api broken after
virtual fact
Posted by pr...@apache.org.
LENS-1446: Cube latestdate api broken after virtual fact
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dd3b1bd1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dd3b1bd1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dd3b1bd1
Branch: refs/heads/current-release-line
Commit: dd3b1bd19d82fb64441d9982a5aa02b7f849e3a7
Parents: 54ab131
Author: Rajitha R <ra...@gmail.com>
Authored: Thu Jun 29 17:23:26 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:56 2017 +0530
----------------------------------------------------------------------
.../apache/lens/cli/commands/LensSchemaCommands.java | 2 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 15 ++++++++-------
.../org/apache/lens/cube/parse/StorageCandidate.java | 8 ++++----
3 files changed, 13 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/dd3b1bd1/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index aca1cf9..cc184a8 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -113,7 +113,7 @@ public class LensSchemaCommands implements CommandMarker {
CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s");
UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s");
CREATE_COMMAND_MAP.put(XVirtualFactTable.class, "create fact --path %s");
- UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --name %s --path %s");
+ UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --fact_name %s --path %s");
CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s");
UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s");
}
http://git-wip-us.apache.org/repos/asf/lens/blob/dd3b1bd1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 5c05e53..3952696 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -161,7 +161,8 @@ public class CubeMetastoreClient {
for (FactTable fact : getAllFacts(cube)) {
for (String storage : fact.getStorages()) {
for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) {
- PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol);
+ PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod,
+ partCol);
if (timeline != null) {// this storage table is partitioned by partCol or not.
Date latest = timeline.getLatestDate();
if (latest != null && latest.after(max)) {
@@ -1078,8 +1079,8 @@ public class CubeMetastoreClient {
List<Partition> partsAdded = new ArrayList<>();
// first update in memory, then add to hive table's partitions. delete is reverse.
partitionTimelineCache.updateForAddition(factOrDimTable, storageName, updatePeriod,
- getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName, factOrDimTable),
- getStorageTableEndDate(storageTableName, factOrDimTable)));
+ getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName,
+ getFactTable(factOrDimTable)), getStorageTableEndDate(storageTableName, getFactTable(factOrDimTable))));
// Adding partition in fact table.
if (storagePartitionDescs.size() > 0) {
partsAdded = getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod,
@@ -1094,17 +1095,17 @@ public class CubeMetastoreClient {
}
}
- public Date getStorageTableStartDate(String storageTable, String factTableName)
+ public Date getStorageTableStartDate(String storageTable, FactTable factTableName)
throws LensException {
List<Date> startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey());
- startDates.add(getFactTable(factTableName).getStartTime());
+ startDates.add(factTableName.getStartTime());
return Collections.max(startDates);
}
- public Date getStorageTableEndDate(String storageTable, String factTableName)
+ public Date getStorageTableEndDate(String storageTable, FactTable factTableName)
throws LensException {
List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey());
- endDates.add((getFactTable(factTableName)).getEndTime());
+ endDates.add(factTableName.getEndTime());
return Collections.min(endDates);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/dd3b1bd1/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index c6ef6d2..2b1add3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -252,8 +252,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
List<Date> startDates = new ArrayList<>();
List<Date> endDates = new ArrayList<>();
for (String storageTablePrefix : getValidStorageTableNames()) {
- startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, fact.getSourceFactName()));
- endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, fact.getSourceFactName()));
+ startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, fact));
+ endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, fact));
}
this.startTime = Collections.min(startDates);
this.endTime = Collections.max(endDates);
@@ -891,7 +891,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
return this.startTime;
}
return getCubeMetastoreClient().getStorageTableStartDate(
- getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact.getName());
+ getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact);
}
private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException {
@@ -900,7 +900,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
return this.endTime;
}
return getCubeMetastoreClient().getStorageTableEndDate(
- getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact.getName());
+ getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact);
}
[02/12] lens git commit: LENS-1442: Optimize algorithm of
CandidateCoveringSetResolver
Posted by pr...@apache.org.
LENS-1442: Optimize algorithm of CandidateCoveringSetResolver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9da5b40e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9da5b40e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9da5b40e
Branch: refs/heads/current-release-line
Commit: 9da5b40e3afbe966256a601bae6204afd5d3e992
Parents: c2a9c93
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Jun 20 16:09:32 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:49 2017 +0530
----------------------------------------------------------------------
.../cube/parse/CandidateCoveringSetsResolver.java | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/9da5b40e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1e9873f..61c28c6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -208,21 +208,24 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
List<Candidate> ucSet = new ArrayList<>(candidates);
// Check if a single set can answer all the measures and exprsWithMeasures
for (Iterator<Candidate> i = ucSet.iterator(); i.hasNext();) {
- boolean evaluable = false;
+ boolean allEvaluable = true;
+ boolean anyEvaluable = false;
Candidate uc = i.next();
for (QueriedPhraseContext msr : msrs) {
- evaluable = uc.isPhraseAnswerable(msr);
- if (!evaluable) {
- break;
- }
+ boolean evaluable = uc.isPhraseAnswerable(msr);
+ allEvaluable &= evaluable;
+ anyEvaluable |= evaluable;
}
- if (evaluable) {
+ if (allEvaluable) {
// single set can answer all the measures as an UnionCandidate
List<Candidate> one = new ArrayList<>();
one.add(uc);
msrCoveringSets.add(one);
i.remove();
}
+ if (!anyEvaluable) { // none evaluable
+ i.remove();
+ }
}
// Sets that contain all measures or no measures are removed from iteration.
// find other facts
[03/12] lens git commit: LENS-1443: Fallback ranges not working for
virtual facts
Posted by pr...@apache.org.
LENS-1443: Fallback ranges not working for virtual facts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c174583f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c174583f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c174583f
Branch: refs/heads/current-release-line
Commit: c174583ff7946d0347e0a0a87272f42c0023aecf
Parents: 9da5b40
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Jun 21 13:20:09 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:50 2017 +0530
----------------------------------------------------------------------
.../apache/lens/cube/metadata/FactTable.java | 38 ++++++++++----------
.../apache/lens/cube/parse/JoinCandidate.java | 7 ++--
.../lens/cube/parse/LeastPartitionResolver.java | 21 +++++------
.../lens/cube/parse/StorageCandidate.java | 2 +-
.../lens/cube/parse/StorageTableResolver.java | 5 +--
5 files changed, 37 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index f87cf44..a463c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -35,132 +35,132 @@ public interface FactTable extends Named {
*
* @return Map of storage to set of update periods
*/
- public Map<String, Set<UpdatePeriod>> getUpdatePeriods();
+ Map<String, Set<UpdatePeriod>> getUpdatePeriods();
/**
* Cube to which this fact belongs to
*
* @return the cube string
*/
- public String getCubeName();
+ String getCubeName();
/**
* The set of Storage names
*
* @return set of strings
*/
- public Set<String> getStorages();
+ Set<String> getStorages();
/**
*The type of the fact
*
* @return table type {@link CubeTableType}
*/
- public CubeTableType getTableType();
+ CubeTableType getTableType();
/**
* Config properties
*
* @return map of string, string
*/
- public Map<String, String> getProperties();
+ Map<String, String> getProperties();
/**
* Valid columns of the fact
*
* @return list of column names
*/
- public Set<String> getValidColumns();
+ Set<String> getValidColumns();
/**
* Weight of the fact
*
* @return weight of the fact in double
*/
- public double weight();
+ double weight();
/**
* Set of all the columns names of the fact
*
* @return set of column names
*/
- public Set<String> getAllFieldNames();
+ Set<String> getAllFieldNames();
/**
*tag for checking data completeness
*
* @return Tag String
*/
- public String getDataCompletenessTag();
+ String getDataCompletenessTag();
/**
* List of columns of the fact
*
* @return set of {@link FieldSchema}
*/
- public List<FieldSchema> getColumns();
+ List<FieldSchema> getColumns();
/**
* Is Aggregated Fact
*
* @return true if fact is Aggregated , false otherwise
*/
- public boolean isAggregated();
+ boolean isAggregated();
/**
* Absolute start time of the fact
*
* @return Absolute Start time of the fact {@link Date}
*/
- public Date getAbsoluteStartTime();
+ Date getAbsoluteStartTime();
/**
* Relative start time of the fact
*
* @return Relative Start time of the fact {@link Date}
*/
- public Date getRelativeStartTime();
+ Date getRelativeStartTime();
/**
* Start time of the fact
*
* @return Start time of the fact {@link Date}
*/
- public Date getStartTime();
+ Date getStartTime();
/**
* Absolute end time of the fact
*
* @return Absolute End time of the fact {@link Date}
*/
- public Date getAbsoluteEndTime();
+ Date getAbsoluteEndTime();
/**
* Relative End time of the Fact
*
* @return Relative end time of the fact {@link Date}
*/
- public Date getRelativeEndTime();
+ Date getRelativeEndTime();
/**
* End time of the fact
*
* @return End time of the fact {@link Date}
*/
- public Date getEndTime();
+ Date getEndTime();
/**
* Is Virtual Fact
*
* @return true if fact is a virtual fact, false otherwise
*/
- public boolean isVirtualFact();
+ boolean isVirtualFact();
/**
* Storage name of the fact
*
* @return Storage name of the fact
*/
- public String getSourceFactName();
+ String getSourceFactName();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
index d9915f4..1c2b9f8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
@@ -91,12 +91,11 @@ public class JoinCandidate implements Candidate {
@Override
public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData)
throws LensException {
+ boolean complete = true;
for (Candidate child : children) {
- if (!child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData)) {
- return false;
- }
+ complete &= child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData);
}
- return true;
+ return complete;
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index 22b1d03..a76e9b1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -48,17 +48,18 @@ class LeastPartitionResolver implements ContextRewriter {
factPartCount.put(candidate, parts);
}
}
+ if (!factPartCount.isEmpty()) {
+ double minPartitions = Collections.min(factPartCount.values());
- double minPartitions = Collections.min(factPartCount.values());
-
- for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
- Candidate candidate = i.next();
- if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) {
- log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}",
- candidate, factPartCount.get(candidate), minPartitions);
- i.remove();
- cubeql.addCandidatePruningMsg(candidate,
- new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS));
+ for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
+ Candidate candidate = i.next();
+ if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) {
+ log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}",
+ candidate, factPartCount.get(candidate), minPartitions);
+ i.remove();
+ cubeql.addCandidatePruningMsg(candidate,
+ new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 6e5aa4c..c6ef6d2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -604,7 +604,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
break;
}
}
- TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeQueryContext);
+ TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getSourceFactName(), cubeQueryContext);
log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange);
if (fallBackRange == null) {
break;
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index d7da8cb..7f0f2d4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -20,6 +20,7 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing;
+import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange;
import java.util.*;
@@ -324,13 +325,13 @@ class StorageTableResolver implements ContextRewriter {
} else if (!sc.getValidUpdatePeriods().contains(UpdatePeriod.CONTINUOUS)) {
if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) {
pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn());
- TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql);
+ TimeRange fallBackRange = getFallbackRange(range, sc.getFact().getSourceFactName(), cubeql);
while (fallBackRange != null) {
pruningCauseForThisTimeRange = null;
if (!client.partColExists(sc.getFact(), sc.getStorageName(),
fallBackRange.getPartitionColumn())) {
pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn());
- fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql);
+ fallBackRange = getFallbackRange(fallBackRange, sc.getFact().getSourceFactName(), cubeql);
} else {
if (!sc.isPartiallyValidForTimeRange(fallBackRange)) {
pruningCauseForThisTimeRange =
[11/12] lens git commit: LENS-1451: Enforcing valid fact tables
doesn't filter out segmentations
Posted by pr...@apache.org.
LENS-1451: Enforcing valid fact tables doesn't filter out segmentations
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3769ef0e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3769ef0e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3769ef0e
Branch: refs/heads/current-release-line
Commit: 3769ef0e8987caf26c931bf2735e3658e30f3ac4
Parents: 41051ea
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 6 19:35:15 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:43:00 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateTableResolver.java | 26 ++++++++++----------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/3769ef0e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index be3b474..86209bd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -104,6 +104,9 @@ class CandidateTableResolver implements ContextRewriter {
private void populateCandidateTables(CubeQueryContext cubeql) throws LensException {
if (cubeql.getCube() != null) {
+ String str = cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
+ List<String> validFactTables =
+ StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
List<FactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube());
if (factTables.isEmpty()) {
throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(),
@@ -112,11 +115,15 @@ class CandidateTableResolver implements ContextRewriter {
for (FactTable fact : factTables) {
if (fact.getUpdatePeriods().isEmpty()) {
log.info("Not considering fact: {} as it has no update periods", fact.getName());
+ } else if (validFactTables != null && !validFactTables.contains(fact.getName())) {
+ log.info("Not considering fact: {} as it's not valid as per user configuration.", fact.getName());
} else {
for (String s : fact.getStorages()) {
StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, s, cubeql);
if (isStorageSupportedOnDriver(sc.getStorageName())) {
cubeql.getCandidates().add(sc);
+ } else {
+ log.info("Not considering {} since storage is not supported on driver.", sc.getName());
}
}
}
@@ -125,7 +132,12 @@ class CandidateTableResolver implements ContextRewriter {
log.info("Populated storage candidates: {}", cubeql.getCandidates());
List<SegmentationCandidate> segmentationCandidates = Lists.newArrayList();
for (Segmentation segmentation : cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) {
- segmentationCandidates.add(new SegmentationCandidate(cubeql, segmentation));
+ if (validFactTables != null && !validFactTables.contains(segmentation.getName())) {
+ log.info("Not considering segmentation: {} as it's not valid as per user configuration.",
+ segmentation.getName());
+ } else {
+ segmentationCandidates.add(new SegmentationCandidate(cubeql, segmentation));
+ }
}
cubeql.getCandidates().addAll(segmentationCandidates);
}
@@ -261,9 +273,6 @@ class CandidateTableResolver implements ContextRewriter {
private void resolveCandidateFactTables(CubeQueryContext cubeql) throws LensException {
if (cubeql.getCube() != null) {
- String str = cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
- List<String> validFactTables =
- StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
Set<QueriedPhraseContext> queriedMsrs = new HashSet<>();
Set<QueriedPhraseContext> dimExprs = new HashSet<>();
@@ -279,15 +288,6 @@ class CandidateTableResolver implements ContextRewriter {
Candidate cand = i.next();
if (cand instanceof StorageCandidate) {
StorageCandidate sc = (StorageCandidate) cand;
- if (validFactTables != null) {
- if (!validFactTables.contains(sc.getFact().getName().toLowerCase())) {
- log.info("Not considering storage candidate:{} as it is not a valid candidate", sc);
- cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.INVALID));
- i.remove();
- continue;
- }
- }
-
// update expression evaluability for this fact
for (String expr : cubeql.getQueriedExprs()) {
cubeql.getExprCtx().updateEvaluables(expr, sc);