You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/06/23 12:25:26 UTC
[38/51] [abbrv] incubator-lens git commit: LENS-604 : Add "time
dimension not supported" as a fact prune cause
LENS-604 : Add "time dimension not supported" as a fact prune cause
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/51cdd988
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/51cdd988
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/51cdd988
Branch: refs/heads/current-release-line
Commit: 51cdd98831f2f392e24f16718b3cd4409d2250ed
Parents: 2542874
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jun 19 10:42:35 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 19 10:42:35 2015 +0530
----------------------------------------------------------------------
.../cube/parse/CandidateTablePruneCause.java | 41 +++++++++----
.../lens/cube/parse/StorageTableResolver.java | 61 +++++++++++++-------
.../lens/cube/parse/TestCubeRewriter.java | 8 +--
.../server/query/QueryAPIErrorResponseTest.java | 2 +-
4 files changed, 74 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 5a1f8f9..75d5581 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
+
import java.util.*;
import org.codehaus.jackson.annotate.JsonWriteNullProperties;
@@ -98,7 +100,19 @@ public class CandidateTablePruneCause {
// no candidate storges for cube table, storage cause will have why each
// storage is not a candidate
NO_CANDIDATE_STORAGES("No candidate storages for any table"),
- // cube table has more weight
+ // time dimension not supported. Either directly or indirectly.
+ TIMEDIM_NOT_SUPPORTED("Queried data not available for time dimensions: %s") {
+ @Override
+ Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
+ Set<String> dims = Sets.newHashSet();
+ for(CandidateTablePruneCause cause: causes){
+ dims.addAll(cause.getUnsupportedTimeDims());
+ }
+ return new Object[]{
+ dims.toString(),
+ };
+ }
+ },
NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE("No fact update periods for given range"),
NO_COLUMN_PART_OF_A_JOIN_PATH("No column part of a join path. Join columns: [%s]") {
Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
@@ -222,7 +236,9 @@ public class CandidateTablePruneCause {
private List<String> joinColumns;
// the columns that are missing default aggregate. only set in case of MISSING_DEFAULT_AGGREGATE
private List<String> columnsMissingDefaultAggregate;
-
+ // if a time dim is not supported by the fact. Would be set if and only if
+ // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute
+ private Set<String> unsupportedTimeDims;
// time covered
private MaxCoveringFactResolver.TimeCovered maxTimeCovered;
// ranges in which fact is invalid
@@ -234,17 +250,22 @@ public class CandidateTablePruneCause {
// Different static constructors for different causes.
public static CandidateTablePruneCause factNotAvailableInRange(List<TimeRange> ranges) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(FACT_NOT_AVAILABLE_IN_RANGE);
cause.invalidRanges = ranges;
return cause;
}
+ public static CandidateTablePruneCause timeDimNotSupported(Set<String> unsupportedTimeDims) {
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(TIMEDIM_NOT_SUPPORTED);
+ cause.unsupportedTimeDims = unsupportedTimeDims;
+ return cause;
+ }
public static CandidateTablePruneCause columnNotFound(Collection<String>... missingColumns) {
List<String> colList = new ArrayList<String>();
for (Collection<String> missing : missingColumns) {
colList.addAll(missing);
}
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.COLUMN_NOT_FOUND);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(COLUMN_NOT_FOUND);
cause.setMissingColumns(colList);
return cause;
}
@@ -262,27 +283,27 @@ public class CandidateTablePruneCause {
for (String column : exprs) {
colList.add(column);
}
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(EXPRESSION_NOT_EVALUABLE);
cause.setMissingExpressions(colList);
return cause;
}
public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS);
+ new CandidateTablePruneCause(MISSING_PARTITIONS);
cause.setMissingPartitions(nonExistingParts);
return cause;
}
public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.LESS_DATA);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA);
cause.setMaxTimeCovered(timeCovered);
return cause;
}
public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String> colSet) {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.NO_COLUMN_PART_OF_A_JOIN_PATH);
+ new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH);
cause.setJoinColumns(new ArrayList<String>() {
{
addAll(colSet);
@@ -292,7 +313,7 @@ public class CandidateTablePruneCause {
}
public static CandidateTablePruneCause noCandidateStorages(Map<String, SkipStorageCause> storageCauses) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.NO_CANDIDATE_STORAGES);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_CANDIDATE_STORAGES);
cause.setStorageCauses(new HashMap<String, SkipStorageCause>());
for (Map.Entry<String, SkipStorageCause> entry : storageCauses.entrySet()) {
String key = entry.getKey();
@@ -303,7 +324,7 @@ public class CandidateTablePruneCause {
}
public static CandidateTablePruneCause missingDefaultAggregate(String... names) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_DEFAULT_AGGREGATE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_DEFAULT_AGGREGATE);
cause.setColumnsMissingDefaultAggregate(Lists.newArrayList(names));
return cause;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 049826a..ce6f434 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST;
import static org.apache.lens.cube.parse.DateUtil.WSPACE;
import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd;
@@ -44,6 +47,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or
@@ -58,16 +62,15 @@ class StorageTableResolver implements ContextRewriter {
CubeMetastoreClient client;
private final boolean failOnPartialData;
private final List<String> validDimTables;
- private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap =
- new HashMap<CubeFactTable, Map<UpdatePeriod, Set<String>>>();
+ private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap = new HashMap<>();
private String processTimePartCol = null;
private final UpdatePeriod maxInterval;
- private final Map<String, Set<String>> nonExistingPartitions = new HashMap<String, Set<String>>();
+ private final Map<String, Set<String>> nonExistingPartitions = new HashMap<>();
private TimeRangeWriter rangeWriter;
private DateFormat partWhereClauseFormat = null;
private PHASE phase;
- static enum PHASE {
+ enum PHASE {
FACT_TABLES, FACT_PARTITIONS, DIM_TABLE_AND_PARTITIONS;
static PHASE first() {
@@ -218,7 +221,7 @@ class StorageTableResolver implements ContextRewriter {
}
if (storageTables.isEmpty()) {
LOG.info("Not considering dim table:" + dimtable + " as no candidate storage tables eixst");
- cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addDimPruningMsgs(dim, dimtable, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
}
@@ -291,7 +294,7 @@ class StorageTableResolver implements ContextRewriter {
}
if (storageTableMap.isEmpty()) {
LOG.info("Not considering fact table:" + fact + " as it does not" + " have any storage tables");
- cubeql.addFactPruningMsgs(fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addFactPruningMsgs(fact, noCandidateStorages(skipStorageCauses));
i.remove();
}
}
@@ -352,24 +355,33 @@ class StorageTableResolver implements ContextRewriter {
private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws SemanticException {
// Find candidate tables wrt supported storages
Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator();
- Map<TimeRange, String> whereClasueForFallback = new LinkedHashMap<TimeRange, String>();
+ Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>();
while (i.hasNext()) {
CandidateFact cfact = i.next();
- List<FactPartition> answeringParts = new ArrayList<FactPartition>();
- HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>();
+ List<FactPartition> answeringParts = new ArrayList<>();
+ HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<>();
PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns();
boolean noPartsForRange = false;
+ Set<String> unsupportedTimeDims = Sets.newHashSet();
for (TimeRange range : cubeql.getTimeRanges()) {
StringBuilder extraWhereClause = new StringBuilder();
Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts);
// If no partitions were found, then we'll fallback.
- String partcol = range.getPartitionColumn();
+ String partCol = range.getPartitionColumn();
+ boolean partColNotSupported = rangeParts.isEmpty();
+ for(String storage: cfact.fact.getStorages()) {
+ String storageTableName = MetastoreUtil.getFactStorageTableName(cfact.fact.getName(), storage).toLowerCase();
+ partColNotSupported &= skipStorageCauses.containsKey(storageTableName)
+ && skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST)
+ && skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol);
+ }
TimeRange prevRange = range;
String sep = "";
while (rangeParts.isEmpty()) {
// TODO: should we add a condition whether on range's partcol any missing partitions are not there
- String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partcol);
- if (!cfact.getColumns().contains(timeDim)) {
+ String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partCol);
+ if (partColNotSupported && !cfact.getColumns().contains(timeDim)) {
+ unsupportedTimeDims.add(cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn()));
break;
}
TimeRange fallBackRange = getFallbackRange(prevRange, cfact, cubeql);
@@ -381,13 +393,13 @@ class StorageTableResolver implements ContextRewriter {
extraWhereClause.append(sep)
.append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim));
sep = " AND ";
- partcol = fallBackRange.getPartitionColumn();
prevRange = fallBackRange;
+ partCol = prevRange.getPartitionColumn();
if (!rangeParts.isEmpty()) {
break;
}
}
- whereClasueForFallback.put(range, extraWhereClause.toString());
+ whereClauseForFallback.put(range, extraWhereClause.toString());
if (rangeParts.isEmpty()) {
LOG.info("No partitions for fallback range:" + range);
noPartsForRange = true;
@@ -399,7 +411,7 @@ class StorageTableResolver implements ContextRewriter {
for (FactPartition factPart : rangeParts) {
for (String table : factPart.getStorageTables()) {
if (!tablePartMap.containsKey(table)) {
- tablePartMap.put(table, new LinkedHashSet<FactPartition>(Arrays.asList(factPart)));
+ tablePartMap.put(table, new LinkedHashSet<>(Collections.singletonList(factPart)));
} else {
LinkedHashSet<FactPartition> storagePart = tablePartMap.get(table);
storagePart.add(factPart);
@@ -414,6 +426,13 @@ class StorageTableResolver implements ContextRewriter {
cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts);
cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString()));
}
+ if (!unsupportedTimeDims.isEmpty()) {
+ LOG.info("Not considering fact table:" + cfact.fact + " as it doesn't support time dimensions: "
+ + unsupportedTimeDims);
+ cubeql.addFactPruningMsgs(cfact.fact, timeDimNotSupported(unsupportedTimeDims));
+ i.remove();
+ continue;
+ }
Set<String> nonExistingParts = missingParts.toSet();
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
@@ -428,13 +447,13 @@ class StorageTableResolver implements ContextRewriter {
* 3. Storage tables do not have the update period for the timerange queried.
*/
if (failOnPartialData && !nonExistingParts.isEmpty()) {
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts));
+ cubeql.addFactPruningMsgs(cfact.fact, missingPartitions(nonExistingParts));
} else if (!skipStorageCauses.isEmpty()) {
- CandidateTablePruneCause cause = CandidateTablePruneCause.noCandidateStorages(skipStorageCauses);
+ CandidateTablePruneCause cause = noCandidateStorages(skipStorageCauses);
cubeql.addFactPruningMsgs(cfact.fact, cause);
} else {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
+ new CandidateTablePruneCause(NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
cubeql.addFactPruningMsgs(cfact.fact, cause);
}
i.remove();
@@ -445,7 +464,7 @@ class StorageTableResolver implements ContextRewriter {
StorageUtil.getMinimalAnsweringTables(answeringParts, minimalStorageTables);
if (minimalStorageTables.isEmpty()) {
LOG.info("Not considering fact table:" + cfact + " as it does not" + " have any storage tables");
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addFactPruningMsgs(cfact.fact, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
}
@@ -463,10 +482,10 @@ class StorageTableResolver implements ContextRewriter {
LinkedHashSet<FactPartition> rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table);
LinkedHashSet<FactPartition> minimalPartsCopy = new LinkedHashSet<FactPartition>(minimalParts);
minimalPartsCopy.retainAll(rangeParts);
- if (!StringUtils.isEmpty(whereClasueForFallback.get(trange))) {
+ if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) {
rangeToWhere.put(
rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
- minimalPartsCopy) + " and " + whereClasueForFallback.get(trange), table);
+ minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table);
} else {
rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql,
cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 0f1ad6e..e04d04c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -947,12 +947,8 @@ public class TestCubeRewriter extends TestQueryRewrite {
MISSING_PARTITIONS);
assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(),
NO_CANDIDATE_STORAGES);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getCause(), NO_CANDIDATE_STORAGES);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getStorageCauses().values().iterator().next().getCause(), SkipStorageCode.PART_COL_DOES_NOT_EXIST);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getStorageCauses().values().iterator().next().getNonExistantPartCols(), Arrays.asList("dt"));
+ assertEquals(pruneCauses.getDetails().get("summary4").iterator().next().getCause(), TIMEDIM_NOT_SUPPORTED);
+ assertTrue(pruneCauses.getDetails().get("summary4").iterator().next().getUnsupportedTimeDims().contains("d_time"));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 4c2a7a4..720229d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -68,7 +68,7 @@ import org.testng.annotations.Test;
import com.google.common.base.Optional;
import lombok.NonNull;
-@Test
+@Test(groups = "unit-test")
public class QueryAPIErrorResponseTest extends LensJerseyTest {
private static final String MOCK_QUERY = "mock-query";