You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2017/04/19 05:36:12 UTC
lens git commit: LENS-1391 : Fix pending review comments on
MaxCoveringFactResolver and LeastPartitionResolver
Repository: lens
Updated Branches:
refs/heads/master f8a387917 -> c4cd6d84e
LENS-1391 : Fix pending review comments on MaxCoveringFactResolver and LeastPartitionResolver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4cd6d84
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4cd6d84
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4cd6d84
Branch: refs/heads/master
Commit: c4cd6d84ec533d84c24717591c890972f5d858cb
Parents: f8a3879
Author: Lavkesh Lahngir <la...@linux.com>
Authored: Wed Apr 19 11:05:57 2017 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Apr 19 11:05:57 2017 +0530
----------------------------------------------------------------------
.../lens/cli/commands/LensSchemaCommands.java | 12 +---------
.../cube/parse/CandidateTablePruneCause.java | 7 ++++++
.../lens/cube/parse/LeastPartitionResolver.java | 15 +++----------
.../cube/parse/MaxCoveringFactResolver.java | 23 ++++++--------------
.../lens/cube/parse/StorageCandidate.java | 5 +++--
.../server/query/QueryAPIErrorResponseTest.java | 2 +-
6 files changed, 22 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index befe4e6..d3f9142 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -20,19 +20,11 @@ package org.apache.lens.cli.commands;
import java.io.File;
import java.io.FilenameFilter;
-import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.lens.api.metastore.SchemaTraverser;
-import org.apache.lens.api.metastore.XBaseCube;
-import org.apache.lens.api.metastore.XDerivedCube;
-import org.apache.lens.api.metastore.XDimension;
-import org.apache.lens.api.metastore.XDimensionTable;
-import org.apache.lens.api.metastore.XFactTable;
-import org.apache.lens.api.metastore.XSegmentation;
-import org.apache.lens.api.metastore.XStorage;
+import org.apache.lens.api.metastore.*;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
import org.springframework.beans.factory.annotation.Autowired;
@@ -42,9 +34,7 @@ import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.shell.support.logging.HandlerUtils;
import org.springframework.stereotype.Component;
-import org.springframework.util.Assert;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@Component
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 1c0d356..29af419 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -235,6 +235,7 @@ public class CandidateTablePruneCause {
// if a time dim is not supported by the fact. Would be set if and only if
// the fact is not partitioned by part col of the time dim and time dim is not a dim attribute
private Set<String> unsupportedTimeDims;
+ private MaxCoveringFactResolver.TimeCovered maxTimeCovered;
// time covered
// ranges in which fact is invalid
private List<TimeRange> invalidRanges;
@@ -295,6 +296,12 @@ public class CandidateTablePruneCause {
return cause;
}
+ public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) {
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA);
+ cause.setMaxTimeCovered(timeCovered);
+ return cause;
+ }
+
public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String> colSet) {
CandidateTablePruneCause cause =
new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH);
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index a9bd164..5adbc23 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -28,7 +28,7 @@ import org.apache.lens.server.api.error.LensException;
import lombok.extern.slf4j.Slf4j;
/**
- * Prune candidate fact sets which require more partitions than minimum parts.
+ * Prune candidates which require more partitions than minimum parts.
*/
@Slf4j
class LeastPartitionResolver implements ContextRewriter {
@@ -41,9 +41,9 @@ class LeastPartitionResolver implements ContextRewriter {
//The number of partitions being calculated is not the actual number of partitions,
// they are number of time values now instead of partitions.
// This seems fine, as the less number of time values actually represent the rollups on time. And with
- // MaxCoveringFactResolver facts with less partitions which are not covering the range would be removed.
+ // MaxCoveringFactResolver candidates with less partitions which are not covering the range would be removed.
for (Candidate candidate : cubeql.getCandidates()) {
- factPartCount.put(candidate, getPartCount(candidate));
+ factPartCount.put(candidate, candidate.getParticipatingPartitions().size());
}
double minPartitions = Collections.min(factPartCount.values());
@@ -60,13 +60,4 @@ class LeastPartitionResolver implements ContextRewriter {
}
}
}
-
- private int getPartCount(Candidate candidate) {
- int parts = 0;
- for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) {
- parts += sc.getNumQueriedParts();
- }
- return parts;
- }
-
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
index 34180d1..86e5634 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
@@ -23,7 +23,6 @@ import java.util.Iterator;
import java.util.Map;
import org.apache.lens.cube.metadata.FactPartition;
-import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline;
import org.apache.lens.server.api.error.LensException;
@@ -47,7 +46,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
public void rewriteContext(CubeQueryContext cubeql) {
if (failOnPartialData) {
// if fail on partial data is true, by the time this resolver starts,
- // all candidate fact sets are covering full time range. We can avoid
+ // all candidates are covering full time range. We can avoid
// redundant computation.
return;
}
@@ -62,7 +61,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
}
private void resolveByTimeCovered(CubeQueryContext cubeql) {
- // For each part column, which candidate fact sets are covering how much amount.
+ // For each part column, which candidates are covering how much amount.
// Later, we'll maximize coverage for each queried part column.
Map<String, Map<Candidate, Long>> partCountsPerPartCol = Maps.newHashMap();
for (Candidate cand : cubeql.getCandidates()) {
@@ -73,7 +72,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
partCountsPerPartCol.get(entry.getKey()).put(cand, entry.getValue());
}
}
- // for each queried partition, prune fact sets that are covering less range than max
+ // for each queried partition, prune candidate that are covering less range than max
for (String partColQueried : cubeql.getPartitionColumnsQueried()) {
if (partCountsPerPartCol.get(partColQueried) != null) {
long maxTimeCovered = Collections.max(partCountsPerPartCol.get(partColQueried).values());
@@ -89,8 +88,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
log.info("Not considering Candidate:{} from Candidate set as it covers less time than the max"
+ " for partition column: {} which is: {}", candidate, partColQueried, timeCovered);
iter.remove();
- cubeql.addCandidatePruningMsg(candidate,
- new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.LESS_DATA));
+ cubeql.addCandidatePruningMsg(candidate, CandidateTablePruneCause.lessData(timeCovered));
}
}
}
@@ -112,7 +110,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
return;
}
- // We prune those candidate fact set, whose dataCompletenessFactor is less than maxDataCompletenessFactor
+ // We prune those candidate whose dataCompletenessFactor is less than maxDataCompletenessFactor
Iterator<Candidate> iter = cubeql.getCandidates().iterator();
while (iter.hasNext()) {
Candidate cand = iter.next();
@@ -121,8 +119,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
log.info("Not considering Candidate :{} from the list as the dataCompletenessFactor for this:{} is "
+ "less than the max:{}", cand, dataCompletenessFactor, maxDataCompletenessFactor);
iter.remove();
- cubeql.addCandidatePruningMsg(cand,
- new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.INCOMPLETE_PARTITION));
+ cubeql.addCandidatePruningMsg(cand, CandidateTablePruneCause.incompletePartitions(null));
}
}
}
@@ -145,19 +142,13 @@ class MaxCoveringFactResolver implements ContextRewriter {
}
/**
- * Returns time covered by fact set for each part column.
+ * Returns time covered by candidate for each part column.
*
* @param cand
* @return
*/
private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) {
Map<String, Long> ret = Maps.newHashMap();
- UpdatePeriod smallest = UpdatePeriod.values()[UpdatePeriod.values().length - 1];
- for (FactPartition part : cand.getParticipatingPartitions()) {
- if (part.getPeriod().compareTo(smallest) < 0) {
- smallest = part.getPeriod();
- }
- }
PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns();
for (FactPartition part : cand.getParticipatingPartitions()) {
if (part.isFound()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index d95cf27..95e3c95 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -152,8 +152,6 @@ public class StorageCandidate implements Candidate, CandidateTable {
*/
@Getter
private Set<String> nonExistingPartitions = new HashSet<>();
- @Getter
- private int numQueriedParts = 0;
/**
* This will be true if this storage candidate has multiple storage tables (one per update period)
@@ -162,6 +160,9 @@ public class StorageCandidate implements Candidate, CandidateTable {
@Getter
private boolean isStorageTblsAtUpdatePeriodLevel;
+ @Getter
+ private int numQueriedParts = 0;
+
public StorageCandidate(StorageCandidate sc) throws LensException {
this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql());
this.validUpdatePeriods.addAll(sc.getValidUpdatePeriods());
http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 064da01..5409d21 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -283,7 +283,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
//Create a StorageTable
XStorageTables tables = new XStorageTables();
- tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY"));
+ tables.getStorageTable().add(createStorageTblElement(testStorage, "DAILY"));
xFactTable.setStorageTables(tables);
createFactFailFast(target, sessionId, xFactTable, mt);