You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/03/26 14:43:25 UTC
[1/3] incubator-lens git commit: LENS-430: Fixes Mailing List link in
docs (Amareshwari Sriramadasu via prongs)
Repository: incubator-lens
Updated Branches:
refs/heads/master 44f4faf8e -> b86424c8c
LENS-430: Fixes Mailing List link in docs (Amareshwari Sriramadasu via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/7f6634a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/7f6634a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/7f6634a8
Branch: refs/heads/master
Commit: 7f6634a82b7eeba0b7d5ea3478ef225b9f32a7a3
Parents: 44f4faf
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Mar 26 19:08:30 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Thu Mar 26 19:08:30 2015 +0530
----------------------------------------------------------------------
src/site/apt/developer/contribute.apt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7f6634a8/src/site/apt/developer/contribute.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/contribute.apt b/src/site/apt/developer/contribute.apt
index bcce161..821ddab 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -493,7 +493,7 @@ Developer Documentation : How to contribute to Apache Lens?
* Stay involved
- Contributors should join the Lens {{{./mail-lists.html}mailing lists}}. In particular, the commit list (to see
+ Contributors should join the Lens {{{../mail-lists.html}mailing lists}}. In particular, the commit list (to see
changes as they are made), the dev list (to join discussions of changes) and the user list (to help others). Also
refer to {{{http://www.apache.org/dev/contributors.html} Apache contributors guide}} and
{{{http://www.apache.org/foundation/voting.html} Apache voting process}}.
[3/3] incubator-lens git commit: LENS-323: Use cube query context for
setting priority(Amareshwari Sriramadasu via prongs)
Posted by pr...@apache.org.
LENS-323: Use cube query context for setting priority(Amareshwari Sriramadasu via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b86424c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b86424c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b86424c8
Branch: refs/heads/master
Commit: b86424c8c4bc46a31582f5de78d91a7cb88f362d
Parents: 36764bb
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Mar 26 19:12:30 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Thu Mar 26 19:12:30 2015 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CandidateDim.java | 29 +++-
.../apache/lens/cube/parse/CandidateFact.java | 28 +++-
.../apache/lens/cube/parse/CandidateTable.java | 12 ++
.../lens/cube/parse/CandidateTableResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 5 +-
.../apache/lens/cube/parse/DimHQLContext.java | 2 +-
.../apache/lens/cube/parse/JoinResolver.java | 4 +-
.../lens/cube/parse/LeastPartitionResolver.java | 2 +-
.../lens/cube/parse/SingleFactHQLContext.java | 2 +-
.../lens/cube/parse/StorageTableResolver.java | 17 ++-
.../org/apache/lens/cube/parse/StorageUtil.java | 2 +-
.../apache/lens/driver/cube/RewriteUtil.java | 14 ++
.../apache/lens/driver/cube/RewriterPlan.java | 132 +++++++++++++++++
.../apache/lens/cube/parse/CubeTestSetup.java | 6 +
.../lens/cube/parse/TestRewriterPlan.java | 147 +++++++++++++++++++
lens-driver-hive/pom.xml | 5 +
.../org/apache/lens/driver/hive/HiveDriver.java | 7 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 18 +--
.../DurationBasedQueryPriorityDecider.java | 89 ++++++-----
.../apache/lens/driver/hive/TestHiveDriver.java | 119 +++++++++++----
.../src/test/resources/priority_tests.data | 6 +-
.../lens/server/api/driver/DriverQueryPlan.java | 47 ++++--
.../server/api/query/AbstractQueryContext.java | 10 ++
.../api/query/DriverSelectorQueryContext.java | 32 ++++
.../lens/server/api/driver/MockDriver.java | 8 -
.../server/query/QueryExecutionServiceImpl.java | 2 +
26 files changed, 613 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java
index 3ac207e..90d0b6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateDim.java
@@ -19,20 +19,30 @@
package org.apache.lens.cube.parse;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
import org.apache.lens.cube.metadata.CubeDimensionTable;
import org.apache.lens.cube.metadata.Dimension;
+import org.apache.lens.cube.metadata.StorageConstants;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
+import lombok.Getter;
+import lombok.Setter;
+
/**
* Holds context of a candidate dim table.
*/
-class CandidateDim implements CandidateTable {
+public class CandidateDim implements CandidateTable {
final CubeDimensionTable dimtable;
- String storageTable;
- String whereClause;
+ @Getter
+ @Setter
+ private String storageTable;
+ @Getter
+ @Setter
+ private String whereClause;
private boolean dbResolved = false;
private boolean whereClauseAdded = false;
private Dimension baseTable;
@@ -109,4 +119,17 @@ class CandidateDim implements CandidateTable {
public Collection<String> getColumns() {
return dimtable.getAllFieldNames();
}
+
+ @Override
+ public Set<String> getStorageTables() {
+ return Collections.singleton(storageTable);
+ }
+
+ @Override
+ public Set<String> getPartsQueried() {
+ if (StringUtils.isBlank(whereClause)) {
+ return Collections.emptySet();
+ }
+ return Collections.singleton(StorageConstants.LATEST_PARTITION_VALUE);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 66c3b25..47b9fc1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -25,6 +25,7 @@ import java.util.*;
import org.apache.lens.cube.metadata.AbstractCubeTable;
import org.apache.lens.cube.metadata.CubeFactTable;
import org.apache.lens.cube.metadata.CubeInterface;
+import org.apache.lens.cube.metadata.FactPartition;
import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
import org.apache.lens.cube.parse.HQLParser.TreeNode;
@@ -41,24 +42,35 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.antlr.runtime.CommonToken;
+import lombok.Getter;
+import lombok.Setter;
+
/**
* Holds context of a candidate fact table.
*/
-class CandidateFact implements CandidateTable {
+public class CandidateFact implements CandidateTable {
public static final Log LOG = LogFactory.getLog(CandidateFact.class.getName());
final CubeFactTable fact;
- Set<String> storageTables;
+ @Getter
+ @Setter
+ private Set<String> storageTables;
// flag to know if querying multiple storage tables is enabled for this fact
- boolean enabledMultiTableSelect;
- int numQueriedParts = 0;
- final Map<TimeRange, String> rangeToWhereClause = new HashMap<TimeRange, String>();
+ @Getter
+ @Setter
+ private boolean enabledMultiTableSelect;
+ @Getter
+ private int numQueriedParts = 0;
+ @Getter
+ private final Set<FactPartition> partsQueried = new HashSet<FactPartition>();
+ @Getter
+ private final Map<TimeRange, String> rangeToWhereClause = new HashMap<TimeRange, String>();
private boolean dbResolved = false;
private CubeInterface baseTable;
private ASTNode selectAST;
private ASTNode whereAST;
private ASTNode groupbyAST;
private ASTNode havingAST;
- List<TimeRangeNode> timenodes = new ArrayList<TimeRangeNode>();
+ private List<TimeRangeNode> timenodes = new ArrayList<TimeRangeNode>();
private final List<Integer> selectIndices = new ArrayList<Integer>();
private final List<Integer> dimFieldIndices = new ArrayList<Integer>();
private Collection<String> columns;
@@ -95,6 +107,10 @@ class CandidateFact implements CandidateTable {
}
}
+ void incrementPartsQueried(int incr) {
+ numQueriedParts += incr;
+ }
+
private void updateTimeRanges(ASTNode root, ASTNode parent, int childIndex) throws SemanticException {
if (root == null) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
index 5eaf7ec..e001ca4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
@@ -19,6 +19,7 @@
package org.apache.lens.cube.parse;
import java.util.Collection;
+import java.util.Set;
import org.apache.lens.cube.metadata.AbstractCubeTable;
@@ -36,6 +37,12 @@ public interface CandidateTable {
String getStorageString(String alias);
/**
+ * Get storage tables corresponding to this candidate
+ * @return
+ */
+ Set<String> getStorageTables();
+
+ /**
* Get candidate table
*
* @return Candidate fact or dim table
@@ -62,4 +69,9 @@ public interface CandidateTable {
* @return set or list of columns
*/
Collection<String> getColumns();
+
+ /**
+ * Get partitions queried
+ */
+ Set<?> getPartsQueried();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 9ee1667..a1022fd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -96,7 +96,7 @@ class CandidateTableResolver implements ContextRewriter {
}
for (CubeFactTable fact : factTables) {
CandidateFact cfact = new CandidateFact(fact, cubeql.getCube());
- cfact.enabledMultiTableSelect = qlEnabledMultiTableSelect;
+ cfact.setEnabledMultiTableSelect(qlEnabledMultiTableSelect);
cubeql.getCandidateFactTables().add(cfact);
}
LOG.info("Populated candidate facts:" + cubeql.getCandidateFactTables());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index df5a212..e06022c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -743,6 +743,8 @@ public class CubeQueryContext {
}
private HQLContextInterface hqlContext;
+ @Getter private Collection<CandidateFact> pickedFacts;
+ @Getter private Collection<CandidateDim> pickedDimTables;
public String toHQL() throws SemanticException {
Set<CandidateFact> cfacts = pickCandidateFactToQuery();
@@ -809,7 +811,8 @@ public class CubeQueryContext {
}
}
LOG.info("Picked Fact:" + cfacts + " dimsToQuery:" + dimsToQuery);
-
+ pickedDimTables = dimsToQuery.values();
+ pickedFacts = cfacts;
if (cfacts != null) {
if (cfacts.size() > 1) {
// Update ASTs for each fact
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
index d83db18..e200b12 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
@@ -102,7 +102,7 @@ abstract class DimHQLContext extends SimpleHQLContext {
boolean added = (originalWhere != null);
for (Dimension dim : queriedDims) {
CandidateDim cdim = dimsToQuery.get(dim);
- if (!cdim.isWhereClauseAdded() && !StringUtils.isBlank(cdim.whereClause)) {
+ if (!cdim.isWhereClauseAdded() && !StringUtils.isBlank(cdim.getWhereClause())) {
appendWhereClause(whereBuf, StorageUtil.getWhereClause(cdim, query.getAliasForTabName(dim.getName())), added);
added = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index ae7886b..7feb31b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -576,8 +576,8 @@ class JoinResolver implements ContextRewriter {
private String getStorageFilter(Map<Dimension, CandidateDim> dimsToQuery, AbstractCubeTable table, String alias) {
String whereClause = "";
if (dimsToQuery != null && dimsToQuery.get(table) != null) {
- if (StringUtils.isNotBlank(dimsToQuery.get(table).whereClause)) {
- whereClause = dimsToQuery.get(table).whereClause;
+ if (StringUtils.isNotBlank(dimsToQuery.get(table).getWhereClause())) {
+ whereClause = dimsToQuery.get(table).getWhereClause();
if (alias != null) {
whereClause = StorageUtil.getWhereClause(whereClause, alias);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index 0514ecf..d0b3ccb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -62,7 +62,7 @@ class LeastPartitionResolver implements ContextRewriter {
private int getPartCount(Set<CandidateFact> set) {
int parts = 0;
for (CandidateFact f : set) {
- parts += f.numQueriedParts;
+ parts += f.getNumQueriedParts();
}
return parts;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
index 5c39982..beaa551 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
@@ -56,7 +56,7 @@ class SingleFactHQLContext extends DimOnlyHQLContext {
// resolve timerange positions and replace it by corresponding where
// clause
for (TimeRange range : query.getTimeRanges()) {
- String rangeWhere = fact.rangeToWhereClause.get(range);
+ String rangeWhere = fact.getRangeToWhereClause().get(range);
if (!StringUtils.isBlank(rangeWhere)) {
ASTNode rangeAST;
try {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index c29d88f..23fd5a6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -224,8 +224,8 @@ class StorageTableResolver implements ContextRewriter {
continue;
}
// pick the first storage table
- candidate.storageTable = storageTables.iterator().next();
- candidate.whereClause = whereClauses.get(candidate.storageTable);
+ candidate.setStorageTable(storageTables.iterator().next());
+ candidate.setWhereClause(whereClauses.get(candidate.getStorageTable()));
}
}
}
@@ -329,15 +329,16 @@ class StorageTableResolver implements ContextRewriter {
noPartsForRange = true;
continue;
}
- cfact.numQueriedParts += rangeParts.size();
+ cfact.incrementPartsQueried(rangeParts.size());
answeringParts.addAll(rangeParts);
- cfact.rangeToWhereClause.put(range, rangeWriter.getTimeRangeWhereClause(cubeql,
+ cfact.getPartsQueried().addAll(rangeParts);
+ cfact.getRangeToWhereClause().put(range, rangeWriter.getTimeRangeWhereClause(cubeql,
cubeql.getAliasForTabName(cubeql.getCube().getName()), rangeParts));
}
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
}
- if (cfact.numQueriedParts == 0 || (failOnPartialData && (noPartsForRange || !nonExistingParts.isEmpty()))) {
+ if (cfact.getNumQueriedParts() == 0 || (failOnPartialData && (noPartsForRange || !nonExistingParts.isEmpty()))) {
LOG.info("Not considering fact table:" + cfact.fact + " as it could" + " not find partition for given ranges: "
+ cubeql.getTimeRanges());
/*
@@ -375,10 +376,10 @@ class StorageTableResolver implements ContextRewriter {
}
Set<String> storageTables = new LinkedHashSet<String>();
storageTables.addAll(minimalStorageTables.keySet());
- cfact.storageTables = storageTables;
+ cfact.setStorageTables(storageTables);
// multi table select is already false, do not alter it
- if (cfact.enabledMultiTableSelect) {
- cfact.enabledMultiTableSelect = enabledMultiTableSelect;
+ if (cfact.isEnabledMultiTableSelect()) {
+ cfact.setEnabledMultiTableSelect(enabledMultiTableSelect);
}
LOG.info("Resolved partitions for fact " + cfact + ": " + answeringParts + " storageTables:" + storageTables);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
index 354c735..24d9340 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
@@ -167,6 +167,6 @@ public final class StorageUtil {
}
public static String getWhereClause(CandidateDim dim, String alias) {
- return getWhereClause(dim.whereClause, alias);
+ return getWhereClause(dim.getWhereClause(), alias);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
index 8ed4070..04be157 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
@@ -26,6 +26,7 @@ import org.apache.lens.api.LensException;
import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.lens.cube.parse.CubeQueryRewriter;
import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
@@ -229,12 +230,19 @@ public final class RewriteUtil {
}
}
+ public static DriverQueryPlan getRewriterPlan(DriverRewriterRunnable rewriter) {
+ RewriterPlan plan = new RewriterPlan(rewriter.cubeQueryCtx);
+ return plan;
+ }
+
public static class DriverRewriterRunnable implements Runnable {
@Getter
private final LensDriver driver;
private final AbstractQueryContext ctx;
private final List<CubeQueryInfo> cubeQueries;
private final String replacedQuery;
+ /** Cube query context - set after rewriting */
+ private List<CubeQueryContext> cubeQueryCtx;
@Getter
/** Indicate if rewrite operation succeeded */
@@ -256,6 +264,9 @@ public final class RewriteUtil {
this.ctx = ctx;
this.cubeQueries = cubeQueries;
this.replacedQuery = replacedQuery;
+ if (cubeQueries != null) {
+ cubeQueryCtx = new ArrayList<CubeQueryContext>(cubeQueries.size());
+ }
}
@Override
@@ -296,6 +307,7 @@ public final class RewriteUtil {
qIndex + "-" + TOHQL_GAUGE);
// toHQL actually generates the rewritten query
String hqlQuery = cqc.toHQL();
+ cubeQueryCtx.add(cqc);
toHQLGauge.markSuccess();
qIndex++;
@@ -310,6 +322,8 @@ public final class RewriteUtil {
builder.append(replacedQuery.substring(start));
rewrittenQuery = builder.toString();
+ // set rewriter plan
+ ctx.getDriverContext().setDriverRewriterPlan(driver, getRewriterPlan(this));
succeeded = true;
ctx.setDriverQuery(driver, rewrittenQuery);
LOG.info("Final rewritten query for driver:" + driver + " is: " + rewrittenQuery);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
new file mode 100644
index 0000000..b05814e
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.cube;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lens.api.query.QueryCost;
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.parse.CandidateTable;
+import org.apache.lens.cube.parse.CubeQueryContext;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * The Rewriter plan
+ */
+@Slf4j
+public final class RewriterPlan extends DriverQueryPlan {
+
+ public RewriterPlan(Collection<CubeQueryContext> cubeQueries) {
+ extractPlan(cubeQueries);
+ }
+
+ @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
+ void extractPlan(Collection<CubeQueryContext> cubeQueries) {
+ int selectCount = 0;
+ int havingCount = 0;
+ int joinCount = 0;
+ int groupbyCount = 0;
+ int orderbyCount = 0;
+
+ for (CubeQueryContext ctx : cubeQueries) {
+ if (ctx.getOrderByAST() != null) {
+ orderbyCount += ctx.getOrderByAST().getChildCount();
+ }
+ if (ctx.getGroupByAST() != null) {
+ groupbyCount += ctx.getGroupByAST().getChildCount();
+ }
+ if (ctx.getHavingAST() != null) {
+ havingCount += ctx.getHavingAST().getChildCount();
+ }
+ if (ctx.getSelectAST() != null) {
+ selectCount += ctx.getSelectAST().getChildCount();
+ }
+ if (ctx.getPickedDimTables() != null && !ctx.getPickedDimTables().isEmpty()) {
+ joinCount += ctx.getPickedDimTables().size();
+ for (CandidateTable dim : ctx.getPickedDimTables()) {
+ addTablesQueried(dim.getStorageTables());
+ if (partitions.get(dim.getName()) == null || partitions.get(dim.getName()).isEmpty()) {
+ // puts storage table to latest part
+ partitions.put(dim.getName(), dim.getPartsQueried());
+ }
+ }
+ }
+ if (ctx.getPickedFacts() != null && !ctx.getPickedFacts().isEmpty()) {
+ for (CandidateTable fact : ctx.getPickedFacts()) {
+ addTablesQueried(fact.getStorageTables());
+ Set<FactPartition> factParts = (Set<FactPartition>) partitions.get(fact.getName());
+ if (factParts == null) {
+ factParts = new HashSet<FactPartition>();
+ partitions.put(fact.getName(), factParts);
+ }
+ factParts.addAll((Set<FactPartition>) fact.getPartsQueried());
+ }
+ } else {
+ // if no facts are there, reducing join count by one, as target would be one of the dimtables picked
+ if (joinCount > 0) {
+ joinCount--;
+ }
+ }
+ for (String table : getTablesQueried()) {
+ if (!tableWeights.containsKey(table)) {
+ Table tbl;
+ try {
+ tbl = ctx.getMetastoreClient().getTable(table);
+ } catch (HiveException e) {
+ log.error("Error while getting table:" + table, e);
+ continue;
+ }
+ String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST);
+ Double weight = 1d;
+ if (costStr != null) {
+ weight = Double.parseDouble(costStr);
+ }
+ tableWeights.put(table, weight);
+ }
+ }
+ }
+ setHasSubQuery(hasSubQuery || cubeQueries.size() > 1);
+ setNumGbys(groupbyCount);
+ setNumJoins(joinCount);
+ setNumOrderBys(orderbyCount);
+ setNumSels(selectCount);
+ setNumHaving(havingCount);
+ setNumAggreagateExprs(-1);
+ setNumSelDistincts(-1);
+ setNumFilters(-1);
+ }
+
+ @Override
+ public String getPlan() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+
+ @Override
+ public QueryCost getCost() {
+ throw new UnsupportedOperationException("Not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 4888ae1..c9ac651 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -27,6 +27,7 @@ import org.apache.lens.api.LensException;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -45,6 +46,8 @@ import org.apache.hadoop.mapred.TextInputFormat;
import org.testng.Assert;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
import lombok.extern.slf4j.Slf4j;
/*
@@ -1326,6 +1329,9 @@ public class CubeTestSetup {
s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
s1.setPartCols(partCols);
s1.setTimePartCols(timePartCols);
+ Map<String, String> tblPros = Maps.newHashMap();
+ tblPros.put(LensConfConstants.STORAGE_COST, "100");
+ s1.setTblProps(tblPros);
dumpPeriods.put(c1, UpdatePeriod.HOURLY);
StorageTableDesc s2 = new StorageTableDesc();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
new file mode 100644
index 0000000..0738b27
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.parse;
+
+import static org.apache.lens.cube.parse.CubeTestSetup.TWO_DAYS_RANGE;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.lens.driver.cube.RewriterPlan;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestRewriterPlan extends TestQueryRewrite {
+
+ Configuration conf = new Configuration();
+
+ TestRewriterPlan() {
+ conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C2");
+ conf.setBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, false);
+ conf.setBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, true);
+ conf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, true);
+ conf.setBoolean(CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER, false);
+ }
+
+ @Test
+ public void testPlanExtractionForSimpleQuery() throws Exception {
+ // simple query
+ CubeQueryContext ctx = rewriteCtx("cube select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf);
+ ctx.toHQL();
+ RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx));
+ Assert.assertNotNull(plan);
+ Assert.assertEquals(plan.getNumSels(), 1);
+ Assert.assertEquals(plan.getNumGbys(), 0);
+ Assert.assertEquals(plan.getNumJoins(), 0);
+ Assert.assertEquals(plan.getNumHaving(), 0);
+ Assert.assertEquals(plan.getNumFilters(), -1);
+ Assert.assertEquals(plan.getNumOrderBys(), 0);
+ Assert.assertEquals(plan.getNumAggreagateExprs(), -1);
+ Assert.assertFalse(plan.getTablesQueried().isEmpty());
+ Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c2_testfact"));
+ Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c2_testfact"), 1.0);
+ Assert.assertFalse(plan.getPartitions().isEmpty());
+ Assert.assertFalse(plan.getPartitions().get("testfact").isEmpty());
+ Assert.assertTrue(plan.getPartitions().get("testfact").size() > 1);
+ }
+
+ @Test
+ public void testPlanExtractionForComplexQuery() throws Exception {
+ // complex query
+ CubeQueryContext ctx = rewriteCtx("cube select citydim.name, SUM(msr2) from testCube where citydim.name != \"XYZ\""
+ + " and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by citydim.name limit 50", conf);
+ ctx.toHQL();
+ RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx));
+ Assert.assertNotNull(plan);
+ Assert.assertEquals(plan.getNumSels(), 2);
+ Assert.assertEquals(plan.getNumGbys(), 1);
+ Assert.assertEquals(plan.getNumJoins(), 1);
+ Assert.assertEquals(plan.getNumHaving(), 1);
+ Assert.assertEquals(plan.getNumOrderBys(), 1);
+ Assert.assertEquals(plan.getNumFilters(), -1);
+ Assert.assertEquals(plan.getNumAggreagateExprs(), -1);
+ Assert.assertFalse(plan.getTablesQueried().isEmpty());
+ Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c2_testfact"));
+ Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c1_citytable"));
+ Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c2_testfact"), 1.0);
+ Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_citytable"), 100.0);
+ Assert.assertFalse(plan.getPartitions().isEmpty());
+ Assert.assertFalse(plan.getPartitions().get("testfact").isEmpty());
+ Assert.assertTrue(plan.getPartitions().get("testfact").size() > 1);
+ Assert.assertFalse(plan.getPartitions().get("citytable").isEmpty());
+ Assert.assertEquals(plan.getPartitions().get("citytable").size(), 1);
+ }
+
+ @Test
+ public void testPlanExtractionForMultipleQueries() throws Exception {
+ // simple query
+ CubeQueryContext ctx1 = rewriteCtx("cube select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf);
+ ctx1.toHQL();
+ CubeQueryContext ctx2 = rewriteCtx("cube select citydim.name, SUM(msr2) from testCube where citydim.name != \"XYZ\""
+ + " and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by citydim.name limit 50", conf);
+ ctx2.toHQL();
+ RewriterPlan plan = new RewriterPlan(Arrays.asList(ctx1, ctx2));
+ Assert.assertNotNull(plan);
+ Assert.assertEquals(plan.getNumSels(), 3);
+ Assert.assertEquals(plan.getNumGbys(), 1);
+ Assert.assertEquals(plan.getNumJoins(), 1);
+ Assert.assertEquals(plan.getNumHaving(), 1);
+ Assert.assertEquals(plan.getNumOrderBys(), 1);
+ Assert.assertEquals(plan.getNumFilters(), -1);
+ Assert.assertEquals(plan.getNumAggreagateExprs(), -1);
+ Assert.assertFalse(plan.getTablesQueried().isEmpty());
+ Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c2_testfact"));
+ Assert.assertTrue(plan.getTablesQueried().contains("TestQueryRewrite.c1_citytable"));
+ Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c2_testfact"), 1.0);
+ Assert.assertEquals(plan.getTableWeights().get("TestQueryRewrite.c1_citytable"), 100.0);
+ Assert.assertFalse(plan.getPartitions().isEmpty());
+ Assert.assertFalse(plan.getPartitions().get("testfact").isEmpty());
+ Assert.assertTrue(plan.getPartitions().get("testfact").size() > 1);
+ Assert.assertFalse(plan.getPartitions().get("citytable").isEmpty());
+ Assert.assertEquals(plan.getPartitions().get("citytable").size(), 1);
+ }
+
+ @Test
+ public void testUnimplemented() throws SemanticException, ParseException {
+ CubeQueryContext ctx = rewriteCtx("cube select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf);
+ ctx.toHQL();
+ RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx));
+ Assert.assertNotNull(plan);
+ try {
+ plan.getPlan();
+ Assert.fail("getPlan is not implemented");
+ } catch (UnsupportedOperationException e) {
+ Assert.assertEquals(e.getMessage(), "Not implemented");
+ }
+
+ try {
+ plan.getCost();
+ Assert.fail("getCost is not implemented");
+ } catch (UnsupportedOperationException e) {
+ Assert.assertEquals(e.getMessage(), "Not implemented");
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index 3b8e73a..080c8f5 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -82,6 +82,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-cube</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 1dc87b6..218dc53 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -333,6 +333,7 @@ public class HiveDriver implements LensDriver {
connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true);
queryPriorityDecider = new DurationBasedQueryPriorityDecider(
+ this,
this.driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES),
this.driverConf.getFloat(HS2_PARTITION_WEIGHT_MONTHLY, MONTHLY_PARTITION_WEIGHT_DEFAULT),
this.driverConf.getFloat(HS2_PARTITION_WEIGHT_DAILY, DAILY_PARTITION_WEIGHT_DEFAULT),
@@ -490,13 +491,13 @@ public class HiveDriver implements LensDriver {
LOG.info("whetherCalculatePriority: " + whetherCalculatePriority);
if (whetherCalculatePriority) {
try {
- // call explain for the plan to be filled
- explain(ctx);
// Inside try since non-data fetching queries can also be executed by async method.
String priority = queryPriorityDecider.decidePriority(ctx).toString();
ctx.getSelectedDriverConf().set("mapred.job.priority", priority);
LOG.info("set priority to " + priority);
- } catch (LensException e) {
+ } catch (Exception e) {
+ // not failing query launch when setting priority fails
+ // priority will be set to usually NORMAL - the default in underlying system.
LOG.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier()
+ "User query: " + ctx.getUserQuery(), e);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index 64a6bba..fce93a9 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -19,9 +19,9 @@
package org.apache.lens.driver.hive;
import java.util.ArrayList;
-import java.util.LinkedHashMap;
+import java.util.HashSet;
import java.util.List;
-import java.util.Map;
+import java.util.Set;
import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryPrepareHandle;
@@ -42,9 +42,6 @@ public class HiveQueryPlan extends DriverQueryPlan {
/** The explain output. */
private String explainOutput;
- /** The partitions. */
- private Map<String, List<String>> partitions;
-
static final QueryCost HIVE_DRIVER_COST = new QueryCost(1, 1.0);
/**
@@ -105,7 +102,6 @@ public class HiveQueryPlan extends DriverQueryPlan {
setPrepareHandle(prepared);
setExecMode(ExecMode.BATCH);
setScanMode(ScanMode.PARTIAL_SCAN);
- partitions = new LinkedHashMap<String, List<String>>();
this.explainOutput = StringUtils.join(explainOutput, '\n');
extractPlanDetails(explainOutput, metastoreConf);
}
@@ -122,7 +118,6 @@ public class HiveQueryPlan extends DriverQueryPlan {
ParserState prevState = state;
ArrayList<ParserState> states = new ArrayList<ParserState>();
Hive metastore = Hive.get(metastoreConf);
- List<String> partList = null;
for (int i = 0; i < explainOutput.size(); i++) {
String line = explainOutput.get(i);
@@ -203,9 +198,9 @@ public class HiveQueryPlan extends DriverQueryPlan {
}
if (partConditionStr != null) {
- List<String> tablePartitions = partitions.get(table);
+ Set<String> tablePartitions = (Set<String>) partitions.get(table);
if (tablePartitions == null) {
- tablePartitions = new ArrayList<String>();
+ tablePartitions = new HashSet<String>();
partitions.put(table, tablePartitions);
}
tablePartitions.add(partConditionStr);
@@ -270,9 +265,4 @@ public class HiveQueryPlan extends DriverQueryPlan {
*/
return HIVE_DRIVER_COST;
}
-
- @Override
- public Map<String, List<String>> getPartitions() {
- return partitions;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
index 4f471cb..c0156e5 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
@@ -18,13 +18,14 @@
*/
package org.apache.lens.driver.hive.priority;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.lens.api.LensException;
import org.apache.lens.api.Priority;
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.AbstractQueryContext;
@@ -32,27 +33,30 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
- CostToPriorityRangeConf costToPriorityRangeMap;
+ private final CostToPriorityRangeConf costToPriorityRangeMap;
/** Partition Weights for priority calculation based on selected partitions **/
/** weight of monthly partition * */
- private float monthlyPartitionWeight;
+ private final float monthlyPartitionWeight;
/** weight of daily partition * */
- private float dailyPartitionWeight;
+ private final float dailyPartitionWeight;
/** weight of hourly partition * */
- private float hourlyPartitionWeight;
+ private final float hourlyPartitionWeight;
+ private final LensDriver driver;
+
/**
* Constructor. Takes three weights for partitions.
- *
+ * @param driver
* @param ranges
* @param monthlyPartitoinWeight
* @param dailyPartitionWeight
* @param hourlyPartitionWeight
*/
- public DurationBasedQueryPriorityDecider(String ranges,
- float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight) {
+ public DurationBasedQueryPriorityDecider(LensDriver driver,
+ String ranges, float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight) {
+ this.driver = driver;
this.costToPriorityRangeMap = new CostToPriorityRangeConf(ranges);
this.monthlyPartitionWeight = monthlyPartitoinWeight;
this.dailyPartitionWeight = dailyPartitionWeight;
@@ -74,27 +78,6 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
}
/**
- * Extract partitions from AbstractQueryContext. Hive currently gives partitions in the format
- * {a:[dt partition1, dt partition2]...}. This method removes the "dt"
- *
- * @param queryContext
- * @return all the tables along with their selected partitions.
- * @throws LensException
- */
- protected Map<String, List<String>> extractPartitions(AbstractQueryContext queryContext) throws LensException {
- Map<String, List<String>> partitions = new HashMap<String, List<String>>();
- for (Map.Entry<String, List<String>> entry : queryContext.getDriverContext().getSelectedDriverQueryPlan()
- .getPartitions().entrySet()) {
- partitions.put(entry.getKey(), new ArrayList<String>());
- for (String s : entry.getValue()) {
- String[] splits = s.split("\\s+");
- partitions.get(entry.getKey()).add(splits[splits.length - 1]); //last split.
- }
- }
- return partitions;
- }
-
- /**
* Calculates total cost based on weights of selected tables and their selected partitions
*
* @param queryContext
@@ -102,21 +85,35 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
* @throws LensException
*/
- float getDurationCost(AbstractQueryContext queryContext) throws LensException {
- final Map<String, List<String>> partitions = extractPartitions(queryContext);
- LOG.info("partitions picked: " + partitions);
+ @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
+ private float getDurationCost(AbstractQueryContext queryContext) throws LensException {
float cost = 0;
- for (String table : partitions.keySet()) {
- for (String partition : partitions.get(table)) {
- if (!partition.equals("latest")) {
- cost += queryContext.getDriverContext().getSelectedDriverQueryPlan().getTableWeight(table)
- * getNormalizedPartitionCost(partition);
+ if (queryContext.getDriverContext().getDriverRewriterPlan(driver) != null) {
+ // the calculation is done only for cube queries involving fact tables
+ // for all other native table queries and dimension only queries, the cost will be zero and priority will
+ // be the highest one associated with zero cost
+ for (Map.Entry<String, Set<?>> entry : queryContext.getDriverContext().getDriverRewriterPlan(driver)
+ .getPartitions().entrySet()) {
+ if (!entry.getValue().isEmpty() && entry.getValue().iterator().next() instanceof FactPartition) {
+ Set<FactPartition> factParts = (Set<FactPartition>)entry.getValue();
+ for (FactPartition partition : factParts) {
+ cost += getTableWeights(partition.getStorageTables(), queryContext) * getNormalizedPartitionCost(
+ partition.getPeriod());
+ }
}
}
}
return cost;
}
+ private float getTableWeights(Set<String> tables, AbstractQueryContext queryContext) {
+ float weight = 0;
+ for (String tblName : tables) {
+ weight += queryContext.getDriverContext().getDriverRewriterPlan(driver).getTableWeight(tblName);
+ }
+ return weight == 0 ? 1 : weight;
+ }
+
/**
* Normalized cost of a partition. PartitionWeight multiplied by number of days in that partition.
*
@@ -124,16 +121,16 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
* @return normalized cost.
* @throws LensException
*/
- float getNormalizedPartitionCost(String partition) throws LensException {
- switch (partition.length()) {
- case 7: //monthly
+ private float getNormalizedPartitionCost(UpdatePeriod updatePeriod) throws LensException {
+ switch (updatePeriod) {
+ case MONTHLY: //monthly
return 30 * monthlyPartitionWeight;
- case 10: // daily
+ case DAILY: // daily
return 1 * dailyPartitionWeight;
- case 13: // hourly
- return (1 / 24) * hourlyPartitionWeight;
+ case HOURLY: // hourly
+ return (1.0f / 24) * hourlyPartitionWeight;
default:
- throw new LensException("Could not recognize partition: " + partition);
+ throw new LensException("Weight not defined for " + updatePeriod);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 8a5839b..b16c346 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -21,6 +21,7 @@ package org.apache.lens.driver.hive;
import static org.testng.Assert.*;
import java.io.*;
+import java.text.ParseException;
import java.util.*;
import org.apache.lens.api.LensConf;
@@ -28,13 +29,14 @@ import org.apache.lens.api.LensException;
import org.apache.lens.api.Priority;
import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.ExplainQueryContext;
-import org.apache.lens.server.api.query.MockQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.util.LensUtil;
@@ -173,7 +175,7 @@ public class TestHiveDriver {
protected void createTestTable(String tableName) throws Exception {
System.out.println("Hadoop Location: " + System.getProperty("hadoop.bin.path"));
String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)" + " TBLPROPERTIES ('"
- + LensConfConstants.STORAGE_COST + "'='500')";
+ + LensConfConstants.STORAGE_COST + "'='500')";
// Create test table
QueryContext context = createContext(createTable, conf);
LensResultSet resultSet = driver.execute(context);
@@ -727,8 +729,9 @@ public class TestHiveDriver {
assertEquals(plan.getTableWeight(dataBase + ".test_part_table"), 500.0);
System.out.println("Parts:" + plan.getPartitions());
assertFalse(plan.getPartitions().isEmpty());
- assertTrue(plan.getPartitions().get(dataBase + ".test_part_table").get(0).contains("today"));
- assertTrue(plan.getPartitions().get(dataBase + ".test_part_table").get(0).contains("dt"));
+ assertEquals(plan.getPartitions().size(), 1);
+ assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("today"));
+ assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("dt"));
}
/**
@@ -792,44 +795,110 @@ public class TestHiveDriver {
*
* @throws IOException
* @throws LensException
+ * @throws ParseException
*/
@Test
- public void testPriority() throws IOException, LensException {
+ public void testPriority() throws IOException, LensException, ParseException {
Configuration conf = new Configuration();
- final MockDriver mockDriver = new MockDriver();
- mockDriver.configure(conf);
- DurationBasedQueryPriorityDecider alwaysNormalPriorityDecider = new DurationBasedQueryPriorityDecider("",
+ DurationBasedQueryPriorityDecider alwaysNormalPriorityDecider = new DurationBasedQueryPriorityDecider(driver,
+ "",
HiveDriver.MONTHLY_PARTITION_WEIGHT_DEFAULT,
- HiveDriver.DAILY_PARTITION_WEIGHT_DEFAULT,
- HiveDriver.HOURLY_PARTITION_WEIGHT_DEFAULT
- );
+ HiveDriver.DAILY_PARTITION_WEIGHT_DEFAULT, HiveDriver.HOURLY_PARTITION_WEIGHT_DEFAULT);
BufferedReader br = new BufferedReader(new InputStreamReader(
TestHiveDriver.class.getResourceAsStream("/priority_tests.data")));
String line;
while ((line = br.readLine()) != null) {
String[] kv = line.split("\\s*:\\s*");
- final List<String> partitions = Arrays.asList(kv[0].trim().split("\\s*,\\s*"));
+ final Set<FactPartition> partitions = getFactParts(Arrays.asList(kv[0].trim().split("\\s*,\\s*")));
final Priority expected = Priority.valueOf(kv[1]);
- final HashMap<LensDriver, String> driverQuery1 = new HashMap<LensDriver, String>() {
- {
- put(mockDriver, "driverQuery1");
+ AbstractQueryContext ctx = createContext("test priority query", conf);
+ ctx.getDriverContext().setDriverRewriterPlan(driver, new DriverQueryPlan() {
+
+ @Override
+ public String getPlan() {
+ return null;
}
- };
- AbstractQueryContext ctx = new MockQueryContext("driverQuery1", new LensConf(), conf,
- driverQuery1.keySet());
- ctx.getDriverContext().setDriverQueryPlans(driverQuery1, ctx);
- ctx.setSelectedDriver(mockDriver);
-
- ((MockDriver.MockQueryPlan) ctx.getDriverContext().getDriverQueryPlan(mockDriver)).setPartitions(
- new HashMap<String, List<String>>() {
+
+ @Override
+ public QueryCost getCost() {
+ return null;
+ }
+ });
+
+ ctx.getDriverContext().getDriverRewriterPlan(driver).getPartitions().putAll(
+ new HashMap<String, Set<FactPartition>>() {
{
put("table1", partitions);
}
- }
- );
+ });
+ ctx.getDriverContext().getDriverRewriterPlan(driver).getTableWeights().putAll(
+ new HashMap<String, Double>() {
+ {
+ put("table1", 1.0);
+ }
+ });
Assert.assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx));
Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
}
+ // test priority without fact partitions
+ AbstractQueryContext ctx = createContext("test priority query", conf);
+ ctx.getDriverContext().setDriverRewriterPlan(driver, new DriverQueryPlan() {
+
+ @Override
+ public String getPlan() {
+ return null;
+ }
+
+ @Override
+ public QueryCost getCost() {
+ return null;
+ }
+ });
+
+ ctx.getDriverContext().getDriverRewriterPlan(driver).getPartitions().putAll(
+ new HashMap<String, Set<String>>() {
+ {
+ put("table1", new HashSet<String>());
+ }
+ });
+ ctx.getDriverContext().getDriverRewriterPlan(driver).getTableWeights().putAll(
+ new HashMap<String, Double>() {
+ {
+ put("table1", 1.0);
+ }
+ });
+ Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx));
+ Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
+
+ // test priority without rewriter plan
+ ctx = createContext("test priority query", conf);
+ Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx));
+ Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
+
+ }
+
+ private Set<FactPartition> getFactParts(List<String> partStrings) throws ParseException {
+ Set<FactPartition> factParts = new HashSet<FactPartition>();
+ for (String partStr : partStrings) {
+ String[] partEls = partStr.split(" ");
+ UpdatePeriod p = null;
+ String partSpec = partEls[1];
+ switch (partSpec.length()) {
+ case 7: //monthly
+ p = UpdatePeriod.MONTHLY;
+ break;
+ case 10: // daily
+ p = UpdatePeriod.DAILY;
+ break;
+ case 13: // hourly
+ p = UpdatePeriod.HOURLY;
+ break;
+ }
+ FactPartition part = new FactPartition(partEls[0], p.format().parse(partSpec), p, null, p.format(),
+ Collections.singleton("table1"));
+ factParts.add(part);
+ }
+ return factParts;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-driver-hive/src/test/resources/priority_tests.data
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/priority_tests.data b/lens-driver-hive/src/test/resources/priority_tests.data
index 230df95..98b82ef 100644
--- a/lens-driver-hive/src/test/resources/priority_tests.data
+++ b/lens-driver-hive/src/test/resources/priority_tests.data
@@ -1,4 +1,4 @@
dt 2014-01-02-01: VERY_HIGH
-dt 2013-12,2014-01-01, dt 2014-01-02-00, dt 2014-01-02-01: HIGH
-dt 2013-12,2014-01, dt 2014-02, dt 2014-02-01-00: NORMAL
-dt 2013-12,2014-01, dt 2014-02, dt 2014-03, dt 2014-04, dt 2014-05: LOW
\ No newline at end of file
+dt 2013-12,dt 2014-01-01, dt 2014-01-02-00, dt 2014-01-02-01: HIGH
+dt 2013-12,dt 2014-01, dt 2014-02, dt 2014-02-01-00: NORMAL
+dt 2013-12,dt 2014-01, dt 2014-02, dt 2014-03, dt 2014-04, dt 2014-05: LOW
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
index 38ebc55..1d41720 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
@@ -22,8 +22,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
@@ -125,7 +126,7 @@ public abstract class DriverQueryPlan {
/**
* The tables queried.
*/
- protected final List<String> tablesQueried = new ArrayList<String>();
+ protected final Set<String> tablesQueried = new HashSet<String>();
/**
* The has sub query.
@@ -187,6 +188,8 @@ public abstract class DriverQueryPlan {
*/
protected QueryPrepareHandle handle;
+ protected Map<String, Set<?>> partitions = new HashMap<String, Set<?>>();
+
/**
* Get the query plan
*
@@ -314,7 +317,7 @@ public abstract class DriverQueryPlan {
*
* @return the tablesQueried
*/
- public List<String> getTablesQueried() {
+ public Set<String> getTablesQueried() {
return tablesQueried;
}
@@ -323,11 +326,20 @@ public abstract class DriverQueryPlan {
*
* @param table the table
*/
- protected void addTablesQueries(String table) {
+ protected void addTablesQueried(String table) {
this.tablesQueried.add(table);
}
/**
+ * Set the list of table names to be queried.
+ *
+ * @param table the table
+ */
+ protected void addTablesQueried(Set<String> tables) {
+ this.tablesQueried.addAll(tables);
+ }
+
+ /**
* Get the number of filters in query
*
* @return the numFilters
@@ -357,8 +369,8 @@ public abstract class DriverQueryPlan {
/**
* Set if query has subquery.
*/
- protected void setHasSubQuery() {
- this.hasSubQuery = true;
+ protected void setHasSubQuery(boolean hasSubQuery) {
+ this.hasSubQuery = hasSubQuery;
}
/**
@@ -579,17 +591,31 @@ public abstract class DriverQueryPlan {
this.handle = handle;
}
+ /**
+ * Get number of aggregate expressions.
+ *
+ * @return
+ */
public int getNumAggreagateExprs() {
return numAggrExprs;
}
/**
+ * Set num aggregate expressions
+ *
+ * @param numAggrs
+ */
+ protected void setNumAggreagateExprs(int numAggrs) {
+ numAggrExprs = numAggrs;
+ }
+
+ /**
* Get list of partitions queried for each table
*
* @return
*/
- public Map<String, List<String>> getPartitions() {
- return null;
+ public Map<String, Set<?>> getPartitions() {
+ return partitions;
}
/**
@@ -600,8 +626,9 @@ public abstract class DriverQueryPlan {
*/
public QueryPlan toQueryPlan() throws UnsupportedEncodingException {
return new QueryPlan(numJoins, numGbys, numSels, numSelDi, numHaving, numObys, numAggrExprs, numFilters,
- tablesQueried, hasSubQuery, execMode != null ? execMode.name() : null, scanMode != null ? scanMode.name()
- : null, tableWeights, joinWeight, gbyWeight, filterWeight, havingWeight, obyWeight, selectWeight, null,
+ new ArrayList<String>(tablesQueried), hasSubQuery, execMode != null ? execMode.name() : null,
+ scanMode != null ? scanMode.name() : null, tableWeights, joinWeight, gbyWeight, filterWeight, havingWeight,
+ obyWeight, selectWeight, null,
URLEncoder.encode(getPlan(), "UTF-8"), getCost(), false, null);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 5233569..6799e0c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -374,6 +374,7 @@ public abstract class AbstractQueryContext implements Serializable {
* Gets HiveConf corresponding to query conf.
*
* Should be called judiciously, because constructing HiveConf from conf object is costly.
+ * The field is set to null after query completion. Should not be accessed after completion.
* @return
*/
public HiveConf getHiveConf() {
@@ -417,4 +418,13 @@ public abstract class AbstractQueryContext implements Serializable {
public String getLogHandle() {
return this.getUserQuery();
}
+
+ public void clearTransientStateAfterLaunch() {
+ driverContext.clearTransientStateAfterLaunch();
+ }
+
+ public void clearTransientStateAfterCompleted() {
+ driverContext.clearTransientStateAfterCompleted();
+ hiveConf = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index fef834f..2fd592a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -136,6 +136,14 @@ public class DriverSelectorQueryContext {
@Setter
/** final driver query - after driver rewrites the query*/
protected String finalDriverQuery;
+
+ @Getter
+ @Setter
+ /**
+ * Plan generated from rewriter. This does not contain final driver plan. This contains the information
+ * extracted at rewriter only
+ */
+ protected DriverQueryPlan rewriterPlan;
}
/**
@@ -295,6 +303,17 @@ public class DriverSelectorQueryContext {
return queries;
}
+ public void setDriverRewriterPlan(LensDriver driver, DriverQueryPlan rewriterPlan) {
+ if (driverQueryContextMap.get(driver) != null) {
+ driverQueryContextMap.get(driver).setRewriterPlan(rewriterPlan);
+ }
+ }
+
+ public DriverQueryPlan getDriverRewriterPlan(LensDriver driver) {
+ return driverQueryContextMap.get(driver) != null
+ ? driverQueryContextMap.get(driver).getRewriterPlan() : null;
+ }
+
public DriverQueryPlan getDriverQueryPlan(LensDriver driver) {
return driverQueryContextMap.get(driver) != null
? driverQueryContextMap.get(driver).getDriverQueryPlan() : null;
@@ -323,4 +342,17 @@ public class DriverSelectorQueryContext {
public void setDriverQueryPlan(LensDriver driver, DriverQueryPlan qp) {
driverQueryContextMap.get(driver).setDriverQueryPlan(qp);
}
+
+ void clearTransientStateAfterLaunch() {
+ for (DriverQueryContext driverCtx : driverQueryContextMap.values()) {
+ driverCtx.driverQueryPlan = null;
+ driverCtx.rewriterPlan = null;
+ }
+ }
+
+ void clearTransientStateAfterCompleted() {
+ for (DriverQueryContext driverCtx : driverQueryContextMap.values()) {
+ driverCtx.driverSpecificConf = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 67c73b8..406e2ac 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@@ -40,9 +39,6 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.cli.ColumnDescriptor;
-import lombok.Getter;
-import lombok.Setter;
-
/**
* The Class MockDriver.
*/
@@ -117,10 +113,6 @@ public class MockDriver implements LensDriver {
tableWeights.put("table3", 3.0);
}
- @Getter
- @Setter
- private Map<String, List<String>> partitions;
-
@Override
public String getPlan() {
return query;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b86424c8/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 390071c..ea2da14 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -578,6 +578,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
launchedQueries.add(ctx);
ctx.setLaunchTime(System.currentTimeMillis());
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
+ ctx.clearTransientStateAfterLaunch();
}
/**
@@ -610,6 +611,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
finishedQueries.add(new FinishedQuery(ctx));
+ ctx.clearTransientStateAfterLaunch();
}
void setSuccessState(QueryContext ctx) throws LensException {
[2/3] incubator-lens git commit: LENS-402: In Query End Notification
mail,
Send status message along with of errorMessage (Amareshwari Sriramadasu via
prongs)
Posted by pr...@apache.org.
LENS-402: In Query End Notification mail, Send status message along with of errorMessage (Amareshwari Sriramadasu via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/36764bb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/36764bb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/36764bb7
Branch: refs/heads/master
Commit: 36764bb7414f395da0b725c46ab11e954e3dbab0
Parents: 7f6634a
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Mar 26 19:11:15 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Thu Mar 26 19:11:15 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 3 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 3 +-
.../lens/server/query/QueryEndNotifier.java | 10 +-
.../lens/server/query/ResultFormatter.java | 2 +-
.../server/query/TestQueryEndEmailNotifier.java | 293 +++++++++++++++++++
.../lens/server/query/TestQueryService.java | 63 ----
6 files changed, 307 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 9e3c723..1dc87b6 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -544,7 +544,8 @@ public class HiveDriver implements LensDriver {
break;
case ERROR:
context.getDriverStatus().setState(DriverQueryState.FAILED);
- context.getDriverStatus().setStatusMessage(
+ context.getDriverStatus().setStatusMessage("Query execution failed!");
+ context.getDriverStatus().setErrorMessage(
"Query failed with errorCode:" + opStatus.getOperationException().getErrorCode() + " with errorMessage: "
+ opStatus.getOperationException().getMessage());
break;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 35a9067..121b56b 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -922,7 +922,8 @@ public class JDBCDriver implements LensDriver {
context.getDriverStatus().setStatusMessage(context.getQueryHandle() + " cancelled");
} else if (ctx.getQueryResult() != null && ctx.getQueryResult().error != null) {
context.getDriverStatus().setState(DriverQueryState.FAILED);
- context.getDriverStatus().setStatusMessage(ctx.getQueryResult().error.getMessage());
+ context.getDriverStatus().setStatusMessage("Query execution failed!");
+ context.getDriverStatus().setErrorMessage(ctx.getQueryResult().error.getMessage());
} else {
context.getDriverStatus().setState(DriverQueryState.SUCCESSFUL);
context.getDriverStatus().setStatusMessage(context.getQueryHandle() + " successful");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index 70f9dba..1c6f186 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -38,6 +38,7 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -153,15 +154,22 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
msgBuilder.append("/httpresultset");
break;
case FAILED:
- msgBuilder.append(queryContext.getStatus().getErrorMessage());
+ msgBuilder.append(queryContext.getStatus().getStatusMessage());
+ if (!StringUtils.isBlank(queryContext.getStatus().getErrorMessage())) {
+ msgBuilder.append("\n Reason:\n");
+ msgBuilder.append(queryContext.getStatus().getErrorMessage());
+ }
break;
case CANCELED:
+ msgBuilder.append(queryContext.getStatus().getStatusMessage());
+ break;
case CLOSED:
default:
break;
}
return msgBuilder.toString();
}
+
@Data
public static class Email {
private final String from;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index 0b215d8..b40f949 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -134,7 +134,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
metricsService.incrCounter(ResultFormatter.class, "formatting-errors");
LOG.warn("Exception while formatting result for " + queryHandle, e);
try {
- queryService.setFailedStatus(ctx, "Result formatting failed!", e.getLocalizedMessage());
+ queryService.setFailedStatus(ctx, "Result formatting failed!", e.getMessage());
} catch (LensException e1) {
LOG.error("Exception while setting failure for " + queryHandle, e1);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
new file mode 100644
index 0000000..5726602
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query;
+
+import java.util.*;
+
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.lens.api.LensConf;
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.query.*;
+import org.apache.lens.api.query.QueryStatus.Status;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.LensServices;
+import org.apache.lens.server.LensTestUtil;
+import org.apache.lens.server.api.LensConfConstants;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.subethamail.wiser.Wiser;
+import org.subethamail.wiser.WiserMessage;
+import org.testng.Assert;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+/**
+ * The query completion email notifier
+ */
+@Test(groups = "unit-test")
+public class TestQueryEndEmailNotifier extends LensJerseyTest {
+
+ /** The Constant LOG. */
+ public static final Log LOG = LogFactory.getLog(TestQueryEndEmailNotifier.class);
+
+ /** The query service. */
+ QueryExecutionServiceImpl queryService;
+
+ /** The lens session id. */
+ LensSessionHandle lensSessionId;
+
+ /** The wiser. */
+ private Wiser wiser;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#setUp()
+ */
+ @BeforeTest
+ public void setUp() throws Exception {
+ super.setUp();
+ wiser = new Wiser();
+ wiser.setHostname("localhost");
+ wiser.setPort(25000);
+ queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
+ Map<String, String> sessionconf = new HashMap<String, String>();
+ sessionconf.put("test.session.key", "svalue");
+ sessionconf.put(LensConfConstants.QUERY_MAIL_NOTIFY, "true");
+ sessionconf.put(LensConfConstants.QUERY_RESULT_EMAIL_CC, "foo1@localhost,foo2@localhost,foo3@localhost");
+ lensSessionId = queryService.openSession("foo@localhost", "bar", sessionconf); // @localhost should be removed
+ // automatically
+ createTable(TEST_TABLE);
+ loadData(TEST_TABLE, TEST_DATA_FILE);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#tearDown()
+ */
+ @AfterTest
+ public void tearDown() throws Exception {
+ dropTable(TEST_TABLE);
+ queryService.closeSession(lensSessionId);
+ super.tearDown();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#configure()
+ */
+ @Override
+ protected Application configure() {
+ return new QueryApp();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#configureClient(org.glassfish.jersey.client.ClientConfig)
+ */
+ @Override
+ protected void configureClient(ClientConfig config) {
+ config.register(MultiPartFeature.class);
+ }
+
+ /** The test table. */
+ public static final String TEST_TABLE = "EMAIL_NOTIFIER_TEST_TABLE";
+
+ /** The Constant TEST_DATA_FILE. */
+ public static final String TEST_DATA_FILE = "./testdata/testdata2.data";
+
+ /**
+ * Creates the table.
+ *
+ * @param tblName the tbl name
+ * @throws InterruptedException the interrupted exception
+ */
+ private void createTable(String tblName) throws InterruptedException {
+ LensTestUtil.createTable(tblName, target(), lensSessionId);
+ }
+
+ /**
+ * Load data.
+ *
+ * @param tblName the tbl name
+ * @param testDataFile the test data file
+ * @throws InterruptedException the interrupted exception
+ */
+ private void loadData(String tblName, final String testDataFile) throws InterruptedException {
+ LensTestUtil.loadData(tblName, testDataFile, target(), lensSessionId);
+ }
+
+ /**
+ * Drop table.
+ *
+ * @param tblName the tbl name
+ * @throws InterruptedException the interrupted exception
+ */
+ private void dropTable(String tblName) throws InterruptedException {
+ LensTestUtil.dropTable(tblName, target(), lensSessionId);
+ }
+
+ private QueryHandle launchAndWaitForQuery(LensConf conf, String query, Status expectedStatus)
+ throws InterruptedException {
+ final WebTarget target = target().path("queryapi/queries");
+ final FormDataMultiPart mp = new FormDataMultiPart();
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
+ MediaType.APPLICATION_XML_TYPE));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), query));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
+ MediaType.APPLICATION_XML_TYPE));
+ final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
+ QueryHandle.class);
+
+ Assert.assertNotNull(handle);
+ LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
+ .get(LensQuery.class);
+ QueryStatus stat = ctx.getStatus();
+ while (!stat.isFinished()) {
+ ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ stat = ctx.getStatus();
+ Thread.sleep(1000);
+ }
+ Assert.assertEquals(ctx.getStatus().getStatus(), expectedStatus);
+ return handle;
+ }
+ /**
+ * Test launch fail.
+ *
+ * @throws InterruptedException the interrupted exception
+ */
+ @Test
+ public void testEmailNotification() throws InterruptedException {
+ wiser.start();
+ LensConf conf = new LensConf();
+ // launch failure
+ QueryHandle handle = launchAndWaitForQuery(conf, "select ID from non_exist_table", QueryStatus.Status.FAILED);
+ List<WiserMessage> messages = new ArrayList<WiserMessage>();
+ for (int i = 0; i < 20; i++) {
+ messages = wiser.getMessages();
+ if (messages.size() > 0) {
+ break;
+ }
+ Thread.sleep(10000);
+ }
+
+ Assert.assertEquals(messages.size(), 4);
+ Assert.assertTrue(messages.get(0).toString().contains(handle.toString()));
+ Assert.assertEquals(messages.size(), 4);
+ Assert.assertTrue(messages.get(0).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(0).toString().contains("Launching query failed"));
+ Assert.assertTrue(messages.get(0).toString().contains("Reason"));
+
+ // rewriter failure
+ handle = launchAndWaitForQuery(conf, "cube select ID from nonexist", QueryStatus.Status.FAILED);
+ messages = new ArrayList<WiserMessage>();
+ for (int i = 0; i < 20; i++) {
+ messages = wiser.getMessages();
+ if (messages.size() > 4) {
+ break;
+ }
+ Thread.sleep(10000);
+ }
+
+ Assert.assertEquals(messages.size(), 8);
+ Assert.assertTrue(messages.get(4).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(4).toString().contains("Launching query failed"));
+ Assert.assertTrue(messages.get(4).toString().contains("Reason"));
+
+ // formatting failure
+ conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false");
+ conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "NonexistentSerde.class");
+ handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE,
+ QueryStatus.Status.FAILED);
+ messages = new ArrayList<WiserMessage>();
+ for (int i = 0; i < 20; i++) {
+ messages = wiser.getMessages();
+ if (messages.size() > 8) {
+ break;
+ }
+ Thread.sleep(10000);
+ }
+
+ Assert.assertEquals(messages.size(), 12);
+ Assert.assertTrue(messages.get(8).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(8).toString().contains("Result formatting failed!"));
+ Assert.assertTrue(messages.get(8).toString().contains("Reason"));
+
+ // execution failure
+ conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true");
+ conf.addProperty(HiveConf.ConfVars.COMPRESSRESULT.name(), "true");
+ conf.addProperty("mapred.compress.map.output", "true");
+ conf.addProperty("mapred.map.output.compression.codec", "nonexisting");
+ handle = launchAndWaitForQuery(conf, "select count(ID) from " + TEST_TABLE, QueryStatus.Status.FAILED);
+ messages = new ArrayList<WiserMessage>();
+ for (int i = 0; i < 20; i++) {
+ messages = wiser.getMessages();
+ if (messages.size() > 12) {
+ break;
+ }
+ Thread.sleep(10000);
+ }
+
+ Assert.assertEquals(messages.size(), 16);
+ Assert.assertTrue(messages.get(12).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(12).toString().contains("Query execution failed!"));
+ Assert.assertTrue(messages.get(12).toString().contains("Reason"));
+
+ // successful query
+ conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true");
+ handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE, QueryStatus.Status.SUCCESSFUL);
+ messages = new ArrayList<WiserMessage>();
+ for (int i = 0; i < 20; i++) {
+ messages = wiser.getMessages();
+ if (messages.size() > 16) {
+ break;
+ }
+ Thread.sleep(10000);
+ }
+
+ Assert.assertEquals(messages.size(), 20);
+ Assert.assertTrue(messages.get(16).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(16).toString().contains("Query SUCCESSFUL"));
+ Assert.assertTrue(messages.get(16).toString().contains("Result available at"));
+
+ wiser.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/36764bb7/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 14a4eb2..e3e3d4b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -66,8 +66,6 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
-import org.subethamail.wiser.Wiser;
-import org.subethamail.wiser.WiserMessage;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
@@ -93,9 +91,6 @@ public class TestQueryService extends LensJerseyTest {
/** The lens session id. */
LensSessionHandle lensSessionId;
- /** The wiser. */
- private Wiser wiser;
-
/*
* (non-Javadoc)
*
@@ -104,9 +99,6 @@ public class TestQueryService extends LensJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
- wiser = new Wiser();
- wiser.setHostname("localhost");
- wiser.setPort(25000);
queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
metricsSvc = (MetricsService) LensServices.get().getService(MetricsService.NAME);
Map<String, String> sessionconf = new HashMap<String, String>();
@@ -779,61 +771,6 @@ public class TestQueryService extends LensJerseyTest {
}
/**
- * Test notification.
- *
- * @throws IOException Signals that an I/O exception has occurred.
- * @throws InterruptedException the interrupted exception
- */
- @Test
- public void testNotification() throws IOException, InterruptedException {
- wiser.start();
- final WebTarget target = target().path("queryapi/queries");
- final FormDataMultiPart mp2 = new FormDataMultiPart();
- LensConf conf = new LensConf();
- conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false");
- conf.addProperty(LensConfConstants.QUERY_MAIL_NOTIFY, "true");
- conf.addProperty(LensConfConstants.QUERY_RESULT_EMAIL_CC, "foo1@localhost,foo2@localhost,foo3@localhost");
- mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
- MediaType.APPLICATION_XML_TYPE));
- mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID, IDSTR from "
- + TEST_TABLE));
- mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
- mp2.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
- MediaType.APPLICATION_XML_TYPE));
- final QueryHandle handle = target.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE),
- QueryHandle.class);
-
- Assert.assertNotNull(handle);
-
- LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
- .get(LensQuery.class);
- Assert.assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
- || ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING)
- || ctx.getStatus().getStatus().equals(Status.SUCCESSFUL));
-
- // wait till the query finishes
- QueryStatus stat = ctx.getStatus();
- while (!stat.isFinished()) {
- ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- stat = ctx.getStatus();
- Thread.sleep(1000);
- }
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
- List<WiserMessage> messages = new ArrayList<WiserMessage>();
- for (int i = 0; i < 20; i++) {
- messages = wiser.getMessages();
- if (messages.size() > 0) {
- break;
- }
- Thread.sleep(10000);
- }
-
- Assert.assertEquals(messages.size(), 4);
- Assert.assertTrue(messages.get(0).toString().contains(handle.toString()));
- wiser.stop();
- }
-
- /**
* Validate persisted result.
*
* @param handle the handle