You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by de...@apache.org on 2016/02/03 13:10:59 UTC
[39/51] [abbrv] lens git commit: LENS-912 : Make keyword optional in
queries
LENS-912 : Make keyword optional in queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b3f993d8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b3f993d8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b3f993d8
Branch: refs/heads/current-release-line
Commit: b3f993d8af508f900b22b7c67813b734b50255d8
Parents: 8a36572
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Jan 27 16:58:18 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jan 27 16:58:18 2016 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/query/QueryStatus.java | 5 +
.../lens/cube/metadata/CubeMetastoreClient.java | 20 +-
.../org/apache/lens/cube/parse/HQLParser.java | 7 +
.../server/query/QueryExecutionServiceImpl.java | 2 +-
.../lens/server/query/QueryServiceResource.java | 3 +-
.../lens/server/rewrite/CubeKeywordRemover.java | 41 ++++
.../apache/lens/server/rewrite/RewriteUtil.java | 150 +++++++++-----
.../rewrite/UserQueryToCubeQueryRewriter.java | 2 +-
.../lens/server/query/TestQueryService.java | 36 ++--
.../server/rewrite/CubeKeywordRemoverTest.java | 46 +++++
.../lens/server/rewrite/TestRewriting.java | 198 +++++++++++++++----
src/site/apt/user/olap-cube.apt | 2 +-
12 files changed, 403 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index 91cbe39..44fd97e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -220,6 +220,11 @@ public class QueryStatus implements Serializable {
return status.equals(Status.QUEUED);
}
+ public boolean failed() {
+ return status.equals(Status.FAILED);
+ }
+
+
/**
* Checks if is valid transition.
*
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index dcb932e..8969d1f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -40,14 +40,16 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
-import org.apache.hadoop.hive.ql.metadata.*;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.thrift.TException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-
import lombok.extern.slf4j.Slf4j;
/**
@@ -234,6 +236,20 @@ public class CubeMetastoreClient {
return latestParts;
}
+ public boolean isLensQueryableTable(String tableName) {
+ try {
+ Table table = getTable(tableName);
+ String typeProperty = table.getProperty(MetastoreConstants.TABLE_TYPE_KEY);
+ if (StringUtils.isBlank(typeProperty)) {
+ return false;
+ }
+ CubeTableType type = CubeTableType.valueOf(typeProperty);
+ return type == CubeTableType.CUBE || type == CubeTableType.DIMENSION;
+ } catch (HiveException e) {
+ return false;
+ }
+ }
+
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index b1deb07..c9aff5d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -833,4 +833,11 @@ public final class HQLParser {
return true;
}
+
+ public static ASTNode leftMostChild(ASTNode node) {
+ while (node.getChildren() != null) {
+ node = (ASTNode) node.getChild(0);
+ }
+ return node;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 2dff9af..672f2be 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -432,7 +432,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
log.info("Driver {} for type {} is loaded", driverPath.getName(), driverType);
} catch (Exception e) {
log.error("Could not load driver {} of type {}", driverPath.getName(), driverType, e);
- throw new LensException("Could not load driver "+driverPath.getName()+ " of type "+ driverType);
+ throw new LensException("Could not load driver "+driverPath.getName()+ " of type "+ driverType, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 08192bd..bb4cfd2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -199,7 +199,8 @@ public class QueryServiceResource {
* {@link org.apache.lens.api.query.SubmitOp#EXECUTE} operation.
* {@link QueryPlan} in case of {@link org.apache.lens.api.query.SubmitOp#EXPLAIN} operation.
* {@link QueryHandleWithResultSet} in case {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT}
- * operation. {@link QueryCostTO} in case of {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation.
+ * operation. {@link org.apache.lens.api.result.QueryCostTO} in case of
+ * {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation.
*/
@POST
@Path("queries")
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java
new file mode 100644
index 0000000..fd8d6e7
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.rewrite;
+
+
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.rewrite.Phase1Rewriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+public class CubeKeywordRemover implements Phase1Rewriter {
+ @Override
+ public String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException {
+ return query.replaceAll("(?i)cube\\s+select", "select");
+ }
+
+ @Override
+ public void init(Configuration rewriteConf) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
index abec2b3..1c0cd35 100644
--- a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
@@ -22,9 +22,8 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.lens.cube.parse.CubeQueryRewriter;
import org.apache.lens.cube.parse.HQLParser;
@@ -38,6 +37,7 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -52,14 +52,6 @@ public final class RewriteUtil {
private RewriteUtil() {
}
-
- /** The cube pattern. */
- static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE
- | Pattern.DOTALL);
-
- /** The matcher. */
- static Matcher matcher = null;
-
/**
* The Class CubeQueryInfo.
*/
@@ -92,14 +84,15 @@ public final class RewriteUtil {
if (log.isDebugEnabled()) {
log.debug("User query AST:{}", ast.dump());
}
- List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>();
- findCubePositions(ast, cubeQueries, query);
+ List<CubeQueryInfo> cubeQueries = new ArrayList<>();
+ findCubePositions(ast, cubeQueries, query, conf);
for (CubeQueryInfo cqi : cubeQueries) {
cqi.query = query.substring(cqi.startPos, cqi.endPos);
}
return cubeQueries;
}
+
/**
* Find cube positions.
*
@@ -108,7 +101,8 @@ public final class RewriteUtil {
* @param originalQuery the original query
* @throws LensException the lens exception
*/
- private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery)
+ private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery,
+ HiveConf conf)
throws LensException {
int childCount = ast.getChildCount();
if (ast.getToken() != null) {
@@ -116,13 +110,22 @@ public final class RewriteUtil {
log.debug("First child: {} Type:{}", ast.getChild(0), ((ASTNode) ast.getChild(0)).getToken().getType());
}
if (ast.getToken().getType() == HiveParser.TOK_QUERY
- && ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
+ && (isCubeKeywordNode((ASTNode) ast.getChild(0)) || isFromNodeWithCubeTable((ASTNode) ast.getChild(0), conf))) {
log.debug("Inside cube clause");
CubeQueryInfo cqi = new CubeQueryInfo();
cqi.cubeAST = ast;
if (ast.getParent() != null) {
ASTNode parent = (ASTNode) ast.getParent();
- cqi.startPos = ast.getCharPositionInLine();
+ if (isCubeKeywordNode((ASTNode) ast.getChild(0))) {
+ cqi.startPos = ast.getCharPositionInLine();
+ } else {
+ ASTNode selectAST = (ASTNode) ast.getChild(1).getChild(1);
+ // Left most child of select AST will have char position just after select / select distinct
+ // Go back one "select[ distinct]"
+ cqi.startPos = getStartPos(originalQuery, HQLParser.leftMostChild(selectAST).getCharPositionInLine(),
+ "distinct");
+ cqi.startPos = getStartPos(originalQuery, cqi.startPos, "select");
+ }
int ci = ast.getChildIndex();
if (parent.getToken() == null || parent.getToken().getType() == HiveParser.TOK_EXPLAIN
|| parent.getToken().getType() == HiveParser.TOK_CREATETABLE) {
@@ -134,7 +137,17 @@ public final class RewriteUtil {
cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine(), ")");
} else if (parent.getToken().getType() == HiveParser.TOK_UNION) {
// one less for the next start and less the size of string 'UNION ALL'
- cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL");
+ ASTNode nextChild = (ASTNode) parent.getChild(ci + 1);
+ if (isCubeKeywordNode((ASTNode) nextChild.getChild(0))) {
+ cqi.endPos = getEndPos(originalQuery, nextChild.getCharPositionInLine() - 1, "UNION ALL");
+ } else {
+ // Go back one "union all select[ distinct]"
+ cqi.endPos = getEndPos(originalQuery, nextChild.getChild(1).getChild(1).getCharPositionInLine() - 1,
+ "distinct");
+ cqi.endPos = getEndPos(originalQuery, cqi.endPos, "select");
+ cqi.endPos = getEndPos(originalQuery, cqi.endPos, "union all");
+ }
+
} else {
// Not expected to reach here
log.warn("Unknown query pattern found with AST:{}", ast.dump());
@@ -156,7 +169,7 @@ public final class RewriteUtil {
cubeQueries.add(cqi);
} else {
for (int childPos = 0; childPos < childCount; ++childPos) {
- findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery);
+ findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery, conf);
}
}
} else {
@@ -164,6 +177,39 @@ public final class RewriteUtil {
}
}
+ private static boolean isCubeTableNode(ASTNode node, HiveConf conf) throws LensException {
+ if (node.getType() == HiveParser.TOK_TABREF || node.getType() == HiveParser.TOK_TABNAME) {
+ return isCubeTableNode((ASTNode) node.getChild(0), conf);
+ }
+ if (node.getText().contains("JOIN")) {
+ if (isCubeTableNode((ASTNode) node.getChild(0), conf)) {
+ for (int i = 1; i < node.getChildCount(); i += 2) {
+ if (!isCubeTableNode((ASTNode) node.getChild(i), conf)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+ return node.getType() == HiveParser.Identifier && getClient(conf).isLensQueryableTable(node.getText());
+ }
+
+ private static boolean isFromNodeWithCubeTable(ASTNode child, HiveConf conf) throws LensException {
+ return child.getType() == HiveParser.TOK_FROM && isCubeTableNode((ASTNode) child.getChild(0), conf);
+ }
+
+ public static CubeMetastoreClient getClient(HiveConf conf) throws LensException {
+ try {
+ return CubeMetastoreClient.getInstance(conf);
+ } catch (HiveException e) {
+ throw new LensException("Couldn't get instance of metastore client", e);
+ }
+ }
+
+ private static boolean isCubeKeywordNode(ASTNode child) {
+ return child.getToken().getType() == HiveParser.KW_CUBE;
+ }
+
/**
* Gets the end pos.
*
@@ -173,17 +219,38 @@ public final class RewriteUtil {
* @return the end pos
*/
private static int getEndPos(String query, int backTrackIndex, String... backTrackStr) {
+ backTrackIndex = backTrack(query, backTrackIndex, backTrackStr);
+ while (backTrackIndex > 0 && Character.isSpaceChar(query.charAt(backTrackIndex - 1))) {
+ backTrackIndex--;
+ }
+ return backTrackIndex;
+ }
+
+ private static int backTrack(String query, int backTrackIndex, String... backTrackStr) {
if (backTrackStr != null) {
String q = query.substring(0, backTrackIndex).toLowerCase();
- for (int i = 0; i < backTrackStr.length; i++) {
- if (q.trim().endsWith(backTrackStr[i].toLowerCase())) {
- backTrackIndex = q.lastIndexOf(backTrackStr[i].toLowerCase());
+ for (String aBackTrackStr : backTrackStr) {
+ if (q.trim().endsWith(aBackTrackStr.toLowerCase())) {
+ backTrackIndex = q.lastIndexOf(aBackTrackStr.toLowerCase());
break;
}
}
}
- while (Character.isSpaceChar(query.charAt(backTrackIndex - 1))) {
- backTrackIndex--;
+ return backTrackIndex;
+ }
+
+ /**
+ * Gets the end pos.
+ *
+ * @param query the query
+ * @param backTrackIndex the back track index
+ * @param backTrackStr the back track str
+ * @return the end pos
+ */
+ private static int getStartPos(String query, int backTrackIndex, String... backTrackStr) {
+ backTrackIndex = backTrack(query, backTrackIndex, backTrackStr);
+ while (backTrackIndex < query.length() && Character.isSpaceChar(query.charAt(backTrackIndex))) {
+ backTrackIndex++;
}
return backTrackIndex;
}
@@ -224,7 +291,7 @@ public final class RewriteUtil {
try {
String replacedQuery = getReplacedQuery(ctx.getPhase1RewrittenQuery());
- Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<LensDriver, DriverRewriterRunnable>();
+ Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<>();
List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
@@ -246,8 +313,7 @@ public final class RewriteUtil {
}
public static DriverQueryPlan getRewriterPlan(DriverRewriterRunnable rewriter) {
- RewriterPlan plan = new RewriterPlan(rewriter.cubeQueryCtx);
- return plan;
+ return new RewriterPlan(rewriter.cubeQueryCtx);
}
public static class DriverRewriterRunnable implements Runnable {
@@ -275,15 +341,15 @@ public final class RewriteUtil {
private String rewrittenQuery;
public DriverRewriterRunnable(LensDriver driver,
- AbstractQueryContext ctx,
- List<CubeQueryInfo> cubeQueries,
- String replacedQuery) {
+ AbstractQueryContext ctx,
+ List<CubeQueryInfo> cubeQueries,
+ String replacedQuery) {
this.driver = driver;
this.ctx = ctx;
this.cubeQueries = cubeQueries;
this.replacedQuery = replacedQuery;
if (cubeQueries != null) {
- cubeQueryCtx = new ArrayList<CubeQueryContext>(cubeQueries.size());
+ cubeQueryCtx = new ArrayList<>(cubeQueries.size());
}
}
@@ -296,7 +362,7 @@ public final class RewriteUtil {
}
MethodMetricsContext rewriteGauge = MethodMetricsFactory
- .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE);
+ .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE);
StringBuilder builder = new StringBuilder();
int start = 0;
CubeQueryRewriter rewriter = null;
@@ -320,7 +386,7 @@ public final class RewriteUtil {
// Parse and rewrite individual cube query
CubeQueryContext cqc = rewriter.rewrite(cqi.query);
MethodMetricsContext toHQLGauge = MethodMetricsFactory
- .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE);
+ .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE);
// toHQL actually generates the rewritten query
String hqlQuery = cqc.toHQL();
cubeQueryCtx.add(cqc);
@@ -368,25 +434,9 @@ public final class RewriteUtil {
log.warn("Driver : {} Skipped for the query rewriting due to ", driver, e);
ctx.setDriverRewriteError(driver, e);
failureCause = new StringBuilder(" Driver :")
- .append(driver.getFullyQualifiedName())
- .append(" Cause :" + e.getLocalizedMessage())
- .toString();
- }
- }
-
- /**
- * Checks if is cube query.
- *
- * @param query the query
- * @return true, if is cube query
- */
- public static boolean isCubeQuery(String query) {
- if (matcher == null) {
- matcher = cubePattern.matcher(query);
- } else {
- matcher.reset(query);
+ .append(driver.getFullyQualifiedName())
+ .append(" Cause :" + e.getLocalizedMessage())
+ .toString();
}
- return matcher.matches();
}
-
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
index 35de4d7..79cee62 100644
--- a/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
@@ -39,7 +39,7 @@ import com.google.common.collect.Lists;
* @see LensConfConstants#QUERY_PHASE1_REWRITERS
*/
public class UserQueryToCubeQueryRewriter {
- List<Phase1Rewriter> phase1RewriterList = Lists.newArrayList();
+ List<Phase1Rewriter> phase1RewriterList = Lists.<Phase1Rewriter>newArrayList(new CubeKeywordRemover());
public UserQueryToCubeQueryRewriter(Configuration conf) throws LensException {
try {
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 494bce5..3facded 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -228,7 +228,7 @@ public class TestQueryService extends LensJerseyTest {
Collection<LensDriver> drivers = queryService.getDrivers();
assertEquals(drivers.size(), 4);
Set<String> driverNames = new HashSet<String>(drivers.size());
- for(LensDriver driver : drivers){
+ for (LensDriver driver : drivers) {
assertEquals(driver.getConf().get("lens.driver.test.drivername"), driver.getFullyQualifiedName());
driverNames.add(driver.getFullyQualifiedName());
}
@@ -458,7 +458,7 @@ public class TestQueryService extends LensJerseyTest {
Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
LensAPIResult result = response.readEntity(LensAPIResult.class);
List<LensErrorTO> childErrors = result.getLensErrorTO().getChildErrors();
- boolean hiveSemanticErrorExists=false;
+ boolean hiveSemanticErrorExists = false;
for (LensErrorTO error : childErrors) {
if (error.getCode() == LensDriverErrorCode.SEMANTIC_ERROR.getLensErrorInfo().getErrorCode()) {
hiveSemanticErrorExists = true;
@@ -1335,7 +1335,7 @@ public class TestQueryService extends LensJerseyTest {
if (driver instanceof HiveDriver) {
addedToHiveDriver =
((HiveDriver) driver).areDBResourcesAddedForSession(sessionHandle.getPublicId().toString(), DB_WITH_JARS);
- if (addedToHiveDriver){
+ if (addedToHiveDriver) {
break; //There are two Hive drivers now both pointing to same hive server. So break after first success
}
}
@@ -1382,7 +1382,8 @@ public class TestQueryService extends LensJerseyTest {
final FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "cube select ID from nonexist"));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(),
+ "cube sdfelect ID from cube_nonexist"));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "estimate"));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
MediaType.APPLICATION_XML_TYPE));
@@ -1392,8 +1393,9 @@ public class TestQueryService extends LensJerseyTest {
LensErrorTO expectedLensErrorTO = LensErrorTO.composedOf(
- LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo().getErrorCode(),
- "Neither cube nor dimensions accessed in the query", TestDataUtils.MOCK_STACK_TRACE);
+ LensCubeErrorCode.SYNTAX_ERROR.getLensErrorInfo().getErrorCode(),
+ "Syntax Error: line 1:5 cannot recognize input near 'sdfelect' 'ID' 'from' in select clause",
+ TestDataUtils.MOCK_STACK_TRACE);
ErrorResponseExpectedData expectedData = new ErrorResponseExpectedData(BAD_REQUEST, expectedLensErrorTO);
expectedData.verify(response);
@@ -1458,17 +1460,17 @@ public class TestQueryService extends LensJerseyTest {
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-CUBE_REWRITE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-DRIVER_ESTIMATE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-CUBE_REWRITE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-DRIVER_ESTIMATE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-CUBE_REWRITE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-DRIVER_ESTIMATE",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")),
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-CUBE_REWRITE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-DRIVER_ESTIMATE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-CUBE_REWRITE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-DRIVER_ESTIMATE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-CUBE_REWRITE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-DRIVER_ESTIMATE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")),
reg.getGauges().keySet().toString());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java
new file mode 100644
index 0000000..956d433
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.rewrite;
+
+import static org.testng.Assert.assertEquals;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class CubeKeywordRemoverTest {
+ CubeKeywordRemover cubeKeywordRemover = new CubeKeywordRemover();
+
+ @DataProvider
+ public Object[][] cubeQueryDataProvider() {
+ return new Object[][]{
+ {"cube select blah blah", "select blah blah"},
+ {"cube\tselect blah blah", "select blah blah"},
+ {"cube\nselect blah blah", "select blah blah"},
+ {"CUBE sElEct blAh blAh", "select blAh blAh"},
+ };
+ }
+
+ @Test(dataProvider = "cubeQueryDataProvider")
+ public void testRewrite(String userQuery, String expected) throws Exception {
+ assertEquals(cubeKeywordRemover.rewrite(userQuery, null, null), expected);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
index 202db82..0e640ec 100644
--- a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
@@ -21,6 +21,7 @@ package org.apache.lens.server.rewrite;
import java.util.*;
import org.apache.lens.api.LensConf;
+import org.apache.lens.cube.metadata.CubeMetastoreClient;
import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.lens.cube.parse.CubeQueryRewriter;
import org.apache.lens.cube.parse.HQLParser;
@@ -35,7 +36,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.parse.*;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ParseException;
import org.mockito.Matchers;
import org.mockito.Mockito;
@@ -75,7 +78,19 @@ public class TestRewriting {
// number of successful queries through mock rewriter
// we use this number to mock failures after successful queries
// change the number, if more tests for success needs to be added
- static final int NUM_SUCCESS = 36;
+ static final int NUM_SUCCESS = 63;
+
+ public static CubeMetastoreClient getMockedClient() {
+ CubeMetastoreClient client = Mockito.mock(CubeMetastoreClient.class);
+ Mockito.when(client.isLensQueryableTable(Matchers.any(String.class))).thenAnswer(new Answer<Boolean>() {
+
+ @Override
+ public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable {
+ return invocationOnMock.getArguments()[0].toString().toLowerCase().contains("cube");
+ }
+ });
+ return client;
+ }
private CubeQueryRewriter getMockedRewriter() throws ParseException, LensException, HiveException {
CubeQueryRewriter mockwriter = Mockito.mock(CubeQueryRewriter.class);
@@ -83,9 +98,8 @@ public class TestRewriting {
@Override
public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
- i++;
// return query for first NUM_SUCCESS calls and fail later
- if (i <= NUM_SUCCESS) {
+ if (++i <= NUM_SUCCESS) {
return getMockedCubeContext((String) args[0]);
} else {
throw new RuntimeException("Mock fail");
@@ -114,7 +128,8 @@ public class TestRewriting {
throws ParseException, LensException {
CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
Mockito.when(context.toHQL()).thenReturn(query.substring(4));
- Mockito.when(context.toAST(Matchers.any(Context.class))).thenReturn(HQLParser.parseHQL(query.substring(4), hconf));
+ Mockito.when(context.toAST(Matchers.any(Context.class)))
+ .thenReturn(HQLParser.parseHQL(query.toLowerCase().replaceFirst("^cube", ""), hconf));
return context;
}
@@ -181,9 +196,10 @@ public class TestRewriting {
drivers.add(driver);
CubeQueryRewriter mockWriter = getMockedRewriter();
+ CubeMetastoreClient mockClient = getMockedClient();
PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter);
+ PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getClient")).toReturn(mockClient);
String q1 = "select name from table";
- Assert.assertFalse(RewriteUtil.isCubeQuery(q1));
List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
Assert.assertEquals(cubeQueries.size(), 0);
QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers);
@@ -192,7 +208,6 @@ public class TestRewriting {
conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
driver.configure(conf, null, null);
String q2 = "cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -201,20 +216,43 @@ public class TestRewriting {
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-1-RewriteUtil-rewriteQuery-toHQL")));
+ "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-1-RewriteUtil-rewriteQuery-toHQL")));
conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
+ assertIsCubeQuery("select name from cube_table", lensConf, conf, drivers);
+ assertIsCubeQuery("select cube_table.name from cube_table", lensConf, conf, drivers);
+ assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim "
+ + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
+ assertIsCubeQuery("select distinct cube_dim.name, cube_dim2.name from cube_dim full outer "
+ + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
+ assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim left outer "
+ + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
+ assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim inner "
+ + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
+
+ q2 = "select * from (select name from cube_table) table1";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "insert overwrite directory 'target/rewrite' cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "insert overwrite directory 'target/rewrite' select name from cube_table";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "insert overwrite local directory 'target/rewrite' cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -222,7 +260,6 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "insert overwrite local directory 'target/example-output' cube select id,name from dim_table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table");
@@ -230,7 +267,6 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "explain cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -238,7 +274,6 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -246,15 +281,20 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "insert overwrite directory 'target/rewrite' select * from (cube select name from table) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "insert overwrite directory 'target/rewrite' select * from (select name from cube_table) a";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "select * from (cube select name from table)a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -262,16 +302,21 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from ( cube select name from table ) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "select * from ( select name from cube_table ) a";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "select * from ( cube select name from table where"
+ " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from"
@@ -279,9 +324,17 @@ public class TestRewriting {
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "select * from ( select name from cube_table where"
+ + " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a";
+ cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from"
+ + " cube_table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName() + "-multiple");
q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -290,15 +343,15 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
reg = LensMetricsRegistry.getStaticRegistry();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-1-RewriteUtil-rewriteQuery-toHQL",
- "lens.MethodMetricGauge.TestRewriting-multiple-"+driver.getFullyQualifiedName()
- +"-2-RewriteUtil-rewriteQuery-toHQL",
- "lens.MethodMetricGauge.TestRewriting-multiple-"+driver.getFullyQualifiedName()+"-RewriteUtil-rewriteQuery")));
+ "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-1-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName()
+ + "-2-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName()
+ + "-RewriteUtil-rewriteQuery")));
conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
q2 = "select * from (cube select name from table) a full outer join"
+ " (cube select name2 from table2) b on a.name=b.name2";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -306,16 +359,31 @@ public class TestRewriting {
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "select * from (select name from cube_table) a full outer join"
+ + " (select name2 from cube_table2) b on a.name=b.name2";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "select * from (cube select name from table) a join (select name2 from table2) b";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "select * from (cube select name from table) a join (select name2 from cube_table2) b";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "select * from (cube select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -323,9 +391,24 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ q2 = "select * from (select name from cube_table union all select distinct name2 from cube_table2) u";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select distinct name2 from cube_table2");
+
+ q2 = "select * from (select distinct name from cube_table union all select name2 from cube_table2) u";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "select distinct name from cube_table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
+
q2 = "insert overwrite directory 'target/rewrite' "
+ "select * from (cube select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -333,8 +416,16 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ q2 = "insert overwrite directory 'target/rewrite' "
+ + "select * from (cube select name from table union all select name2 from cube_table2) u";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
+
q2 = "select u.* from (select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
@@ -342,7 +433,6 @@ public class TestRewriting {
runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select u.* from (select name from table union all cube select name2 from table2)u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
@@ -351,7 +441,6 @@ public class TestRewriting {
q2 = "select * from (cube select name from table union all cube select name2"
+ " from table2 union all cube select name3 from table3) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -362,7 +451,6 @@ public class TestRewriting {
q2 = "select * from ( cube select name from table union all cube"
+ " select name2 from table2 union all cube select name3 from table3 ) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -372,7 +460,6 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");
q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -380,8 +467,15 @@ public class TestRewriting {
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
+ q2 = "select * from (cube select name from table union all select" + " name2 from cube_table2) u group by u.name";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -390,16 +484,21 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
q2 = "create table temp1 as cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ q2 = "create table temp1 as select name from cube_table";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+
q2 = "create table temp1 as select * from (cube select name from table union all cube select"
+ " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -407,9 +506,18 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ q2 = "create table temp1 as select * from (select name from cube_table union all cube select"
+ + " name2 from table2) u group by u.name";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+
+
q2 = "create table temp1 as cube select name from table where"
+ " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
runRewrites(RewriteUtil.rewriteQuery(ctx));
@@ -417,6 +525,15 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query,
"cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");
+ q2 = "create table temp1 as select name from cube_table where"
+ + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query,
+ "select name from cube_table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");
+
// failing query for second driver
MockDriver driver2 = new MockDriver();
driver2.configure(conf, null, null);
@@ -460,4 +577,13 @@ public class TestRewriting {
Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
}
+
+ private void assertIsCubeQuery(String query, LensConf lensConf, Configuration conf, List<LensDriver> drivers)
+ throws LensException {
+ List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(query, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, query);
+ QueryContext ctx = new QueryContext(query, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/src/site/apt/user/olap-cube.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/olap-cube.apt b/src/site/apt/user/olap-cube.apt
index c47ac30..4bed623 100644
--- a/src/site/apt/user/olap-cube.apt
+++ b/src/site/apt/user/olap-cube.apt
@@ -283,7 +283,7 @@ lens-shell>
+---+
- CUBE SELECT [DISTINCT] select_expr, select_expr, ...
+ [CUBE] SELECT [DISTINCT] select_expr, select_expr, ...
FROM cube_table_reference
[WHERE [where_condition AND] [TIME_RANGE_IN(colName, from, to)]]
[GROUP BY col_list]