You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/03/10 14:35:08 UTC
[1/3] incubator-lens git commit: LENS-396: reduce logs in
MethodMetricsFactory.createMethodGauge(Amareshwari Sriramadasu via prongs)
Repository: incubator-lens
Updated Branches:
refs/heads/master 803448a18 -> 85b3d9e9e
LENS-396: reduce logs in MethodMetricsFactory.createMethodGauge(Amareshwari Sriramadasu via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ce7712c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ce7712c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ce7712c7
Branch: refs/heads/master
Commit: ce7712c76839c463d5104cd4c3e379b54c302778
Parents: 803448a
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Mar 10 18:40:56 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Tue Mar 10 18:40:56 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/api/metrics/MethodMetricsFactory.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ce7712c7/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
index 3227d20..5b6ec3b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
@@ -170,11 +170,9 @@ public class MethodMetricsFactory {
if (StringUtils.isBlank(uid)) {
return DisabledMethodMetricsContext.getInstance();
}
- LOG.info("query metricid:" + uid);
StringBuilder metricName = new StringBuilder();
if (appendToStackName) {
String stackName = conf.get(LensConfConstants.QUERY_METRIC_DRIVER_STACK_NAME);
- LOG.info("query metric stackname:" + stackName);
metricName.append(stackName);
metricName.append("-");
} else {
[2/3] incubator-lens git commit: LENS-394: Solves unexpected end of
stament in QueryPurger in tests (Jaideep Dhok via prongs)
Posted by pr...@apache.org.
LENS-394: Solves unexpected end of stament in QueryPurger in tests (Jaideep Dhok via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/93536b8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/93536b8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/93536b8f
Branch: refs/heads/master
Commit: 93536b8f3f7dd8d9cc8fa13cec655cf40d398fb5
Parents: ce7712c
Author: Jaideep Dhok <jd...@apache.org>
Authored: Tue Mar 10 19:02:50 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Tue Mar 10 19:02:50 2015 +0530
----------------------------------------------------------------------
lens-ml-lib/src/test/resources/lens-site.xml | 2 --
.../java/org/apache/lens/server/api/LensConfConstants.java | 2 +-
lens-server/src/main/resources/lensserver-default.xml | 2 +-
lens-server/src/test/resources/lens-site.xml | 7 -------
src/site/apt/admin/config.apt | 8 ++++----
5 files changed, 6 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/93536b8f/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 4fdd79a..e2f86ae 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -158,6 +158,4 @@
<value>org.apache.lens.server.MockNonLensService</value>
<description>Implementation class for session service</description>
</property>
-
-
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/93536b8f/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 0bf78f0..cdc5fd7 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -792,7 +792,7 @@ public final class LensConfConstants {
public static final String SERVER_DB_VALIDATION_QUERY = SERVER_PFX + "db.validation.query";
/** default value of the validation query */
- public static final String DEFAULT_SERVER_DB_VALIDATION_QUERY = "select 1";
+ public static final String DEFAULT_SERVER_DB_VALIDATION_QUERY = "select 1 from INFORMATION_SCHEMA.SYSTEM_USERS";
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/93536b8f/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 238d329..57fbbc7 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -280,7 +280,7 @@
</property>
<property>
<name>lens.server.db.validation.query</name>
- <value>select 1</value>
+ <value>select 1 from INFORMATION_SCHEMA.SYSTEM_USERS</value>
<description>BasicDatasource has a config parameter called validationQuery. This query is run before passing
the Datasource to the application to verify if connection is valid.\
{{{http://commons.apache.org/proper/commons-dbcp/configuration.html}dbcp configuration}} \
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/93536b8f/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index b20f565..2adf762 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -125,13 +125,6 @@
</property>
<property>
- <name>lens.server.db.validation.query</name>
- <value>select 1 from INFORMATION_SCHEMA.SYSTEM_USERS</value>
- <description>Validation query for hsql
- </description>
- </property>
-
- <property>
<name>lens.server.servicenames</name>
<value>session,query,metastore,scheduler,quota,mocknonlens</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/93536b8f/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 794b748..9e06da3 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -56,10 +56,10 @@ Lens server configuration
*--+--+---+--+
|16|lens.server.db.jdbc.user|SA|JDBC User for Finished queries table. Also used for database based user config loaders|
*--+--+---+--+
-|17|lens.server.db.validation.query|select 1|BasicDatasource has a config parameter called validationQuery. This query is run before passing the Datasource to the application to verify if connection is valid.\ |
-| | | |{{{http://commons.apache.org/proper/commons-dbcp/configuration.html}dbcp configuration}} \ |
-| | | |{{{http://stackoverflow.com/questions/10684244/dbcp-validationquery-for-different-databases}validationquery for different databases}} \ |
-| | | | |
+|17|lens.server.db.validation.query|select 1 from INFORMATION_SCHEMA.SYSTEM_USERS|BasicDatasource has a config parameter called validationQuery. This query is run before passing the Datasource to the application to verify if connection is valid.\ |
+| | | |{{{http://commons.apache.org/proper/commons-dbcp/configuration.html}dbcp configuration}} \ |
+| | | |{{{http://stackoverflow.com/questions/10684244/dbcp-validationquery-for-different-databases}validationquery for different databases}} \ |
+| | | | |
*--+--+---+--+
|18|lens.server.domain|company.com|Server domain. This will be used in authentication and Sending emails to users. For the server, the username will be without domain and domain will come through conf.|
*--+--+---+--+
[3/3] incubator-lens git commit: LENS-393: Do query field validation
for chain ref columns as well (Amareshwari Sriramadasu via prongs)
Posted by pr...@apache.org.
LENS-393: Do query field validation for chain ref columns as well (Amareshwari Sriramadasu via prongs)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/85b3d9e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/85b3d9e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/85b3d9e9
Branch: refs/heads/master
Commit: 85b3d9e9e89e26f560b4116ad16806eb2ba086c9
Parents: 93536b8
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Mar 10 19:04:29 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Tue Mar 10 19:04:29 2015 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/AliasReplacer.java | 38 +++++++++++++++-----
.../apache/lens/cube/parse/CubeTestSetup.java | 2 ++
.../lens/cube/parse/TestBaseCubeQueries.java | 19 ++++++++--
.../lens/cube/parse/TestQueryRewrite.java | 4 +--
4 files changed, 50 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/85b3d9e9/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
index 6ef72d3..d81fab1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
@@ -99,7 +99,7 @@ class AliasReplacer implements ContextRewriter {
// Finds all queried dim-attributes and measures from cube
// If all fields in cube are not queryable together, does the validation
- // wrt to dervided cubes.
+ // wrt to derived cubes.
private void doFieldValidation(CubeQueryContext cubeql) throws SemanticException {
CubeInterface cube = cubeql.getCube();
if (cube != null) {
@@ -125,34 +125,41 @@ class AliasReplacer implements ContextRewriter {
} catch (HiveException e) {
throw new SemanticException(e);
}
- // remove chained ref columns from field valdation
+ // remove chained ref columns from field validation
Iterator<String> iter = queriedDimAttrs.iterator();
+ Set<String> chainedSrcColumns = new HashSet<String>();
while (iter.hasNext()) {
String attr = iter.next();
if (cube.getDimAttributeByName(attr) instanceof ReferencedDimAtrribute
&& ((ReferencedDimAtrribute) cube.getDimAttributeByName(attr)).isChainedColumn()) {
iter.remove();
+ ReferencedDimAtrribute rdim = (ReferencedDimAtrribute)cube.getDimAttributeByName(attr);
+ chainedSrcColumns.addAll(cube.getChainByName(rdim.getChainName()).getSourceColumns());
}
}
+ for (JoinChain chainQueried : cubeql.getJoinchains().values()) {
+ chainedSrcColumns.addAll(chainQueried.getSourceColumns());
+ }
// do validation
// Find atleast one derived cube which contains all the dimensions
// queried.
boolean derivedCubeFound = false;
for (DerivedCube dcube : dcubes) {
- if (dcube.getDimAttributeNames().containsAll(queriedDimAttrs)) {
+ if (dcube.getDimAttributeNames().containsAll(chainedSrcColumns)
+ && dcube.getDimAttributeNames().containsAll(queriedDimAttrs)) {
// remove all the measures that are covered
queriedMsrs.removeAll(dcube.getMeasureNames());
derivedCubeFound = true;
}
}
- if (!derivedCubeFound && !queriedDimAttrs.isEmpty()) {
- throw new SemanticException(ErrorMsg.FIELDS_NOT_QUERYABLE, queriedDimAttrs.toString());
+ Set<String> nonQueryableFields = getNonQueryableAttributes(cubeql);
+ if (!derivedCubeFound && !nonQueryableFields.isEmpty()) {
+ throw new SemanticException(ErrorMsg.FIELDS_NOT_QUERYABLE, nonQueryableFields.toString());
}
if (!queriedMsrs.isEmpty()) {
- // Add appropriate message to know which fields are not queryable
- // together
- if (!queriedDimAttrs.isEmpty()) {
- throw new SemanticException(ErrorMsg.FIELDS_NOT_QUERYABLE, queriedDimAttrs.toString() + " and "
+ // Add appropriate message to know which fields are not queryable together
+ if (!nonQueryableFields.isEmpty()) {
+ throw new SemanticException(ErrorMsg.FIELDS_NOT_QUERYABLE, nonQueryableFields.toString() + " and "
+ queriedMsrs.toString());
} else {
throw new SemanticException(ErrorMsg.FIELDS_NOT_QUERYABLE, queriedMsrs.toString());
@@ -162,6 +169,19 @@ class AliasReplacer implements ContextRewriter {
}
}
+ private Set<String> getNonQueryableAttributes(CubeQueryContext cubeql) {
+ Set<String> nonQueryableFields = new LinkedHashSet<String>();
+ nonQueryableFields.addAll(cubeql.getQueriedDimAttrs());
+ for (String joinChainAlias : cubeql.getJoinchains().keySet()) {
+ if (cubeql.getColumnsQueried(joinChainAlias) != null) {
+ for (String chaincol : cubeql.getColumnsQueried(joinChainAlias)) {
+ nonQueryableFields.add(joinChainAlias + "." + chaincol);
+ }
+ }
+ }
+ return nonQueryableFields;
+ }
+
private void extractTabAliasForCol(CubeQueryContext cubeql) throws SemanticException {
Set<String> columns = cubeql.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE);
if (columns == null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/85b3d9e9/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 2350097..3fbdee9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -722,6 +722,8 @@ public class CubeTestSetup {
measures.add("msr13");
measures.add("msr14");
dimensions = new HashSet<String>();
+ dimensions.add("cityid");
+ dimensions.add("stateid");
dimensions.add("dim1");
dimensions.add("dim2");
dimensions.add("dim11");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/85b3d9e9/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 78e0b80..1f03db6 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -58,11 +58,26 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.FIELDS_NOT_QUERYABLE.getErrorCode());
Assert.assertTrue(e.getMessage().contains("dim2") && e.getMessage().contains("msr1"));
- e = getSemanticExceptionInRewrite("select dim2, cityid, SUM(msr2) from basecube" + " where " + TWO_DAYS_RANGE,
+ e = getSemanticExceptionInRewrite("select cityStateCapital, SUM(msr1) from basecube" + " where " + TWO_DAYS_RANGE,
+ conf);
+ Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.FIELDS_NOT_QUERYABLE.getErrorCode());
+ Assert.assertTrue(e.getMessage().contains("citystatecapital") && e.getMessage().contains("msr1"));
+
+ e = getSemanticExceptionInRewrite("select cityState.name, SUM(msr1) from basecube" + " where " + TWO_DAYS_RANGE,
+ conf);
+ Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.FIELDS_NOT_QUERYABLE.getErrorCode());
+ Assert.assertTrue(e.getMessage().contains("citystate.name") && e.getMessage().contains("msr1"));
+
+ e = getSemanticExceptionInRewrite("select cubeState.name, SUM(msr1) from basecube" + " where " + TWO_DAYS_RANGE,
+ conf);
+ Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.FIELDS_NOT_QUERYABLE.getErrorCode());
+ Assert.assertTrue(e.getMessage().contains("cubestate.name") && e.getMessage().contains("msr1"));
+
+ e = getSemanticExceptionInRewrite("select dim2, countryid, SUM(msr2) from basecube" + " where " + TWO_DAYS_RANGE,
conf);
Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(),
ErrorMsg.FIELDS_NOT_QUERYABLE.getErrorCode());
- Assert.assertTrue(e.getMessage().contains("dim2") && e.getMessage().contains("cityid"));
+ Assert.assertTrue(e.getMessage().contains("dim2") && e.getMessage().contains("countryid"));
e = getSemanticExceptionInRewrite("select newmeasure from basecube" + " where " + TWO_DAYS_RANGE, conf);
Assert.assertEquals(e.getCanonicalErrorMsg().getErrorCode(),
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/85b3d9e9/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
index 66f160d..a1451b9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
@@ -84,8 +84,8 @@ public abstract class TestQueryRewrite {
protected SemanticException getSemanticExceptionInRewrite(String query, Configuration conf) throws ParseException {
try {
- rewrite(query, conf);
- Assert.fail("Should have thrown exception");
+ String hql = rewrite(query, conf);
+ Assert.fail("Should have thrown exception. But rewrote the query : " + hql);
// unreachable
return null;
} catch (SemanticException e) {