You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2017/10/05 03:14:03 UTC

[1/8] lens git commit: LENS-1463 : Valid Columns addition in Virtual Fact

Repository: lens
Updated Branches:
  refs/heads/current-release-line ff4f3726a -> 2772efb27


LENS-1463 : Valid Columns addition in Virtual Fact


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/358efef9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/358efef9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/358efef9

Branch: refs/heads/current-release-line
Commit: 358efef9a3214f2c57655f2ea227b646971ae005
Parents: ff4f372
Author: Rajitha R <ra...@gmail.com>
Authored: Mon Aug 28 16:31:33 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:11:42 2017 +0800

----------------------------------------------------------------------
 .../lens/cube/metadata/CubeVirtualFactTable.java       |  9 ++++++++-
 .../apache/lens/cube/parse/TestVirtualFactQueries.java | 13 +++++++++++++
 .../src/test/resources/schema/cubes/base/testcube2.xml |  2 ++
 .../src/test/resources/schema/facts/testfact9_base.xml |  3 ++-
 .../resources/schema/facts/virtual/virtualfact.xml     |  2 ++
 5 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/358efef9/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
index 1fc74b0..ce2a1a6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -92,7 +92,14 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable
 
   @Override
   public Set<String> getValidColumns() {
-    return this.sourceCubeFactTable.getValidColumns();
+    String validColsStr =
+      MetastoreUtil.getNamedStringValue(this.getProperties(), MetastoreUtil.getValidColumnsKey(getName()));
+    if (validColsStr == null) {
+      return this.sourceCubeFactTable.getValidColumns();
+    } else {
+      return new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(),
+        ',')));
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lens/blob/358efef9/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
index 4bcb376..94e836c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestVirtualFactQueries.java
@@ -25,8 +25,11 @@ import static org.apache.lens.cube.parse.CubeQueryConfUtil.DISABLE_AGGREGATE_RES
 import static org.apache.lens.cube.parse.CubeTestSetup.*;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
 
+import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.server.api.LensServerAPITestUtil;
+import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.conf.Configuration;
 
@@ -119,5 +122,15 @@ public class TestVirtualFactQueries extends TestQueryRewrite {
 
   }
 
+  @Test
+  public void testVirtualFactValidColumns() throws Exception {
+
+    try {
+      rewriteCtx("select SUM(msr4) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1"));
+      fail("Rewrite should not succeed here");
+    } catch (LensException exc) {
+      assertEquals(exc.getErrorCode(), LensCubeErrorCode.COLUMN_NOT_FOUND.getLensErrorInfo().getErrorCode());
+    }
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/lens/blob/358efef9/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
index 9740e00..237e85a 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube2.xml
@@ -27,5 +27,7 @@
   <measures>
     <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr1" display_string="Measure1"
              description="first measure"/>
+    <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr4" display_string="Measure4"
+             description="fourth measure"/>
   </measures>
 </x_base_cube>

http://git-wip-us.apache.org/repos/asf/lens/blob/358efef9/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/testfact9_base.xml b/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
index 56e5746..9c94fe2 100644
--- a/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
+++ b/lens-cube/src/test/resources/schema/facts/testfact9_base.xml
@@ -22,7 +22,8 @@
 <x_fact_table name="testfact9_base" cube_name="testcube2" weight="5.0" xmlns="uri:lens:cube:0.1">
   <columns>
 
-    <column name="msr1" _type="float" comment="second measure"/>
+    <column name="msr1" _type="float" comment="first measure"/>
+    <column name="msr4" _type="float" comment="fourth measure"/>
   </columns>
   <properties>
     <property name="cube.fact.testfact.cubename" value="testCube"/>

http://git-wip-us.apache.org/repos/asf/lens/blob/358efef9/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
index 13f6773..a7daae5 100644
--- a/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
+++ b/lens-cube/src/test/resources/schema/facts/virtual/virtualfact.xml
@@ -23,5 +23,7 @@
                       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
   <properties>
     <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
+    <property name="cube.fact.virtualfact1.valid.columns"
+              value="msr1"/>
   </properties>
 </x_virtual_fact_table>

[7/8] lens git commit: LENS-1467: CubeQueryContext.getAllFilters is returning incorrect list of filters in case there is an "OR" in the filters

Posted by ra...@apache.org.
LENS-1467: CubeQueryContext.getAllFilters is returning incorrect list of filters in case there is an "OR" in the filters


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9a678d8c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9a678d8c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9a678d8c

Branch: refs/heads/current-release-line
Commit: 9a678d8c2602860aac81bb31801c6c6acb946054
Parents: 13cbc81
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Aug 30 16:10:54 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:13:18 2017 +0800

----------------------------------------------------------------------
 .../lens/cube/parse/CubeQueryContext.java       | 67 ++++++-------------
 .../cube/parse/StorageCandidateHQLContext.java  |  3 +-
 .../apache/lens/cube/parse/join/JoinClause.java |  9 +++
 .../lens/cube/parse/CubeQueryContextTest.java   | 70 ++++++++++++++++++++
 4 files changed, 100 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/9a678d8c/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 8b9583a..bff5c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -49,7 +49,6 @@ import org.apache.lens.cube.metadata.*;
 import org.apache.lens.cube.metadata.join.TableRelationship;
 import org.apache.lens.cube.parse.join.AutoJoinContext;
 import org.apache.lens.cube.parse.join.JoinClause;
-import org.apache.lens.cube.parse.join.JoinTree;
 import org.apache.lens.cube.parse.join.JoinUtils;
 import org.apache.lens.server.api.error.LensException;
 
@@ -1137,14 +1136,13 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST,
     return ImmutableSet.copyOf(this.queriedTimeDimCols);
   }
 
-  String getWhere(StorageCandidateHQLContext sc, AutoJoinContext autoJoinCtx,
-    ASTNode node, String cubeAlias,
-    boolean shouldReplaceDimFilter, String storageTable,
-    Map<Dimension, CandidateDim> dimToQuery) throws LensException {
+  String getWhere(StorageCandidateHQLContext sc, AutoJoinContext autoJoinCtx, String cubeAlias,
+    boolean shouldReplaceDimFilter, Map<Dimension, CandidateDim> dimToQuery) throws LensException {
     String whereString;
     if (autoJoinCtx != null && shouldReplaceDimFilter) {
       List<String> allfilters = new ArrayList<>();
-      getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(sc.getStorageCandidate()), dimToQuery);
+      getAllFilters(sc.getQueryAst().getWhereAST(), cubeAlias, allfilters,
+        autoJoinCtx.getJoinClause(sc.getStorageCandidate()), dimToQuery);
       whereString = StringUtils.join(allfilters, " and ");
     } else {
       whereString = HQLParser.getString(sc.getQueryAst().getWhereAST());
@@ -1152,58 +1150,33 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST,
     return whereString;
   }
 
-  private void getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters,
-                                    JoinClause joinClause,  Map<Dimension, CandidateDim> dimToQuery)
-    throws LensException {
-
-    if (node.getToken().getType() == HiveParser.KW_AND) {
-      // left child is and
-      if (node.getChild(0).getType() == HiveParser.KW_AND) {
-        // take right corresponding to right
-        String table = getTableFromFilterAST((ASTNode) node.getChild(1));
-        allFilters.add(getFilter(table, cubeAlias, node, joinClause, 1, dimToQuery));
-      } else if (node.getChildCount() > 1) {
-        for (int i = 0; i < node.getChildCount(); i++) {
-          String table = getTableFromFilterAST((ASTNode) node.getChild(i));
-          allFilters.add(getFilter(table, cubeAlias, node, joinClause, i, dimToQuery));
-        }
+  protected static void getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters, JoinClause joinClause,
+    Map<Dimension, CandidateDim> dimToQuery) throws LensException {
+    if (node.getToken().getType() == HiveParser.KW_AND || node.getToken().getType() == HiveParser.TOK_WHERE) {
+      for (int i = 0; i < node.getChildCount(); i++) {
+        ASTNode child = (ASTNode) node.getChild(i);
+        getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery);
       }
-    } else if (node.getParent() == null
-        && node.getToken().getType() != HiveParser.KW_AND
-      && node.getChild(0).getType() != HiveParser.KW_AND) {
-      // if node is the only child
-      allFilters.add(HQLParser.getString((ASTNode) node));
-    }
-    for (int i = 0; i < node.getChildCount(); i++) {
-      ASTNode child = (ASTNode) node.getChild(i);
-      getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery);
+    } else {
+      String table = getTableFromFilterAST(node);
+      allFilters.add(getFilter(table, cubeAlias, node, joinClause, dimToQuery));
     }
   }
 
-  private String getFilter(String table, String cubeAlias, ASTNode node,  JoinClause joinClause,
-                           int index,  Map<Dimension, CandidateDim> dimToQuery)
+  private static String getFilter(String table, String cubeAlias, ASTNode node,  JoinClause joinClause,
+                           Map<Dimension, CandidateDim> dimToQuery)
     throws LensException{
     String filter;
-    if (table != null && !table.equals(cubeAlias) && getStarJoin(joinClause, table) != null) {
+    if (table != null && !table.equals(cubeAlias) && joinClause.getStarJoin(table) != null) {
       //rewrite dim filter to fact filter if its a star join with fact
-      filter = buildFactSubqueryFromDimFilter(getStarJoin(joinClause, table),
-          (ASTNode) node.getChild(index), table, dimToQuery, cubeAlias);
+      filter = buildFactSubqueryFromDimFilter(joinClause.getStarJoin(table), node, table, dimToQuery, cubeAlias);
     } else {
-      filter = HQLParser.getString((ASTNode) node.getChild(index));
+      filter = HQLParser.getString(node);
     }
     return filter;
   }
 
-  private TableRelationship getStarJoin(JoinClause joinClause, String table) {
-    for (Map.Entry<TableRelationship, JoinTree>  entry : joinClause.getJoinTree().getSubtrees().entrySet()) {
-      if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) {
-        return entry.getKey();
-      }
-    }
-    return null;
-  }
-
-  private String getTableFromFilterAST(ASTNode node) {
+  private static String getTableFromFilterAST(ASTNode node) {
 
     if (node.getToken().getType() == HiveParser.DOT) {
       ASTNode n = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
@@ -1222,7 +1195,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST,
     return null;
   }
 
-  private String buildFactSubqueryFromDimFilter(TableRelationship tabRelation, ASTNode dimFilter,
+  private static String buildFactSubqueryFromDimFilter(TableRelationship tabRelation, ASTNode dimFilter,
                                                 String dimAlias, Map<Dimension, CandidateDim> dimToQuery,
                                                 String cubeAlias)
     throws LensException {

http://git-wip-us.apache.org/repos/asf/lens/blob/9a678d8c/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index 494b08e..993aa4c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -90,9 +90,8 @@ public class StorageCandidateHQLContext extends DimHQLContext {
       String qualifiedStorageTable = getStorageCandidate().getStorageName();
       String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1);
       String where = getCubeQueryContext().getWhere(this, getCubeQueryContext().getAutoJoinCtx(),
-        getQueryAst().getWhereAST(),
         getCubeQueryContext().getAliasForTableName(getStorageCandidate().getBaseTable().getName()),
-        getCubeQueryContext().shouldReplaceDimFilterWithFactFilter(), storageTable, getDimsToQuery());
+        getCubeQueryContext().shouldReplaceDimFilterWithFactFilter(), getDimsToQuery());
       setWhere(where);
     }
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/9a678d8c/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
index 8661496..9e8f9bc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/JoinClause.java
@@ -50,6 +50,15 @@ public class JoinClause implements Comparable<JoinClause> {
     this.dimsInPath = dimsInPath;
   }
 
+  public TableRelationship getStarJoin(String table) {
+    for (Map.Entry<TableRelationship, JoinTree>  entry : getJoinTree().getSubtrees().entrySet()) {
+      if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) {
+        return entry.getKey();
+      }
+    }
+    return null;
+  }
+
   void initChainColumns() {
     for (List<TableRelationship> path : chain.values()) {
       for (TableRelationship edge : path) {

http://git-wip-us.apache.org/repos/asf/lens/blob/9a678d8c/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeQueryContextTest.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeQueryContextTest.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeQueryContextTest.java
new file mode 100644
index 0000000..19e4f44
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeQueryContextTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import static org.testng.Assert.assertEquals;
+
+import static com.google.common.collect.Lists.newArrayList;
+
+import java.util.List;
+
+import org.apache.lens.cube.parse.join.JoinClause;
+
+import org.mockito.Mockito;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Created on 28/08/17.
+ */
+public class CubeQueryContextTest {
+  private JoinClause jc;
+
+  @BeforeClass
+  public void intClass() {
+    jc = Mockito.mock(JoinClause.class);
+    Mockito.when(jc.getStarJoin(Mockito.anyString())).thenReturn(null);
+  }
+
+  @DataProvider
+  public Object[][] testCases() {
+    return new Object[][]{
+      {"testcube.x=1 and (testcube.dt=yesterday or (testcube.dt=today and testcube.pt=yesterday))",
+        newArrayList("((testcube.x) = 1)",
+          "(((testcube.dt) = yesterday) or (((testcube.dt) = today) and ((testcube.pt) = yesterday)))"), },
+      {"testcube.x=1 and (testcube.dt=yesterday and (testcube.dt=today and testcube.pt=yesterday))",
+        newArrayList("((testcube.x) = 1)", "((testcube.dt) = yesterday)",
+          "((testcube.dt) = today)", "((testcube.pt) = yesterday)"), },
+      {"testcube.x=1 and (testcube.dt = yesterday or "
+        + "(case when true and false then 1 else 0 end))",
+        newArrayList("((testcube.x) = 1)",
+          "(((testcube.dt) = yesterday) or case  when ( true  and  false ) then 1 else 0 end)"), },
+    };
+  }
+
+  @Test(dataProvider = "testCases")
+  public void testGetAllFilters(String expr, List<String> expected) throws Exception {
+    List<String> allFilters = newArrayList();
+    CubeQueryContext.getAllFilters(HQLParser.parseExpr(expr), "testcube", allFilters, jc, Maps.newHashMap());
+    assertEquals(allFilters, expected);
+  }
+}

[8/8] lens git commit: LENS-1468: Expressions in having clauses are not getting rewritten properly for join queries

Posted by ra...@apache.org.
LENS-1468: Expressions in having clauses are not getting rewritten properly for join queries


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2772efb2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2772efb2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2772efb2

Branch: refs/heads/current-release-line
Commit: 2772efb275742eb4c03da5e7635bdbcfab63cebc
Parents: 9a678d8
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Sun Sep 3 18:27:59 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:13:31 2017 +0800

----------------------------------------------------------------------
 .../lens/cube/parse/ExpressionResolver.java     | 55 +++++++++++++++-----
 .../cube/parse/StorageCandidateHQLContext.java  |  4 ++
 .../lens/cube/parse/UnionQueryWriter.java       |  8 +++
 .../lens/cube/parse/TestBaseCubeQueries.java    | 32 +++++++++++-
 .../resources/schema/cubes/base/basecube.xml    |  6 +++
 .../resources/schema/cubes/derived/der2.xml     |  2 +
 6 files changed, 92 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 822e25e..553468f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
 
 import org.antlr.runtime.CommonToken;
 
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import lombok.*;
 import lombok.extern.slf4j.Slf4j;
@@ -262,7 +263,7 @@ class ExpressionResolver implements ContextRewriter {
 
   @RequiredArgsConstructor
   @ToString
-  private static class PickedExpression {
+  static class PickedExpression {
     private final String srcAlias;
     private final ExprSpecContext pickedCtx;
     private transient ASTNode reWrittenAST = null;
@@ -286,11 +287,13 @@ class ExpressionResolver implements ContextRewriter {
 
   static class ExpressionResolverContext {
     @Getter
-    private Map<String, Set<ExpressionContext>> allExprsQueried = new HashMap<String, Set<ExpressionContext>>();
-    private Map<String, Set<PickedExpression>> pickedExpressions = new HashMap<String, Set<PickedExpression>>();
+    private Map<String, Set<ExpressionContext>> allExprsQueried = new HashMap<>();
+    private Map<String, Set<PickedExpression>> pickedExpressions = new HashMap<>();
+    @Getter
+    private Map<DimHQLContext, Map<String, Set<PickedExpression>>> pickedExpressionsPerCandidate = new HashMap<>();
     private Map<String, ASTNode> nonPickedExpressionsForCandidate = new HashMap<String, ASTNode>();
     private final CubeQueryContext cubeql;
-
+    private boolean replacedHavingExpressions = false;
     ExpressionResolverContext(CubeQueryContext cubeql) {
       this.cubeql = cubeql;
     }
@@ -411,7 +414,7 @@ class ExpressionResolver implements ContextRewriter {
         // Replace picked expressions in all the base trees
         replacePickedExpressions(sc);
       }
-
+      pickedExpressionsPerCandidate.put(sc, Maps.newHashMap(pickedExpressions));
       pickedExpressions.clear();
       nonPickedExpressionsForCandidate.clear();
 
@@ -430,16 +433,42 @@ class ExpressionResolver implements ContextRewriter {
       replaceAST(cubeql, queryAST.getJoinAST());
       replaceAST(cubeql, queryAST.getGroupByAST());
       // Resolve having expression for StorageCandidate
-      if (queryAST.getHavingAST() != null) {
-        replaceAST(cubeql, queryAST.getHavingAST());
-      } else if (cubeql.getHavingAST() != null) {
-        ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
-        replaceAST(cubeql, havingCopy);
-        queryAST.setHavingAST(havingCopy);
-      }
       replaceAST(cubeql, queryAST.getOrderByAST());
     }
-
+    public void replaceHavingExpressions() throws LensException {
+      replaceHavingExpressions(pickedExpressionsPerCandidate);
+    }
+    public void replaceHavingExpressions(
+      Map<DimHQLContext, Map<String, Set<PickedExpression>>> pickedExpressionsPerCandidate) throws LensException {
+      if (cubeql.getHavingAST() != null && !replacedHavingExpressions) {
+        HQLParser.bft(cubeql.getHavingAST(), visited -> {
+          ASTNode node1 = visited.getNode();
+          int childcount = node1.getChildCount();
+          for (int i = 0; i < childcount; i++) {
+            ASTNode current = (ASTNode) node1.getChild(i);
+            if (current.getToken().getType() == DOT) {
+              // This is for the case where column name is prefixed by table name
+              // or table alias
+              // For example 'select fact.id, dim2.id ...'
+              // Right child is the column name, left child.ident is table name
+              ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
+              ASTNode colIdent = (ASTNode) current.getChild(1);
+              String column = colIdent.getText().toLowerCase();
+
+              Optional<PickedExpression> exprOptional = pickedExpressionsPerCandidate.values().stream()
+                .filter(x -> x.containsKey(column)).map(x -> x.get(column)).flatMap(Collection::stream)
+                .filter(x -> x.srcAlias.equals(tabident.getText().toLowerCase())).findFirst();
+
+              if (exprOptional.isPresent()) {
+                PickedExpression expr = exprOptional.get();
+                node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql));
+              }
+            }
+          }
+        });
+        replacedHavingExpressions = true;
+      }
+    }
     private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws LensException {
       if (node == null) {
         return;

http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index 993aa4c..21cdb61 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -129,6 +129,10 @@ public class StorageCandidateHQLContext extends DimHQLContext {
   @Override
   protected void setMissingExpressions() throws LensException {
     setFrom(getFromTable());
+    if (getQueryAst().getHavingAST() != null) {
+      getStorageCandidate().getCubeQueryContext().getExprCtx().replaceHavingExpressions();
+      getQueryAst().setHavingAST(getCubeQueryContext().getHavingAST());
+    }
     String whereString = genWhereClauseWithDimPartitions(getWhere());
     StringBuilder whereStringBuilder = (whereString != null) ? new StringBuilder(whereString) :  new StringBuilder();
 

http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 9dc7ee6..4eb086b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.parse;
 
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
 
 import static org.apache.lens.cube.parse.HQLParser.*;
 
@@ -30,6 +31,7 @@ import java.util.*;
 
 import org.apache.lens.cube.metadata.*;
 import org.apache.lens.cube.metadata.join.JoinPath;
+import org.apache.lens.cube.parse.ExpressionResolver.PickedExpression;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -460,6 +462,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
     queryAst.setSelectAST(outerSelectAst);
 
     // Iterate over the StorageCandidates and add non projected having columns in inner select ASTs
+    Map<DimHQLContext, Map<String, Set<PickedExpression>>> pickedExpressionsPerCandidate = new HashMap<>();
+    for (CubeQueryContext cubeQueryContext : storageCandidates.stream()
+      .map(StorageCandidateHQLContext::getCubeQueryContext).collect(toSet())) {
+      pickedExpressionsPerCandidate.putAll(cubeQueryContext.getExprCtx().getPickedExpressionsPerCandidate());
+    }
+    cubeql.getExprCtx().replaceHavingExpressions(pickedExpressionsPerCandidate);
     for (StorageCandidateHQLContext sc : storageCandidates) {
       aliasDecider.setCounter(selectAliasCounter);
       processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc);

http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index cf29dff..35cb2b5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -835,13 +835,41 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
       }
     }
   }
+  @Test
+  public void testHavingOnTwoExpressions() throws Exception {
+    CubeQueryContext ctx1 = rewriteCtx("select dim1, msr2 from basecube where " + TWO_DAYS_RANGE
+      + "having effectivemsr2 > 0 and complexmsr12 > 10", conf);
+    CubeQueryContext ctx2 = rewriteCtx("select dim1, msr2 from basecube where " + TWO_DAYS_RANGE
+      + "having effectivemsr2 > 0 and complexmsr12 > 10", conf);
+    // shuffle join candidate order in ctx2
+    for (Candidate candidate : ctx2.getCandidates()) {
+      if (candidate instanceof JoinCandidate) {
+        JoinCandidate jc = (JoinCandidate) candidate;
+        List<Candidate> children = jc.getChildren();
+        Collections.reverse(children);
+      }
+    }
+    // toHQL outputs are tested in other functions, not testing here.
+
+    // test having clauses are same in both
+    String having1 = ctx1.toHQL().substring(ctx1.toHQL().indexOf("HAVING"));
+    String having2 = ctx2.toHQL().substring(ctx2.toHQL().indexOf("HAVING"));
+    assertEquals(having1, having2, "having1: " + having1 + "\nhaving2: " + having2);
+
+    // assert order of facts is differnet in to hqls
+    int ind11 = ctx1.toHQL().indexOf("c1_testfact1_base");
+    int ind21 = ctx2.toHQL().indexOf("c1_testfact1_base");
+
+    int ind12 = ctx1.toHQL().indexOf("c1_testfact2_base");
+    int ind22 = ctx2.toHQL().indexOf("c1_testfact2_base");
+
+    assertTrue((ind11 < ind21 && ind12 > ind22) || (ind11 > ind21 && ind12 < ind22));
+  }
 
   @Test
   public void testMultiFactQueryWithHaving() throws Exception {
 
     String hqlQuery, expected1, expected2;
-    String endSubString = "mq2 on mq1.dim1 <=> mq2.dim1 AND mq1.dim11 <=> mq2.dim11";
-    String joinSubString = "mq1 full outer join ";
 
     // only One having clause, that too answerable from one fact
     hqlQuery = rewrite("select dim1, dim11, msr12 from basecube where " + TWO_DAYS_RANGE

http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
index 6bb5eb9..6cc3201 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml
@@ -269,6 +269,12 @@
     <expression _type="double" name="flooredmsr12" display_string="Floored msr12" description="floored measure12">
       <expr_spec expr="floor(msr12)"/>
     </expression>
+    <expression _type="double" name="complexmsr12" display_string="Floored msr12" description="floored measure12">
+      <expr_spec expr="floor(msr12)+0"/>
+    </expression>
+    <expression _type="double" name="effectivemsr2" display_string="effective msr12" description="effective measure2">
+      <expr_spec expr="msr2 + msr21 + msr22"/>
+    </expression>
     <expression _type="String" name="cityandstate" display_string="City and State"
                 description="city and state together">
       <expr_spec expr="concat(cityname, &quot;:&quot;, statename_cube)"/>

http://git-wip-us.apache.org/repos/asf/lens/blob/2772efb2/lens-cube/src/test/resources/schema/cubes/derived/der2.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/derived/der2.xml b/lens-cube/src/test/resources/schema/cubes/derived/der2.xml
index 337e7f4..213fe07 100644
--- a/lens-cube/src/test/resources/schema/cubes/derived/der2.xml
+++ b/lens-cube/src/test/resources/schema/cubes/derived/der2.xml
@@ -33,6 +33,8 @@
   <measure_names>
     <measure_name>directmsr</measure_name>
     <measure_name>msr2</measure_name>
+    <measure_name>msr21</measure_name>
+    <measure_name>msr22</measure_name>
     <measure_name>msr12</measure_name>
     <measure_name>msr14</measure_name>
     <measure_name>msr13</measure_name>

[6/8] lens git commit: LENS-1465: Sync queries blocking threads

Posted by ra...@apache.org.
LENS-1465: Sync queries blocking threads


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/13cbc811
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/13cbc811
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/13cbc811

Branch: refs/heads/current-release-line
Commit: 13cbc811342378d8fb39f4d03be08145fbc4f986
Parents: 1d66797
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Aug 30 16:10:13 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:13:09 2017 +0800

----------------------------------------------------------------------
 .../org/apache/lens/server/api/query/QueryContext.java   | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/13cbc811/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 466fb25..8176f03 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -198,7 +198,7 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
   @Getter
   @Setter
   private transient Future queryLauncher;
-  transient List<QueryDriverStatusUpdateListener> driverStatusUpdateListeners = Lists.newArrayList();
+  transient List<QueryDriverStatusUpdateListener> driverStatusUpdateListeners = Lists.newCopyOnWriteArrayList();
   @Getter
   @Setter
   List<FailedAttempt> failedAttempts = Lists.newArrayList();
@@ -572,10 +572,9 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
       getDriverStatus().setStatusMessage("Query " + getQueryHandleString() + " " + state.name().toLowerCase());
     }
     getDriverStatus().setState(state);
-    synchronized (this) {
-      for (QueryDriverStatusUpdateListener listener : this.driverStatusUpdateListeners) {
-        listener.onDriverStatusUpdated(getQueryHandle(), getDriverStatus());
-      }
+
+    for (QueryDriverStatusUpdateListener listener : this.driverStatusUpdateListeners) {
+      listener.onDriverStatusUpdated(getQueryHandle(), getDriverStatus());
     }
   }
 
@@ -588,7 +587,7 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
   }
 
 
-  public synchronized void registerStatusUpdateListener(QueryDriverStatusUpdateListener driverStatusUpdateListener) {
+  public void registerStatusUpdateListener(QueryDriverStatusUpdateListener driverStatusUpdateListener) {
     this.driverStatusUpdateListeners.add(driverStatusUpdateListener);
   }

[2/8] lens git commit: LENS-1473: Cubevirtualfact table relative start time has incorrect config field name

Posted by ra...@apache.org.
LENS-1473: Cubevirtualfact table relative start time has incorrect config field name


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/717214cf
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/717214cf
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/717214cf

Branch: refs/heads/current-release-line
Commit: 717214cf9f17337fa6769f03afb3391b39b014ee
Parents: 358efef
Author: Rajitha R <ra...@gmail.com>
Authored: Wed Sep 6 18:33:38 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:12:08 2017 +0800

----------------------------------------------------------------------
 .../java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/717214cf/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
index ce2a1a6..8b55985 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -147,7 +147,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable
   }
 
   public Date getRelativeStartTime() {
-    String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME);
+    String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME);
     Date relativeDate = null;
     if (StringUtils.isNotBlank(relativeStartTime)) {
       relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, true, true);

[4/8] lens git commit: LENS-1475 : Override equals(Object obj) and hashcode() methods of SegmentationCandidate

Posted by ra...@apache.org.
LENS-1475 : Override equals(Object obj) and hashcode() methods of SegmentationCandidate


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0f58445e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0f58445e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0f58445e

Branch: refs/heads/current-release-line
Commit: 0f58445e0e2bf4262a624f006c678786a00485d5
Parents: ae6bda3
Author: Sushil Mohanty <su...@gmail.com>
Authored: Thu Sep 14 12:42:57 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:12:29 2017 +0800

----------------------------------------------------------------------
 .../lens/cube/parse/SegmentationCandidate.java  | 21 ++++++++++++++++++++
 1 file changed, 21 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/0f58445e/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index 055d6ef..9378009 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -404,4 +404,25 @@ public class SegmentationCandidate implements Candidate {
     return cubeQueryContextMap.entrySet().stream().filter(entry -> entry.getValue().getPickedCandidate() == null)
       .collect(toMap(Map.Entry::getKey, entry -> entry.getValue().getStoragePruningMsgs()));
   }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (super.equals(obj)) {
+      return true;
+    }
+
+    if (obj == null || !(obj instanceof SegmentationCandidate)) {
+      return false;
+    }
+
+    SegmentationCandidate segmantationCandidate = (SegmentationCandidate) obj;
+    return (segmantationCandidate.segmentation.getSegments().equals(this.segmentation.getSegments())
+        && segmantationCandidate.segmentation.getBaseCube().equals(this.segmentation.getBaseCube()));
+  }
+
+  @Override
+  public int hashCode() {
+    return segmentation.hashCode();
+  }
+
 }

[5/8] lens git commit: LENS-1464: One or two queued queries failing with ConcurrentModificationException on restart

Posted by ra...@apache.org.
LENS-1464: One or two queued queries failing with ConcurrentModificationException on restart


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1d667975
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1d667975
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1d667975

Branch: refs/heads/current-release-line
Commit: 1d6679754240f2980e9bffd278a3ee04ac98db02
Parents: 0f58445
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Aug 30 15:59:23 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:12:56 2017 +0800

----------------------------------------------------------------------
 lens-server/pom.xml                             |  4 +--
 .../server/query/QueryExecutionServiceImpl.java | 32 +++++++++-----------
 2 files changed, 16 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/1d667975/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 2065f04..f56f7bc 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -402,8 +402,8 @@
           </execution>
         </executions>
         <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
+          <source>1.8</source>
+          <target>1.8</target>
           <configFile>enunciate.xml</configFile>
         </configuration>
       </plugin>

http://git-wip-us.apache.org/repos/asf/lens/blob/1d667975/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index e70d290..4d141e5 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -3476,27 +3476,23 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
 
     // Add resources if either they haven't been marked as added on the session, or if Hive driver says they need
     // to be added to the corresponding hive driver
-    if (!hiveDriver.areDBResourcesAddedForSession(sessionIdentifier, ctx.getDatabase())) {
-      Collection<ResourceEntry> dbResources = session.getDBResources(ctx.getDatabase());
-
-      if (CollectionUtils.isNotEmpty(dbResources)) {
-        log.info("Proceeding to add resources for DB {} for query {} resources: {}", session.getCurrentDatabase(),
-          ctx.getLogHandle(), dbResources);
-
-        List<ResourceEntry> failedDBResources = addResources(dbResources, sessionHandle, hiveDriver);
-        Iterator<ResourceEntry> itr = dbResources.iterator();
-        while (itr.hasNext()) {
-          ResourceEntry res = itr.next();
-          if (!failedDBResources.contains(res)) {
-            itr.remove();
-          }
+    synchronized (session) {
+      if (!hiveDriver.areDBResourcesAddedForSession(sessionIdentifier, ctx.getDatabase())) {
+        Collection<ResourceEntry> dbResources = session.getDBResources(ctx.getDatabase());
+
+        if (CollectionUtils.isNotEmpty(dbResources)) {
+          log.info("Proceeding to add resources for DB {} for query {} resources: {}", session.getCurrentDatabase(),
+            ctx.getLogHandle(), dbResources);
+
+          List<ResourceEntry> failedDBResources = addResources(dbResources, sessionHandle, hiveDriver);
+          dbResources.removeIf(res -> !failedDBResources.contains(res));
+        } else {
+          log.info("No need to add DB resources for session: {} db= {}", sessionIdentifier,
+            session.getCurrentDatabase());
         }
-      } else {
-        log.info("No need to add DB resources for session: {} db= {}", sessionIdentifier, session.getCurrentDatabase());
+        hiveDriver.setResourcesAddedForSession(sessionIdentifier, ctx.getDatabase());
       }
-      hiveDriver.setResourcesAddedForSession(sessionIdentifier, ctx.getDatabase());
     }
-
     // Get pending session resources which needed to be added for this database
     Collection<ResourceEntry> pendingResources =
       session.getPendingSessionResourcesForDatabase(ctx.getDatabase());

[3/8] lens git commit: LENS-1474 : Fix NPE in ExpressionResolver for SegmentationCandidate

Posted by ra...@apache.org.
LENS-1474 : Fix NPE in ExpressionResolver for SegmentationCandidate


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ae6bda33
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ae6bda33
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ae6bda33

Branch: refs/heads/current-release-line
Commit: ae6bda33ed501e15dceeecea83c86adf189e4453
Parents: 717214c
Author: Rajitha R <ra...@gmail.com>
Authored: Mon Sep 11 09:53:21 2017 +0530
Committer: rajub <ra...@lazada.com>
Committed: Thu Oct 5 11:12:15 2017 +0800

----------------------------------------------------------------------
 .../apache/lens/cube/parse/ExpressionResolver.java | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/ae6bda33/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 8906fae..822e25e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -314,12 +314,14 @@ class ExpressionResolver implements ContextRewriter {
     }
 
     ExpressionContext getExpressionContext(String expr, String alias) {
-      for (ExpressionContext ec : allExprsQueried.get(expr)) {
-        if (ec.getSrcAlias().equals(alias)) {
-          return ec;
+      if (allExprsQueried.get(expr) != null) {
+        for (ExpressionContext ec : allExprsQueried.get(expr)) {
+          if (ec.getSrcAlias().equals(alias)) {
+            return ec;
+          }
         }
       }
-      throw new IllegalArgumentException("no expression available for " + expr + " alias:" + alias);
+      return null;
     }
 
     boolean hasMeasures(String expr, CubeInterface cube) {
@@ -511,8 +513,11 @@ class ExpressionResolver implements ContextRewriter {
       expressionContexts.add(expressionContext);
       for (StorageCandidate sc : scSet) {
         storageTableNames.add(sc.getStorageTable());
-        expressionContexts.add(sc.getCubeQueryContext().getExprCtx()
-          .getExpressionContext(expressionContext.getExprCol().getName(), expressionContext.getSrcAlias()));
+        ExpressionContext ex = sc.getCubeQueryContext().getExprCtx().getExpressionContext(expressionContext.
+            getExprCol().getName(), expressionContext.getSrcAlias());
+        if (ex != null) {
+          expressionContexts.add(ex);
+        }
       }
       for (ExpressionContext ec : expressionContexts) {
         for (CandidateTable table : ec.evaluableExpressions.keySet()) {