You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/08/11 14:21:11 UTC
[48/50] [abbrv] incubator-lens git commit: LENS-711 : Allow chain ref
columns to have multiple chain destinations
LENS-711 : Allow chain ref columns to have multiple chain destinations
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/bab05e4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/bab05e4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/bab05e4f
Branch: refs/heads/current-release-line
Commit: bab05e4fa42cacdf7d605bc61a7261d39222e974
Parents: 9abfef1
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Aug 11 13:36:31 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Aug 11 13:36:31 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 4 +-
.../apache/lens/cli/TestLensCubeCommands.java | 54 +++++-
lens-cli/src/test/resources/sample-cube.xml | 33 ++++
lens-cli/src/test/resources/test-detail.xml | 32 ++++
.../lens/cube/metadata/CubeMetastoreClient.java | 6 +-
.../lens/cube/metadata/MetastoreUtil.java | 2 +-
.../cube/metadata/ReferencedDimAtrribute.java | 58 +++++--
.../lens/cube/parse/CandidateTableResolver.java | 3 +-
.../cube/parse/DenormalizationResolver.java | 42 +++--
.../apache/lens/cube/parse/FieldValidator.java | 5 +-
.../apache/lens/cube/parse/JoinResolver.java | 10 +-
.../cube/metadata/TestCubeMetastoreClient.java | 9 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 169 ++++++++++++-------
.../lens/cube/parse/TestBaseCubeQueries.java | 16 +-
.../cube/parse/TestDenormalizationResolver.java | 32 +++-
.../parse/TestTimeRangeWriterWithQuery.java | 104 ++++++------
lens-examples/src/main/resources/sales-cube.xml | 1 +
.../apache/lens/server/metastore/JAXBUtils.java | 37 ++--
.../server/metastore/TestMetastoreService.java | 17 +-
19 files changed, 442 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index e6cb87d..02ea2d1 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -403,13 +403,13 @@
which the attribute is refering to.
For ex : userid refers user.id, xuser.id, yuser.id, zuser.id.
- Alternately, ref_spec could be a chained column specifed with chain name and column name.
+ Alternately, ref_spec could be list of chained columns each specifed with chain name and column name.
</xs:documentation>
</xs:annotation>
<xs:complexType>
<xs:choice maxOccurs="1" minOccurs="1">
<xs:element type="x_table_references" name="table_references" maxOccurs="1" minOccurs="1"/>
- <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="1" minOccurs="1"/>
+ <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="unbounded" minOccurs="1"/>
</xs:choice>
</xs:complexType>
</xs:element>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
index 9912eef..8de61e6 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
@@ -24,8 +24,9 @@ import java.io.*;
import java.net.URL;
import java.util.Arrays;
-import org.apache.lens.api.metastore.XJoinChains;
+import org.apache.lens.api.metastore.*;
import org.apache.lens.cli.commands.LensCubeCommands;
+import org.apache.lens.cli.commands.LensDimensionCommands;
import org.apache.lens.cli.table.XJoinChainTable;
import org.apache.lens.client.LensClient;
@@ -50,6 +51,12 @@ public class TestLensCubeCommands extends LensCliApplicationTest {
@Test
public void testCubeCommands() throws Exception {
LensClient client = new LensClient();
+ LensDimensionCommands dimensionCommand = new LensDimensionCommands();
+ dimensionCommand.setClient(client);
+ dimensionCommand.createDimension(new File(
+ TestLensCubeCommands.class.getClassLoader().getResource("test-detail.xml").toURI()));
+ dimensionCommand.createDimension(new File(
+ TestLensCubeCommands.class.getClassLoader().getResource("test-dimension.xml").toURI()));
LensCubeCommands command = new LensCubeCommands();
command.setClient(client);
LOG.debug("Starting to test cube commands");
@@ -72,18 +79,57 @@ public class TestLensCubeCommands extends LensCliApplicationTest {
}
cubeList = command.showCubes();
assertFalse(cubeList.contains("sample_cube"));
+ dimensionCommand.dropDimension("test_detail");
+ dimensionCommand.dropDimension("test_dim");
}
private void testJoinChains(LensCubeCommands command) {
String joinChains = command.showJoinChains("sample_cube");
- assertEquals(joinChains, new XJoinChainTable(new XJoinChains()).toString());
+ XJoinChains chains = new XJoinChains();
+ XJoinChain chain1 = new XJoinChain();
+ chain1.setPaths(new XJoinPaths());
+ XJoinPath path = new XJoinPath();
+ path.setEdges(new XJoinEdges());
+ XJoinEdge edge1 = new XJoinEdge();
+ XTableReference ref1 = new XTableReference();
+ ref1.setTable("sample_cube");
+ ref1.setColumn("dim2");
+ XTableReference ref2 = new XTableReference();
+ ref2.setTable("test_detail");
+ ref2.setColumn("id");
+ edge1.setFrom(ref1);
+ edge1.setTo(ref2);
+ path.getEdges().getEdge().add(edge1);
+ chain1.setName("testdetailchain");
+ chain1.getPaths().getPath().add(path);
+ chain1.setDestTable("test_detail");
+ chains.getJoinChain().add(chain1);
+ XJoinChain chain2 = new XJoinChain();
+ chain2.setPaths(new XJoinPaths());
+ XJoinPath path2 = new XJoinPath();
+ path2.setEdges(new XJoinEdges());
+ XJoinEdge edge2 = new XJoinEdge();
+ XTableReference ref3 = new XTableReference();
+ ref3.setTable("sample_cube");
+ ref3.setColumn("dim1");
+ XTableReference ref4 = new XTableReference();
+ ref4.setTable("test_dim");
+ ref4.setColumn("id");
+ edge2.setFrom(ref3);
+ edge2.setTo(ref4);
+ path2.getEdges().getEdge().add(edge2);
+ chain2.setName("testdimchain");
+ chain2.getPaths().getPath().add(path2);
+ chain2.setDestTable("test_dim");
+ chains.getJoinChain().add(chain2);
+ assertEquals(joinChains, new XJoinChainTable(chains).toString());
}
private void testFields(LensCubeCommands command) {
String fields = command.showQueryableFields("sample_cube", true);
for (String field : Arrays
- .asList("dim1", "dim2", "dim3", "measure1", "measure2", "measure3", "measure4", "expr_msr5")) {
- assertTrue(fields.contains(field));
+ .asList("dim1", "dim2", "dim3", "dimdetail", "measure1", "measure2", "measure3", "measure4", "expr_msr5")) {
+ assertTrue(fields.contains(field), fields + " do not contain " + field);
}
assertTrue(fields.contains("measure3 + measure4 + 0.01"));
assertTrue(fields.replace("measure3 + measure4 + 0.01", "blah").contains("measure3 + measure4"));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cli/src/test/resources/sample-cube.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml
index f0eba57..d72d279 100644
--- a/lens-cli/src/test/resources/sample-cube.xml
+++ b/lens-cli/src/test/resources/sample-cube.xml
@@ -41,6 +41,13 @@
</table_references>
</ref_spec>
</dim_attribute>
+ <dim_attribute name="dimDetail" type="string" description="City name to which the customer belongs"
+ display_string="Customer City">
+ <ref_spec>
+ <chain_ref_column chain_name="testdimchain" ref_col="detail" />
+ <chain_ref_column chain_name="testdetailchain" ref_col="name" />
+ </ref_spec>
+ </dim_attribute>
</dim_attributes>
<expressions>
<expression name="expr_msr5" type="DOUBLE">
@@ -48,4 +55,30 @@
<expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/>
</expression>
</expressions>
+ <join_chains>
+ <join_chain name="testdimchain">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="sample_cube" column="dim1" />
+ <to table="test_dim" column="id" />
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ <join_chain name="testdetailchain">
+ <paths>
+ <path>
+ <edges>
+ <edge>
+ <from table="sample_cube" column="dim2" />
+ <to table="test_detail" column="id" />
+ </edge>
+ </edges>
+ </path>
+ </paths>
+ </join_chain>
+ </join_chains>
</x_base_cube>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cli/src/test/resources/test-detail.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/test-detail.xml b/lens-cli/src/test/resources/test-detail.xml
new file mode 100644
index 0000000..bb54354
--- /dev/null
+++ b/lens-cli/src/test/resources/test-detail.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_dimension name="test_detail" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <attributes>
+ <dim_attribute name="id" type="INT" />
+ <dim_attribute name="name" type="STRING" />
+ </attributes>
+
+ <properties>
+ <property name="dimension.test_dim.timed.dimension" value="dt" />
+ </properties>
+</x_dimension>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index f0a0dfe..daf7434 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -1257,8 +1257,10 @@ public class CubeMetastoreClient {
/** extract from table properties */
public List<String> getTimePartColNamesOfTable(Table table) {
- List<String> ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS),
- ","));
+ List<String> ret = null;
+ if (table.getParameters().containsKey(MetastoreConstants.TIME_PART_COLUMNS)) {
+ ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS), ","));
+ }
return ret == null ? new ArrayList<String>() : ret;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 8fbfe63..59a30a9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -39,7 +39,7 @@ public class MetastoreUtil {
}
public static final String getStorageTableName(String cubeTableName, String storagePrefix) {
- return storagePrefix + cubeTableName;
+ return (storagePrefix + cubeTableName).toLowerCase();
}
public static String getStorageClassKey(String name) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
index 742c6a0..a60281b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
@@ -23,6 +23,7 @@ import java.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
@@ -30,15 +31,20 @@ import lombok.ToString;
@EqualsAndHashCode(callSuper = true)
@ToString(callSuper = true)
public class ReferencedDimAtrribute extends BaseDimAttribute {
+ private static final char CHAIN_REF_COL_SEPARATOR = ',';
+
@Getter
- private final List<TableReference> references = new ArrayList<TableReference>();
+ private final List<TableReference> references = new ArrayList<>();
// boolean whether to say the key is only a denormalized variable kept or can
// be used in join resolution as well
@Getter private Boolean isJoinKey = true;
- @Getter
- private String chainName = null;
- @Getter
- private String refColumn = null;
+ @Getter private List<ChainRefCol> chainRefColumns = new ArrayList<>();
+
+ @Data
+ public static class ChainRefCol {
+ private final String chainName;
+ private final String refColumn;
+ }
public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference) {
this(column, displayString, reference, null, null, null);
@@ -89,9 +95,15 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
public ReferencedDimAtrribute(FieldSchema column, String displayString, String chainName, String refColumn,
Date startTime, Date endTime, Double cost, Long numOfDistinctValues) {
+ this(column, displayString,
+ Collections.singletonList(new ChainRefCol(chainName.toLowerCase(), refColumn.toLowerCase())), startTime, endTime,
+ cost, numOfDistinctValues);
+ }
+
+ public ReferencedDimAtrribute(FieldSchema column, String displayString, List<ChainRefCol> chainRefCols,
+ Date startTime, Date endTime, Double cost, Long numOfDistinctValues) {
super(column, displayString, startTime, endTime, cost, numOfDistinctValues);
- this.chainName = chainName.toLowerCase();
- this.refColumn = refColumn.toLowerCase();
+ chainRefColumns.addAll(chainRefCols);
this.isJoinKey = false;
}
@@ -110,9 +122,21 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
@Override
public void addProperties(Map<String, String> props) {
super.addProperties(props);
- if (chainName != null) {
- props.put(MetastoreUtil.getDimRefChainNameKey(getName()), chainName);
- props.put(MetastoreUtil.getDimRefChainColumnKey(getName()), refColumn);
+ if (!chainRefColumns.isEmpty()) {
+ StringBuilder chainNamesValue = new StringBuilder();
+ StringBuilder refColsValue = new StringBuilder();
+ Iterator<ChainRefCol> iter = chainRefColumns.iterator();
+ // Add the first without appending separator
+ ChainRefCol chainRefCol = iter.next();
+ chainNamesValue.append(chainRefCol.getChainName());
+ refColsValue.append(chainRefCol.getRefColumn());
+ while (iter.hasNext()) {
+ chainRefCol = iter.next();
+ chainNamesValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getChainName());
+ refColsValue.append(CHAIN_REF_COL_SEPARATOR).append(chainRefCol.getRefColumn());
+ }
+ props.put(MetastoreUtil.getDimRefChainNameKey(getName()), chainNamesValue.toString());
+ props.put(MetastoreUtil.getDimRefChainColumnKey(getName()), refColsValue.toString());
} else {
props.put(MetastoreUtil.getDimensionSrcReferenceKey(getName()),
MetastoreUtil.getReferencesString(references));
@@ -128,10 +152,14 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
*/
public ReferencedDimAtrribute(String name, Map<String, String> props) {
super(name, props);
- String chName = props.get(MetastoreUtil.getDimRefChainNameKey(getName()));
- if (!StringUtils.isBlank(chName)) {
- this.chainName = chName;
- this.refColumn = props.get(MetastoreUtil.getDimRefChainColumnKey(getName()));
+ String chNamesStr = props.get(MetastoreUtil.getDimRefChainNameKey(getName()));
+ if (!StringUtils.isBlank(chNamesStr)) {
+ String refColsStr = props.get(MetastoreUtil.getDimRefChainColumnKey(getName()));
+ String[] chainNames = StringUtils.split(chNamesStr, ",");
+ String[] refCols = StringUtils.split(refColsStr, ",");
+ for (int i = 0; i < chainNames.length; i++) {
+ chainRefColumns.add(new ChainRefCol(chainNames[i], refCols[i]));
+ }
this.isJoinKey = false;
} else {
String refListStr = props.get(MetastoreUtil.getDimensionSrcReferenceKey(getName()));
@@ -152,6 +180,6 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
* @return true/false
*/
public boolean isChainedColumn() {
- return chainName != null;
+ return !chainRefColumns.isEmpty();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 50a4d53..3e73d02 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -243,8 +243,7 @@ class CandidateTableResolver implements ContextRewriter {
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(cubeql.getCubeTbls().get(chain.getName()));
if (!checkForColumnExists(cfact, chain.getSourceColumns())) {
// check if chain is optional or not
- if (optdim == null || optdim.isRequiredInJoinChain
- || (optdim != null && optdim.requiredForCandidates.contains(cfact))) {
+ if (optdim == null) {
log.info("Not considering fact table:{} as columns {} are not available", cfact,
chain.getSourceColumns());
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(chain.getSourceColumns()));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 4a95d5a..517e8fc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
import java.util.*;
import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
@@ -55,20 +56,21 @@ public class DenormalizationResolver implements ContextRewriter {
public static class ReferencedQueriedColumn {
ReferencedDimAtrribute col;
AbstractCubeTable srcTable;
- transient List<TableReference> references;
+ transient List<TableReference> references = new ArrayList<>();
+ transient List<ChainRefCol> chainRefCols = new ArrayList<>();
ReferencedQueriedColumn(ReferencedDimAtrribute col, AbstractCubeTable srcTable) {
this.col = col;
this.srcTable = srcTable;
- references = new ArrayList<TableReference>();
references.addAll(col.getReferences());
+ chainRefCols.addAll(col.getChainRefColumns());
}
}
@ToString
public static class PickedReference {
- ReferencedDimAtrribute ref;
TableReference reference;
+ ChainRefCol chainRef;
String srcAlias;
String pickedFor;
@@ -78,22 +80,22 @@ public class DenormalizationResolver implements ContextRewriter {
this.pickedFor = pickedFor;
}
- PickedReference(ReferencedDimAtrribute ref, String srcAlias, String pickedFor) {
+ PickedReference(ChainRefCol chainRef, String srcAlias, String pickedFor) {
this.srcAlias = srcAlias;
- this.ref = ref;
+ this.chainRef = chainRef;
this.pickedFor = pickedFor;
}
String getDestTable() {
- if (ref != null && ref.isChainedColumn()) {
- return ref.getChainName();
+ if (chainRef != null) {
+ return chainRef.getChainName();
}
return reference.getDestTable();
}
String getRefColumn() {
- if (ref != null && ref.isChainedColumn()) {
- return ref.getRefColumn();
+ if (chainRef != null) {
+ return chainRef.getRefColumn();
}
return reference.getDestColumn();
}
@@ -150,9 +152,10 @@ public class DenormalizationResolver implements ContextRewriter {
refCols.add(refer);
// Add to optional tables
if (refer.col.isChainedColumn()) {
- cubeql.addOptionalDimTable(refer.col.getChainName(), table, false, refer.col.getName(), true,
- refer.col.getRefColumn());
-
+ for (ChainRefCol refCol : refer.col.getChainRefColumns()) {
+ cubeql.addOptionalDimTable(refCol.getChainName(), table, false, refer.col.getName(), true,
+ refCol.getRefColumn());
+ }
} else {
for (TableReference reference : refer.col.getReferences()) {
cubeql.addOptionalDimTable(reference.getDestTable(), table, false, refer.col.getName(), true,
@@ -255,8 +258,21 @@ public class DenormalizationResolver implements ContextRewriter {
addPickedReference(refered.col.getName(), picked);
pickedRefs.add(picked);
} else {
+ Iterator<ChainRefCol> iter = refered.chainRefCols.iterator();
+ while (iter.hasNext()) {
+ // remove unreachable references
+ ChainRefCol reference = iter.next();
+ if (!cubeql.getAutoJoinCtx().isReachableDim(
+ (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) {
+ iter.remove();
+ }
+ }
+ if (refered.chainRefCols.isEmpty()) {
+ throw new SemanticException("No chain reference column available for " + refered);
+ }
PickedReference picked =
- new PickedReference(refered.col, cubeql.getAliasForTableName(refered.srcTable.getName()), tbl);
+ new PickedReference(refered.chainRefCols.iterator().next(),
+ cubeql.getAliasForTableName(refered.srcTable.getName()), tbl);
addPickedReference(refered.col.getName(), picked);
pickedRefs.add(picked);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
index 03377dd..1a1232b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
@@ -25,6 +25,7 @@ import org.apache.lens.cube.error.FieldsCannotBeQueriedTogetherException;
import org.apache.lens.cube.metadata.CubeInterface;
import org.apache.lens.cube.metadata.DerivedCube;
import org.apache.lens.cube.metadata.ReferencedDimAtrribute;
+import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -168,7 +169,9 @@ public class FieldValidator implements ContextRewriter {
if (cube.getDimAttributeByName(colName) instanceof ReferencedDimAtrribute
&& ((ReferencedDimAtrribute) cube.getDimAttributeByName(colName)).isChainedColumn()) {
ReferencedDimAtrribute rdim = (ReferencedDimAtrribute) cube.getDimAttributeByName(colName);
- chainSourceColumns.addAll(cube.getChainByName(rdim.getChainName()).getSourceColumns());
+ for (ChainRefCol refCol : rdim.getChainRefColumns()) {
+ chainSourceColumns.addAll(cube.getChainByName(refCol.getChainName()).getSourceColumns());
+ }
} else {
// This is a dim attribute, needs to be validated
dimAttributes.add(colName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index 7d04d19..a6e9340 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -890,9 +890,17 @@ class JoinResolver implements ContextRewriter {
public boolean isReachableDim(Dimension dim) {
Aliased<Dimension> aliased = Aliased.create(dim);
- return allPaths.containsKey(aliased) && !allPaths.get(aliased).isEmpty();
+ return isReachableDim(aliased);
+ }
+
+ public boolean isReachableDim(Dimension dim, String alias) {
+ Aliased<Dimension> aliased = Aliased.create(dim, alias);
+ return isReachableDim(aliased);
}
+ private boolean isReachableDim(Aliased<Dimension> aliased) {
+ return allPaths.containsKey(aliased) && !allPaths.get(aliased).isEmpty();
+ }
}
static String getJoinTypeStr(JoinType joinType) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 077396e..5d66039 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -24,6 +24,7 @@ import static org.testng.Assert.assertEquals;
import java.util.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
@@ -597,10 +598,10 @@ public class TestCubeMetastoreClient {
assertEquals(citychain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.cityid");
assertEquals(citychain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id");
Assert.assertNotNull(cube2.getDimAttributeByName("zipcityname"));
- assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getChainName(),
- "cityfromzip");
- assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getRefColumn(),
- "name");
+ ChainRefCol zipCityChain = ((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname"))
+ .getChainRefColumns().get(0);
+ assertEquals(zipCityChain.getChainName(), "cityfromzip");
+ assertEquals(zipCityChain.getRefColumn(), "name");
client.createDerivedCube(CUBE_NAME, DERIVED_CUBE_NAME, measures, dimensions, new HashMap<String, String>(), 0L);
Assert.assertTrue(client.tableExists(DERIVED_CUBE_NAME));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 1d4e7dd..ae8984f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -30,6 +30,7 @@ import java.util.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
@@ -636,6 +637,7 @@ public class CubeTestSetup {
// not creating test_time_dim_hour_id2 ref dim attribute to avoid the reference in schema graph for other paths
// the column is only defined in chain
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id2", "int", "ref dim")));
+ cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id2", "int", "ref dim")));
cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("testdim3id", "int", "direct id to testdim3"),
"Timedim reference", new TableReference("testdim3", "id"), null, null, null));
@@ -644,16 +646,26 @@ public class CubeTestSetup {
references.add(new TableReference("hourdim", "full_hour"));
cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("test_time_dim", "date", "ref dim"),
"Timedim full date", references, null, null, null, false));
+ List<ChainRefCol> chainRefs = new ArrayList<>();
+ chainRefs.add(new ChainRefCol("timehourchain", "full_hour"));
+ chainRefs.add(new ChainRefCol("timedatechain", "full_date"));
cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("test_time_dim2", "date", "chained dim"),
- "Timedim full date", "timechain", "full_hour", null, null, null));
+ "Timedim full date", chainRefs, null, null, null, null));
Set<JoinChain> joinchains = new HashSet<JoinChain>();
- JoinChain timeChain = new JoinChain("timechain", "time chain", "time dim thru dim");
+ JoinChain timeHourChain = new JoinChain("timehourchain", "time chain", "time dim thru hour dim");
List<TableReference> paths = new ArrayList<TableReference>();
paths.add(new TableReference("testcube", "test_time_dim_hour_id2"));
paths.add(new TableReference("hourdim", "id"));
- timeChain.addPath(paths);
- joinchains.add(timeChain);
+ timeHourChain.addPath(paths);
+ joinchains.add(timeHourChain);
+
+ JoinChain timeDateChain = new JoinChain("timedatechain", "time chain", "time dim thru date dim");
+ paths = new ArrayList<TableReference>();
+ paths.add(new TableReference("testcube", "test_time_dim_day_id2"));
+ paths.add(new TableReference("daydim", "id"));
+ timeDateChain.addPath(paths);
+ joinchains.add(timeDateChain);
joinchains.add(new JoinChain("cubeState", "cube-state", "state thru cube") {
{
addPath(new ArrayList<TableReference>() {
@@ -887,6 +899,7 @@ public class CubeTestSetup {
dimensions.add("dim1");
dimensions.add("location");
dimensions.add("d_time");
+ dimensions.add("test_time_dim");
client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME3, measures, dimensions, derivedProperties, 20L);
// create base cube facts
@@ -949,6 +962,7 @@ public class CubeTestSetup {
factColumns.add(new FieldSchema("stateid", "int", "city id"));
factColumns.add(new FieldSchema("dim1", "string", "base dim"));
factColumns.add(new FieldSchema("dim11", "string", "base dim"));
+ factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id"));
// create cube fact
client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
@@ -1094,8 +1108,8 @@ public class CubeTestSetup {
factColumns.add(new FieldSchema("zipcode", "int", "zip"));
factColumns.add(new FieldSchema("cityid", "int", "city id"));
factColumns.add(new FieldSchema("stateid", "int", "city id"));
- factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id"));
- factColumns.add(new FieldSchema("test_time_dim_hour_id2", "int", "time id"));
+ factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id"));
+ factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id"));
factColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity"));
Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
@@ -1149,67 +1163,27 @@ public class CubeTestSetup {
// create cube fact
client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
factValidityProperties, storageTables);
-
- CubeFactTable fact = client.getFactTable(factName);
-
- Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
- Storage.getPrefix(c4)));
- table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd"),
- StoreAllPartitionTimeline.class.getCanonicalName());
- table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd2"),
- StoreAllPartitionTimeline.class.getCanonicalName());
- client.pushHiveTable(table);
- // Add all hourly partitions for two days
- Calendar cal = Calendar.getInstance();
- cal.setTime(TWODAYS_BACK);
- Date temp = cal.getTime();
- List<StoragePartitionDesc> storagePartitionDescs = Lists.newArrayList();
- List<String> partitions = Lists.newArrayList();
- StoreAllPartitionTimeline ttdStoreAll =
- new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
- "ttd");
- StoreAllPartitionTimeline ttd2StoreAll =
- new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
- "ttd2");
- while (!(temp.after(NOW))) {
- Map<String, Date> timeParts = new HashMap<String, Date>();
- timeParts.put("ttd", temp);
- timeParts.put("ttd2", temp);
- TimePartition tp = TimePartition.of(HOURLY, temp);
- ttdStoreAll.add(tp);
- ttd2StoreAll.add(tp);
- partitions.add(HOURLY.format().format(temp));
- StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
- storagePartitionDescs.add(sPartSpec);
- cal.add(Calendar.HOUR_OF_DAY, 1);
- temp = cal.getTime();
- }
- client.addPartitions(storagePartitionDescs, c4);
+ client.getTimelines(factName, c1, null, null);
+ client.getTimelines(factName, c4, null, null);
client.clearHiveTableCache();
- table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
- Storage.getPrefix(c4)));
+ CubeFactTable fact = client.getFactTable(factName);
+ Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1)));
assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
- for(UpdatePeriod period: Lists.newArrayList(DAILY, MINUTELY, MONTHLY, YEARLY, QUARTERLY)) {
- for(String partCol: Lists.newArrayList("ttd", "ttd2")) {
- assertTimeline(client, fact.getName(), c4, period, partCol, EndsAndHolesPartitionTimeline.class);
+ for (UpdatePeriod period: Lists.newArrayList(MINUTELY, MINUTELY, DAILY, MONTHLY, YEARLY, QUARTERLY)) {
+ for (String partCol: Lists.newArrayList("dt")) {
+ assertTimeline(client, fact.getName(), c1, period, partCol, EndsAndHolesPartitionTimeline.class);
}
}
- assertTimeline(client, fact.getName(), c4, HOURLY, "ttd", ttdStoreAll);
- assertTimeline(client, fact.getName(), c4, HOURLY, "ttd2", ttd2StoreAll);
- // Add all hourly partitions for TWO_DAYS_RANGE_BEFORE_4_DAYS
- cal.setTime(BEFORE_4_DAYS_START);
- temp = cal.getTime();
- while (!(temp.after(BEFORE_4_DAYS_END))) {
- Map<String, Date> timeParts = new HashMap<String, Date>();
- timeParts.put("ttd", temp);
- timeParts.put("ttd2", temp);
- StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
- client.addPartition(sPartSpec, c4);
- cal.add(Calendar.HOUR_OF_DAY, 1);
- temp = cal.getTime();
+ table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4)));
+ assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+ for (UpdatePeriod period: Lists.newArrayList(MINUTELY, MINUTELY, DAILY, MONTHLY, YEARLY, QUARTERLY)) {
+ for (String partCol: Lists.newArrayList("ttd", "ttd2")) {
+ assertTimeline(client, fact.getName(), c4, period, partCol, EndsAndHolesPartitionTimeline.class);
+ }
}
}
+
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName,
UpdatePeriod updatePeriod, String timeDim, PartitionTimeline expectedTimeline)
throws Exception {
@@ -1351,7 +1325,7 @@ public class CubeTestSetup {
factValidityProperties, storageTables);
}
- private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws HiveException, LensException {
+ private void createCubeFactOnlyHourly(CubeMetastoreClient client) throws Exception {
String factName = "testFact2";
List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size());
for (CubeMeasure measure : cubeMeasures) {
@@ -1363,7 +1337,8 @@ public class CubeTestSetup {
// add dimensions of the cube
factColumns.add(new FieldSchema("zipcode", "int", "zip"));
factColumns.add(new FieldSchema("cityid", "int", "city id"));
- factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id"));
+ factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id"));
+ factColumns.add(new FieldSchema("test_time_dim_hour_id2", "int", "time id"));
factColumns.add(new FieldSchema("cdim2", "int", "cycledim id"));
Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
@@ -1379,16 +1354,27 @@ public class CubeTestSetup {
s1.setPartCols(partCols);
s1.setTimePartCols(timePartCols);
+ StorageTableDesc s2 = new StorageTableDesc();
+ s2.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ ArrayList<FieldSchema> s2PartCols = new ArrayList<FieldSchema>();
+ s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition"));
+ s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition"));
+ s2.setPartCols(s2PartCols);
+ s2.setTimePartCols(Arrays.asList("ttd", "ttd2"));
+
storageAggregatePeriods.put(c1, updates);
+ storageAggregatePeriods.put(c4, updates);
Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
storageTables.put(c1, s1);
+ storageTables.put(c4, s2);
// create cube fact
client
.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 10L,
factValidityProperties, storageTables);
- CubeFactTable fact2 = client.getFactTable(factName);
+ CubeFactTable fact = client.getFactTable(factName);
// Add all hourly partitions for two days
Calendar cal = Calendar.getInstance();
cal.setTime(TWODAYS_BACK);
@@ -1396,7 +1382,7 @@ public class CubeTestSetup {
while (!(temp.after(NOW))) {
Map<String, Date> timeParts = new HashMap<String, Date>();
timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp);
- StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY);
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
try {
client.addPartition(sPartSpec, c1);
} catch (HiveException e) {
@@ -1414,11 +1400,64 @@ public class CubeTestSetup {
while (!(temp.after(BEFORE_4_DAYS_END))) {
Map<String, Date> timeParts = new HashMap<String, Date>();
timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp);
- StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY);
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c1);
cal.add(Calendar.HOUR_OF_DAY, 1);
temp = cal.getTime();
}
+ client.clearHiveTableCache();
+
+ Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
+ Storage.getPrefix(c4)));
+ table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd"),
+ StoreAllPartitionTimeline.class.getCanonicalName());
+ table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd2"),
+ StoreAllPartitionTimeline.class.getCanonicalName());
+ client.pushHiveTable(table);
+ // Add all hourly partitions for two days on C4
+ cal = Calendar.getInstance();
+ cal.setTime(TWODAYS_BACK);
+ temp = cal.getTime();
+ List<StoragePartitionDesc> storagePartitionDescs = Lists.newArrayList();
+ List<String> partitions = Lists.newArrayList();
+ StoreAllPartitionTimeline ttdStoreAll =
+ new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
+ "ttd");
+ StoreAllPartitionTimeline ttd2StoreAll =
+ new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
+ "ttd2");
+ while (!(temp.after(NOW))) {
+ Map<String, Date> timeParts = new HashMap<String, Date>();
+ timeParts.put("ttd", temp);
+ timeParts.put("ttd2", temp);
+ TimePartition tp = TimePartition.of(HOURLY, temp);
+ ttdStoreAll.add(tp);
+ ttd2StoreAll.add(tp);
+ partitions.add(HOURLY.format().format(temp));
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
+ storagePartitionDescs.add(sPartSpec);
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ temp = cal.getTime();
+ }
+ client.addPartitions(storagePartitionDescs, c4);
+ client.clearHiveTableCache();
+ table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4)));
+ assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+ assertTimeline(client, fact.getName(), c4, HOURLY, "ttd", ttdStoreAll);
+ assertTimeline(client, fact.getName(), c4, HOURLY, "ttd2", ttd2StoreAll);
+
+ // Add all hourly partitions for TWO_DAYS_RANGE_BEFORE_4_DAYS
+ cal.setTime(BEFORE_4_DAYS_START);
+ temp = cal.getTime();
+ while (!(temp.after(BEFORE_4_DAYS_END))) {
+ Map<String, Date> timeParts = new HashMap<String, Date>();
+ timeParts.put("ttd", temp);
+ timeParts.put("ttd2", temp);
+ StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
+ client.addPartition(sPartSpec, c4);
+ cal.add(Calendar.HOUR_OF_DAY, 1);
+ temp = cal.getTime();
+ }
}
private void createCubeFactOnlyHourlyRaw(CubeMetastoreClient client) throws HiveException, LensException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index f65bd28..0d0b927 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -72,8 +72,8 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertEquals(e.getCanonicalErrorMsg().getErrorCode(),
ErrorMsg.EXPRESSION_NOT_IN_ANY_FACT.getErrorCode());
// no fact has the all the dimensions queried
- e = getSemanticExceptionInRewrite("select dim1, stateid, msr3, msr13 from basecube" + " where " + TWO_DAYS_RANGE,
- conf);
+ e = getSemanticExceptionInRewrite("select dim1, test_time_dim, msr3, msr13 from basecube where "
+ + TWO_DAYS_RANGE, conf);
assertEquals(e.getCanonicalErrorMsg().getErrorCode(),
ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
PruneCauses.BriefAndDetailedError pruneCauses = extractPruneCause(e);
@@ -83,17 +83,13 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(matcher.matches(), pruneCauses.getBrief());
assertEquals(matcher.groupCount(), 1);
String columnSetsStr = matcher.group(1);
- assertNotEquals(columnSetsStr.indexOf("stateid"), -1);
+ assertNotEquals(columnSetsStr.indexOf("test_time_dim"), -1, columnSetsStr);
assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1);
- assertEquals(pruneCauses.getDetails().get("testfact3_base,testfact3_raw_base"),
- Arrays.asList(CandidateTablePruneCause.columnNotFound("stateid")));
+ assertEquals(pruneCauses.getDetails().get("testfact3_base,testfact1_raw_base,testfact3_raw_base"),
+ Arrays.asList(CandidateTablePruneCause.columnNotFound("test_time_dim")));
assertEquals(pruneCauses.getDetails().get("testfact_deprecated,testfact2_raw_base,testfact2_base"),
Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr13")));
- assertTrue(pruneCauses.getDetails().containsKey("testfact1_base,testfact1_raw_base")
- || pruneCauses.getDetails().containsKey("testfact1_raw_base,testfact1_base"));
- String fact1BaseKey = pruneCauses.getDetails().containsKey("testfact1_base,testfact1_raw_base")
- ? "testfact1_base,testfact1_raw_base" : "testfact1_raw_base,testfact1_base";
- assertEquals(pruneCauses.getDetails().get(fact1BaseKey),
+ assertEquals(pruneCauses.getDetails().get("testfact1_base"),
Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED)));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index 7d9183c..1bf1a5c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -237,13 +237,43 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
for (CandidateFact cfact : cubeql.getCandidateFacts()) {
candidateFacts.add(cfact.getName().toLowerCase());
}
- // testfact contains test_time_dim_hour_id, but not dim2 - it should have been removed.
+ // testfact contains test_time_dim_day_id, but not dim2 - it should have been removed.
Assert.assertFalse(candidateFacts.contains("testfact"));
// summary2 contains dim2, but not test_time_dim2 - it should have been removed.
Assert.assertFalse(candidateFacts.contains("summary2"));
}
@Test
+ public void testCubeQueryWithHourDimJoin() throws Exception {
+ Configuration tConf = new Configuration(conf);
+ tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C4");
+ tConf.set(CubeQueryConfUtil.getValidFactTablesKey(cubeName), "testFact2");
+ tConf.set(CubeQueryConfUtil.getValidStorageTablesKey("testFact2"), "C1_testFact2");
+ String hqlQuery = rewrite("select test_time_dim2, msr2 from testcube where " + TWO_DAYS_RANGE, tConf);
+ String expected =
+ getExpectedQuery(cubeName, "select timehourchain.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName()
+ + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null,
+ " group by timehourchain . full_hour ", null,
+ getWhereForHourly2days("c1_testfact2"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+
+ @Test
+ public void testCubeQueryWithDayDimJoin() throws Exception {
+ Configuration tConf = new Configuration(conf);
+ tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C4");
+ tConf.set(CubeQueryConfUtil.getValidFactTablesKey(cubeName), "testFact");
+ tConf.set(CubeQueryConfUtil.getValidStorageTablesKey("testFact"), "C1_testFact");
+ String hqlQuery = rewrite("select test_time_dim2, msr2 from testcube where " + TWO_DAYS_RANGE, tConf);
+ String expected =
+ getExpectedQuery(cubeName, "select timedatechain.full_date, sum(testcube.msr2) FROM ", " join " + getDbName()
+ + "c4_dayDimTbl timedatechain on testcube.test_time_dim_day_id2 = timedatechain.id", null,
+ " group by timedatechain . full_date ", null,
+ getWhereForDailyAndHourly2days(cubeName, "c1_testfact"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+
+ @Test
public void testCubeQueryWithOptionalDimsRemoved() throws Exception {
String hqlQuery = rewrite("select cityzip.code, dim22, msr11 from basecube where " + TWO_DAYS_RANGE,
conf);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
index 00d92b5..8da740b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
@@ -76,7 +76,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
}
@Test
- public void testCubeQuery() throws Exception {
+ public void testCubeQueryContinuousUpdatePeriod() throws Exception {
SemanticException th = null;
try {
rewrite("cube select" + " SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf);
@@ -109,7 +109,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
UpdatePeriod.CONTINUOUS.format()));
String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
// multiple range query
//from date 4 days back
@@ -132,7 +132,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
UpdatePeriod.CONTINUOUS.format()));
expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
// format option in the query
conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
@@ -143,23 +143,25 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
getUptoHour(CubeTestSetup.NOW), TestTimeRangeWriter.DB_FORMAT));
expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
}
@Test
public void testCubeQueryWithTimeDim() throws Exception {
+ Configuration tconf = new Configuration(conf);
// hourly partitions for two days
- conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
- conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3,C4");
- conf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
- conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
+ tconf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
+ tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C4");
+ tconf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
+ tconf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
+ tconf.set(CubeQueryConfUtil.getValidUpdatePeriodsKey("testfact", "C4"), "MONTHLY,DAILY,HOURLY");
String query =
"SELECT test_time_dim, msr2 FROM testCube where " + "time_range_in(test_time_dim, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')";
- String hqlQuery = rewrite(query, conf);
+ String hqlQuery = rewrite(query, tconf);
Map<String, String> whereClauses = new HashMap<String, String>();
- whereClauses.put(CubeTestSetup.getDbName() + "c4_testfact", TestBetweenTimeRangeWriter.getBetweenClause("hourdim",
+ whereClauses.put(CubeTestSetup.getDbName() + "c4_testfact2", TestBetweenTimeRangeWriter.getBetweenClause("hourdim",
"full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
getUptoHour(getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT));
System.out.println("HQL:" + hqlQuery);
@@ -167,29 +169,29 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
getExpectedQuery(cubeName, "select hourdim.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName()
+ "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null,
" GROUP BY hourdim.full_hour", null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
query =
"SELECT msr2 FROM testCube where " + "time_range_in(test_time_dim, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')";
- hqlQuery = rewrite(query, conf);
+ hqlQuery = rewrite(query, tconf);
System.out.println("HQL:" + hqlQuery);
expected =
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
+ "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, null, null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
query =
"SELECT msr2 FROM testCube where testcube.cityid > 2 and " + "time_range_in(test_time_dim, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW)
+ "') and testcube.cityid != 5";
- hqlQuery = rewrite(query, conf);
+ hqlQuery = rewrite(query, tconf);
System.out.println("HQL:" + hqlQuery);
expected =
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
+ "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", " testcube.cityid > 2 ",
" and testcube.cityid != 5", null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
// multiple range query
hqlQuery =
@@ -197,11 +199,11 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
"select SUM(msr2) from testCube" + " where time_range_in(test_time_dim, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')"
+ " OR time_range_in(test_time_dim, '" + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_START) + "','"
- + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", conf);
+ + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", tconf);
whereClauses = new HashMap<String, String>();
whereClauses.put(
- CubeTestSetup.getDbName() + "c4_testfact",
+ CubeTestSetup.getDbName() + "c4_testfact2",
TestBetweenTimeRangeWriter.getBetweenClause("hourdim", "full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
getUptoHour(getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField())),
TestTimeRangeWriter.DB_FORMAT)
@@ -213,67 +215,70 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
+ "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
hqlQuery =
rewrite(
"select to_date(test_time_dim), SUM(msr2) from testCube" + " where time_range_in(test_time_dim, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')"
+ " OR time_range_in(test_time_dim, '" + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_START) + "','"
- + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", conf);
+ + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", tconf);
expected =
getExpectedQuery(cubeName, "select to_date(hourdim.full_hour), sum(testcube.msr2) FROM ", " join "
+ getDbName() + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null,
" group by to_date(hourdim.full_hour)", null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
}
@Test
public void testCubeQueryWithTimeDimThruChain() throws Exception {
// hourly partitions for two days
- conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
- conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3,C4");
- conf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
- conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
+ Configuration tconf = new Configuration(conf);
+ tconf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
+ tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C4");
+ tconf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
+ tconf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
+ tconf.set(CubeQueryConfUtil.getValidUpdatePeriodsKey("testfact", "C4"), "MONTHLY,DAILY,HOURLY");
String query =
"SELECT test_time_dim2, msr2 FROM testCube where " + "time_range_in(test_time_dim2, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')";
- String hqlQuery = rewrite(query, conf);
+ String hqlQuery = rewrite(query, tconf);
Map<String, String> whereClauses = new HashMap<String, String>();
- whereClauses.put(CubeTestSetup.getDbName() + "c4_testfact", TestBetweenTimeRangeWriter.getBetweenClause("timechain",
- "full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
+ whereClauses.put(CubeTestSetup.getDbName() + "c4_testfact2", TestBetweenTimeRangeWriter.getBetweenClause(
+ "timehourchain", "full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
getUptoHour(getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT));
System.out.println("HQL:" + hqlQuery);
String expected =
- getExpectedQuery(cubeName, "select timechain.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName()
- + "c4_hourDimTbl timechain on testcube.test_time_dim_hour_id2 = timechain.id", null,
- " GROUP BY timechain.full_hour", null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ getExpectedQuery(cubeName, "select timehourchain.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName()
+ + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null,
+ " GROUP BY timehourchain.full_hour", null, whereClauses);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
query =
"SELECT msr2 FROM testCube where " + "time_range_in(test_time_dim2, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')";
- hqlQuery = rewrite(query, conf);
+ hqlQuery = rewrite(query, tconf);
System.out.println("HQL:" + hqlQuery);
expected =
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
- + "c4_hourDimTbl timechain on testcube.test_time_dim_hour_id2 = timechain.id", null, null, null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, null, null,
+ whereClauses);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
query =
"SELECT msr2 FROM testCube where testcube.cityid > 2 and " + "time_range_in(test_time_dim2, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW)
+ "') and testcube.cityid != 5";
- hqlQuery = rewrite(query, conf);
+ hqlQuery = rewrite(query, tconf);
System.out.println("HQL:" + hqlQuery);
expected =
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
- + "c4_hourDimTbl timechain on testcube.test_time_dim_hour_id2 = timechain.id", " testcube.cityid > 2 ",
- " and testcube.cityid != 5", null, whereClauses);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id",
+ " testcube.cityid > 2 ", " and testcube.cityid != 5", null, whereClauses);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
// multiple range query
hqlQuery =
@@ -281,37 +286,38 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
"select SUM(msr2) from testCube" + " where time_range_in(test_time_dim2, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')"
+ " OR time_range_in(test_time_dim2, '" + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_START) + "','"
- + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", conf);
+ + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", tconf);
whereClauses = new HashMap<String, String>();
whereClauses.put(
- CubeTestSetup.getDbName() + "c4_testfact",
- TestBetweenTimeRangeWriter.getBetweenClause("timechain", "full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
+ CubeTestSetup.getDbName() + "c4_testfact2",
+ TestBetweenTimeRangeWriter.getBetweenClause("timehourchain", "full_hour", getUptoHour(CubeTestSetup.TWODAYS_BACK),
getUptoHour(getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField())),
TestTimeRangeWriter.DB_FORMAT)
+ " OR "
- + TestBetweenTimeRangeWriter.getBetweenClause("timechain", "full_hour", getUptoHour(BEFORE_4_DAYS_START),
+ + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain", "full_hour", getUptoHour(BEFORE_4_DAYS_START),
getUptoHour(getOneLess(BEFORE_4_DAYS_END, UpdatePeriod.HOURLY.calendarField())),
TestTimeRangeWriter.DB_FORMAT));
expected =
getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName()
- + "c4_hourDimTbl timechain on testcube.test_time_dim_hour_id2 = timechain.id", null, null, null, whereClauses);
+ + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, null, null,
+ whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
hqlQuery =
rewrite(
"select to_date(test_time_dim2), SUM(msr2) from testCube" + " where time_range_in(test_time_dim2, '"
+ CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')"
+ " OR time_range_in(test_time_dim2, '" + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_START) + "','"
- + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", conf);
+ + CubeTestSetup.getDateUptoHours(BEFORE_4_DAYS_END) + "')", tconf);
expected =
- getExpectedQuery(cubeName, "select to_date(timechain.full_hour), sum(testcube.msr2) FROM ", " join "
- + getDbName() + "c4_hourDimTbl timechain on testcube.test_time_dim_hour_id2 = timechain.id", null,
- " group by to_date(timechain.full_hour)", null, whereClauses);
+ getExpectedQuery(cubeName, "select to_date(timehourchain.full_hour), sum(testcube.msr2) FROM ", " join "
+ + getDbName() + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null,
+ " group by to_date(timehourchain.full_hour)", null, whereClauses);
System.out.println("HQL:" + hqlQuery);
- TestCubeRewriter.compareQueries(expected, hqlQuery);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-examples/src/main/resources/sales-cube.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml
index 4923aa3..2d67f5c 100644
--- a/lens-examples/src/main/resources/sales-cube.xml
+++ b/lens-examples/src/main/resources/sales-cube.xml
@@ -58,6 +58,7 @@
display_string="Customer City">
<ref_spec>
<chain_ref_column chain_name="customer_city" ref_col="name" />
+ <chain_ref_column chain_name="customer_details" ref_col="customer_city_name" />
</ref_spec>
</dim_attribute>
<dim_attribute name="production_city_name" type="STRING" description="City name in which the product was produced"
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 70bd20d..d6c4f17 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -29,6 +29,7 @@ import javax.xml.datatype.XMLGregorianCalendar;
import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -183,12 +184,12 @@ public final class JAXBUtils {
xd.isJoinKey(),
xd.getNumDistinctValues()
);
- } else if (xd.getRefSpec() != null && xd.getRefSpec().getChainRefColumn() != null) {
+ } else if (xd.getRefSpec() != null && xd.getRefSpec().getChainRefColumn() != null
+ && !xd.getRefSpec().getChainRefColumn().isEmpty()) {
hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
xd.getDescription()),
xd.getDisplayString(),
- xd.getRefSpec().getChainRefColumn().getChainName(),
- xd.getRefSpec().getChainRefColumn().getRefCol(),
+ getChainRefColumns(xd.getRefSpec().getChainRefColumn()),
startDate,
endDate,
null,
@@ -208,6 +209,14 @@ public final class JAXBUtils {
return hiveDim;
}
+ private static List<ChainRefCol> getChainRefColumns(List<XChainColumn> chainCols) {
+ List<ChainRefCol> chainRefCols = new ArrayList<>();
+ for (XChainColumn chainCol : chainCols) {
+ chainRefCols.add(new ChainRefCol(chainCol.getChainName(), chainCol.getRefCol()));
+ }
+ return chainRefCols;
+ }
+
/**
* Get XMLGregorianCalendar from Date.
*
@@ -318,20 +327,22 @@ public final class JAXBUtils {
xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime()));
if (cd instanceof ReferencedDimAtrribute) {
ReferencedDimAtrribute rd = (ReferencedDimAtrribute) cd;
- List<TableReference> dimRefs = rd.getReferences();
XDimAttribute.RefSpec refspec = XCF.createXDimAttributeRefSpec();
- if (rd.getChainName() != null) {
- XChainColumn xcc = new XChainColumn();
- xcc.setChainName(rd.getChainName());
- xcc.setRefCol(rd.getRefColumn());
- if (baseTable.getChainByName(rd.getChainName()) == null) {
- log.error("Missing chain definition for " + rd.getChainName());
- } else {
- xcc.setDestTable(baseTable.getChainByName(rd.getChainName()).getDestTable());
+ if (!rd.getChainRefColumns().isEmpty()) {
+ for (ChainRefCol crCol : rd.getChainRefColumns()) {
+ XChainColumn xcc = new XChainColumn();
+ xcc.setChainName(crCol.getChainName());
+ xcc.setRefCol(crCol.getRefColumn());
+ if (baseTable.getChainByName(crCol.getChainName()) == null) {
+ log.error("Missing chain definition for " + crCol.getChainName());
+ } else {
+ xcc.setDestTable(baseTable.getChainByName(crCol.getChainName()).getDestTable());
+ }
+ refspec.getChainRefColumn().add(xcc);
}
- refspec.setChainRefColumn(xcc);
xd.setJoinKey(Boolean.valueOf(false));
} else {
+ List<TableReference> dimRefs = rd.getReferences();
refspec.setTableReferences(new XTableReferences());
refspec.getTableReferences().getTableReference().addAll(xTabReferencesFromHiveTabReferences(dimRefs));
xd.setJoinKey(rd.useAsJoinKey());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bab05e4f/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 6e3afc6..877f707 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -295,7 +295,7 @@ public class TestMetastoreService extends LensJerseyTest {
xcc.setChainName("chain1");
xcc.setRefCol("col2");
xd3.setRefSpec(cubeObjectFactory.createXDimAttributeRefSpec());
- xd3.getRefSpec().setChainRefColumn(xcc);
+ xd3.getRefSpec().getChainRefColumn().add(xcc);
xd3.setNumDistinctValues(1000L);
// add attribute with complex type
@@ -636,7 +636,7 @@ public class TestMetastoreService extends LensJerseyTest {
boolean chainValidated = false;
for (XDimAttribute attr : actual.getDimAttributes().getDimAttribute()) {
if (attr.getName().equalsIgnoreCase("testdim2col2")) {
- assertEquals(attr.getRefSpec().getChainRefColumn().getDestTable(), "testdim");
+ assertEquals(attr.getRefSpec().getChainRefColumn().get(0).getDestTable(), "testdim");
chainValidated = true;
break;
}
@@ -653,13 +653,12 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(hcube.getDimAttributeByName("testdim2col2").getDescription(), "ref chained dimension");
assertEquals(((BaseDimAttribute) hcube.getDimAttributeByName("dim4")).getType(),
"struct<a:int,b:array<string>,c:map<int,array<struct<x:int,y:array<int>>>");
- assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getType(), "string");
- assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getChainName(), "chain1");
- assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getRefColumn(), "col2");
- assertEquals((((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2"))
- .getNumOfDistinctValues().get()), Long.valueOf(1000));
- assertEquals((((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2"))
- .getNumOfDistinctValues().get()), Long.valueOf(1000));
+ ReferencedDimAtrribute testdim2col2 = (ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2");
+ assertEquals(testdim2col2.getType(), "string");
+ assertEquals(testdim2col2.getChainRefColumns().get(0).getChainName(), "chain1");
+ assertEquals(testdim2col2.getChainRefColumns().get(0).getRefColumn(), "col2");
+ assertEquals(testdim2col2.getNumOfDistinctValues().get(), Long.valueOf(1000));
+ assertEquals((testdim2col2.getNumOfDistinctValues().get()), Long.valueOf(1000));
assertEquals(((BaseDimAttribute) hcube.getDimAttributeByName("dim2")).getNumOfDistinctValues().isPresent(),
false);