You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by su...@apache.org on 2014/11/30 06:41:29 UTC
incubator-lens git commit: LENS-59. Add metastore changes to store
multiple path names. ( Amareshwari Sriramadasu via sumasai)
Repository: incubator-lens
Updated Branches:
refs/heads/master 6cafe6404 -> 204a387a7
LENS-59. Add metastore changes to store multiple path names. ( Amareshwari Sriramadasu via sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/204a387a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/204a387a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/204a387a
Branch: refs/heads/master
Commit: 204a387a7ca6a5bf3312bd485b049bd4d086f724
Parents: 6cafe64
Author: suma.shivaprasad <su...@inmobi.com>
Authored: Sun Nov 30 11:11:19 2014 +0530
Committer: suma.shivaprasad <su...@inmobi.com>
Committed: Sun Nov 30 11:11:19 2014 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 26 ++++-
.../org/apache/lens/cube/metadata/Cube.java | 101 ++++++++++++++++++-
.../lens/cube/metadata/CubeInterface.java | 27 ++++-
.../lens/cube/metadata/CubeMetastoreClient.java | 25 ++++-
.../apache/lens/cube/metadata/DerivedCube.java | 18 ++++
.../apache/lens/cube/metadata/JoinChain.java | 92 +++++++++++++++++
.../lens/cube/metadata/MetastoreConstants.java | 8 ++
.../lens/cube/metadata/MetastoreUtil.java | 37 ++++++-
.../cube/metadata/ReferencedDimAtrribute.java | 85 +++++++---------
.../lens/cube/metadata/TableReference.java | 4 +-
.../lens/cube/parse/CandidateTableResolver.java | 7 ++
.../apache/lens/cube/parse/JoinResolver.java | 44 +++++++-
.../cube/metadata/TestCubeMetastoreClient.java | 66 +++++++++++-
.../metastore/CubeMetastoreServiceImpl.java | 6 +-
.../apache/lens/server/metastore/JAXBUtils.java | 71 ++++++++++++-
.../server/metastore/TestMetastoreService.java | 54 +++++++++-
16 files changed, 603 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 000d27e..2a7030f 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -48,6 +48,7 @@
<xs:element type="x_measures" name="measures"/>
<xs:element type="x_dim_attributes" name="dim_attributes"/>
<xs:element type="x_expressions" name="expressions"/>
+ <xs:element type="x_joinchains" name="joinchains"/>
<xs:element type="x_properties" name="properties" minOccurs="0"/>
<xs:element type="x_measure_names" name="measure_names"/>
<xs:element type="x_dim_attr_names" name="dim_attr_names"/>
@@ -172,6 +173,7 @@
</xs:annotation>
<xs:sequence>
<xs:element type="x_tablereferences" name="references"/>
+ <xs:element type="x_chaincolumn" name="chainrefcolumn" maxOccurs="1" minOccurs="1"/>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="type" use="required"/>
@@ -183,6 +185,10 @@
<xs:attribute type="xs:string" name="expr"/>
<xs:attribute type="xs:string" name="cube_table"/>
</xs:complexType>
+ <xs:complexType name="x_chaincolumn">
+ <xs:attribute type="xs:string" name="chainName" use="required"/>
+ <xs:attribute type="xs:string" name="refcol" use="required"/>
+ </xs:complexType>
<xs:complexType name="x_tablereference">
<xs:attribute type="xs:string" name="dest_table" use="required"/>
<xs:attribute type="xs:string" name="dest_column" use="required"/>
@@ -192,7 +198,24 @@
<xs:element type="x_tablereference" name="table_references" maxOccurs="unbounded" minOccurs="1"/>
</xs:sequence>
</xs:complexType>
-
+ <xs:complexType name="x_joinchain">
+ <xs:sequence>
+ <xs:element type="x_tablereferences" name="paths" maxOccurs="unbounded" minOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="description" />
+ <xs:attribute type="xs:string" name="display_string" />
+ </xs:complexType>
+ <xs:complexType name="x_joinchains">
+ <xs:annotation>
+ <xs:documentation>
+ Set of join chains.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element type="x_joinchain" name="chains" maxOccurs="unbounded" minOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
<xs:complexType name="x_dim_attributes">
<xs:annotation>
<xs:documentation>
@@ -435,4 +458,3 @@
</xs:element>
</xs:schema>
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
index d7041ad..7ffb62c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
@@ -26,6 +26,8 @@ import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import lombok.Getter;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -33,8 +35,10 @@ import org.apache.hadoop.hive.ql.metadata.Table;
public class Cube extends AbstractBaseTable implements CubeInterface {
private final Set<CubeMeasure> measures;
private final Set<CubeDimAttribute> dimensions;
+ @Getter private final Set<JoinChain> joinChains;
private final Map<String, CubeMeasure> measureMap;
private final Map<String, CubeDimAttribute> dimMap;
+ private final Map<String, JoinChain> chainMap;
public Cube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions) {
this(name, measures, dimensions, new HashMap<String, String>());
@@ -46,14 +50,20 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
public Cube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions, Map<String, String> properties,
double weight) {
- this(name, measures, dimensions, null, properties, weight);
+ this(name, measures, dimensions, null, null, properties, weight);
}
public Cube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions, Set<ExprColumn> expressions,
+ Set<JoinChain> joinChains,
Map<String, String> properties, double weight) {
super(name, expressions, properties, weight);
this.measures = measures;
this.dimensions = dimensions;
+ if (joinChains != null) {
+ this.joinChains = joinChains;
+ } else {
+ this.joinChains = new HashSet<JoinChain>();
+ }
measureMap = new HashMap<String, CubeMeasure>();
for (CubeMeasure m : measures) {
@@ -65,6 +75,11 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
dimMap.put(dim.getName().toLowerCase(), dim);
}
+ chainMap = new HashMap<String, JoinChain>();
+ for (JoinChain chain : this.joinChains) {
+ chainMap.put(chain.getName().toLowerCase(), chain);
+ }
+
addProperties();
}
@@ -72,6 +87,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
super(tbl);
this.measures = getMeasures(getName(), getProperties());
this.dimensions = getDimensions(getName(), getProperties());
+ this.joinChains = getJoinChains(getName(), getProperties());
measureMap = new HashMap<String, CubeMeasure>();
for (CubeMeasure m : measures) {
measureMap.put(m.getName().toLowerCase(), m);
@@ -81,6 +97,11 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
for (CubeDimAttribute dim : dimensions) {
addAllDimsToMap(dim);
}
+
+ chainMap = new HashMap<String, JoinChain>();
+ for (JoinChain chain : joinChains) {
+ chainMap.put(chain.getName().toLowerCase(), chain);
+ }
}
private void addAllDimsToMap(CubeDimAttribute dim) {
@@ -121,6 +142,8 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
setMeasureProperties(getProperties(), measures);
MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeDimensionListKey(getName()), dimensions);
setDimensionProperties(getProperties(), dimensions);
+ MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeJoinChainListKey(getName()), joinChains);
+ setJoinChainProperties(getProperties(), joinChains);
}
private static void setMeasureProperties(Map<String, String> props, Set<CubeMeasure> measures) {
@@ -135,6 +158,12 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
}
}
+ private static void setJoinChainProperties(Map<String, String> props, Set<JoinChain> chains) {
+ for (JoinChain chain : chains) {
+ chain.addProperties(props);
+ }
+ }
+
public static Set<CubeMeasure> getMeasures(String name, Map<String, String> props) {
Set<CubeMeasure> measures = new HashSet<CubeMeasure>();
String measureStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeMeasureListKey(name));
@@ -175,6 +204,19 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
return dimensions;
}
+ private static Set<JoinChain> getJoinChains(String name, Map<String, String> props) {
+ Set<JoinChain> joinChains = new HashSet<JoinChain>();
+ String joinChainsStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeJoinChainListKey(name));
+ if (!StringUtils.isBlank(joinChainsStr)) {
+ String[] cnames = joinChainsStr.split(",");
+ for (String chainName : cnames) {
+ JoinChain chain = new JoinChain(chainName, props);
+ joinChains.add(chain);
+ }
+ }
+ return joinChains;
+ }
+
@Override
public boolean equals(Object obj) {
if (!super.equals(obj)) {
@@ -195,6 +237,13 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
} else if (!this.getDimAttributes().equals(other.getDimAttributes())) {
return false;
}
+ if (this.getJoinChains() == null) {
+ if (other.getJoinChains() != null) {
+ return false;
+ }
+ } else if (!this.getJoinChains().equals(other.getJoinChains())) {
+ return false;
+ }
return true;
}
@@ -217,6 +266,10 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
return cubeCol;
}
+ public JoinChain getChainByName(String name) {
+ return chainMap.get(name == null ? name : name.toLowerCase());
+ }
+
/**
* Alters the measure if already existing or just adds if it is new measure.
*
@@ -265,6 +318,43 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
}
/**
+ * Alters the joinchain if already existing or just adds if it is new chain
+ *
+ * @param joinchain
+ * @throws HiveException
+ */
+ public void alterJoinChain(JoinChain joinchain) throws HiveException {
+ if (joinchain == null) {
+ throw new NullPointerException("Cannot add null joinchain");
+ }
+
+ // Replace dimension if already existing
+ if (chainMap.containsKey(joinchain.getName().toLowerCase())) {
+ joinChains.remove(getChainByName(joinchain.getName()));
+ LOG.info("Replacing joinchain " + getChainByName(joinchain.getName()) + " with " + joinchain);
+ }
+
+ joinChains.add(joinchain);
+ chainMap.put(joinchain.getName().toLowerCase(), joinchain);
+ MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeJoinChainListKey(getName()), joinChains);
+ joinchain.addProperties(getProperties());
+ }
+
+ /**
+ * Remove the joinchain with name specified
+ *
+ * @param chainName
+ */
+ public void removeJoinChain(String chainName) {
+ if (chainMap.containsKey(chainName.toLowerCase())) {
+ LOG.info("Removing dimension " + getDimAttributeByName(chainName));
+ joinChains.remove(getChainByName(chainName));
+ chainMap.remove(chainName.toLowerCase());
+ MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeJoinChainListKey(getName()), joinChains);
+ }
+ }
+
+ /**
* Remove the dimension with name specified
*
* @param dimName
@@ -358,6 +448,15 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
}
@Override
+ public Set<String> getJoinChainNames() {
+ Set<String> chainNames = new HashSet<String>();
+ for (JoinChain f : getJoinChains()) {
+ chainNames.add(f.getName().toLowerCase());
+ }
+ return chainNames;
+ }
+
+ @Override
public boolean allFieldsQueriable() {
String canbeQueried = getProperties().get(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE);
if (canbeQueried != null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
index 62a5d31..7389b4c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
@@ -47,6 +47,13 @@ public interface CubeInterface extends Named {
public Set<ExprColumn> getExpressions();
/**
+ * Get all joinchains defined on the cube
+ *
+ * @return set {@link JoinChain}
+ */
+ public Set<JoinChain> getJoinChains();
+
+ /**
* Get dimension attribute given by name
*
* @param dimAttrName
@@ -77,6 +84,15 @@ public interface CubeInterface extends Named {
public ExprColumn getExpressionByName(String exprName);
/**
+ * Get joinchain by given by name
+ *
+ * @param chainName Chain name
+ *
+ * @return A {@link JoinChain} object
+ */
+ public JoinChain getChainByName(String chainName);
+
+ /**
* Get cube column given by column name.
*
* It can be a measure, dimension attribute or an expression.
@@ -117,15 +133,22 @@ public interface CubeInterface extends Named {
public Set<String> getDimAttributeNames();
/**
+ * Get all join chain names
+ *
+ * @return Set of strings
+ */
+ public Set<String> getJoinChainNames();
+
+ /**
* Get all expression names
- *
+ *
* @return Set of strings
*/
public Set<String> getExpressionNames();
/**
* Get all field names reachable from cube
- *
+ *
* @return Set of strings
*/
public Set<String> getAllFieldNames();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 1fddd71..42f5e0a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -196,7 +196,30 @@ public class CubeMetastoreClient {
*/
public void createCube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions,
Set<ExprColumn> expressions, Map<String, String> properties) throws HiveException {
- Cube cube = new Cube(name, measures, dimensions, expressions, properties, 0L);
+ Cube cube = new Cube(name, measures, dimensions, expressions, null, properties, 0L);
+ createCube(cube);
+ }
+
+ /**
+ * Create cube defined by measures, dimensions and properties
+ *
+ * @param name
+ * Name of the cube
+ * @param measures
+ * Measures of the cube
+ * @param dimensions
+ * Dimensions of the cube
+ * @param expressions
+ * Expressions of the cube
+ * @param chains
+ * JoinChains of the cube
+ * @param properties
+ * Properties of the cube
+ * @throws HiveException
+ */
+ public void createCube(String name, Set<CubeMeasure> measures, Set<CubeDimAttribute> dimensions,
+ Set<ExprColumn> expressions, Set<JoinChain> chains, Map<String, String> properties) throws HiveException {
+ Cube cube = new Cube(name, measures, dimensions, expressions, chains, properties, 0L);
createCube(cube);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
index f511cfe..cbb2f81 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DerivedCube.java
@@ -279,4 +279,22 @@ public class DerivedCube extends AbstractCubeTable implements CubeInterface {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public Set<JoinChain> getJoinChains() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public JoinChain getChainByName(String chainName) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Set<String> getJoinChainNames() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
new file mode 100644
index 0000000..ebdf952
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
+import org.apache.commons.lang.StringUtils;
+
+@EqualsAndHashCode
+@ToString
+public class JoinChain implements Named {
+ @Getter private final String name;
+ @Getter private final String displayString;
+ @Getter private final String description;
+ // There can be more than one path associated with same name.
+ // c1.r1->t1.k1
+ // c1.r2->t1.k2
+ @Getter private final List<List<TableReference>> paths = new ArrayList<List<TableReference>>();
+
+ public void addProperties(Map<String, String> props) {
+ props.put(MetastoreUtil.getJoinChainNumChainsKey(getName()), String.valueOf(paths.size()));
+ for (int i = 0; i< paths.size(); i++) {
+ props.put(MetastoreUtil.getJoinChainFullChainKey(getName(), i), MetastoreUtil.getReferencesString(paths.get(i)));
+ }
+ props.put(MetastoreUtil.getJoinChainDisplayKey(getName()), displayString);
+ props.put(MetastoreUtil.getJoinChainDescriptionKey(getName()), description);
+ }
+
+ /**
+ * Construct join chain
+ *
+ * @param name
+ * @param display
+ * @param description
+ */
+ public JoinChain(String name, String display, String description) {
+ this.name = name.toLowerCase();
+ this.displayString = display;
+ this.description = description;
+ }
+
+ /**
+ * This is used only for serializing
+ *
+ * @param name
+ * @param props
+ */
+ public JoinChain(String name, Map<String, String> props) {
+ this.name = name;
+ int numChains = Integer.parseInt(props.get(MetastoreUtil.getJoinChainNumChainsKey(getName())));
+ for (int i = 0; i < numChains; i++) {
+ List<TableReference> chain = new ArrayList<TableReference>();
+ String refListStr = props.get(MetastoreUtil.getJoinChainFullChainKey(getName(), i));
+ String refListDims[] = StringUtils.split(refListStr, ",");
+ for (String refDimRaw : refListDims) {
+ chain.add(new TableReference(refDimRaw));
+ }
+ paths.add(chain);
+ }
+ this.description = props.get(MetastoreUtil.getJoinChainDescriptionKey(name));
+ this.displayString = props.get(MetastoreUtil.getJoinChainDisplayKey(name));
+ }
+
+ public void addPath(List<TableReference> path) {
+ if (path.size() <= 1) {
+ throw new IllegalArgumentException("Path should contain atlease two links");
+ }
+ this.paths.add(path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index ddc87dc..5df35a5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -30,6 +30,7 @@ public interface MetastoreConstants {
public static final String CUBE_KEY_PFX = "cube.";
public static final String MEASURES_LIST_SFX = ".measures.list";
public static final String DIMENSIONS_LIST_SFX = ".dimensions.list";
+ public static final String JOIN_CHAIN_LIST_SFX = ".joinchains.list";
public static final String TIMED_DIMENSIONS_LIST_SFX = ".timed.dimensions.list";
public static final String PARENT_CUBE_SFX = ".parent.cube";
public static final String CUBE_ALL_FIELDS_QUERIABLE = "cube.allfields.queriable";
@@ -73,6 +74,8 @@ public interface MetastoreConstants {
// dimension constants
public static final String DIM_KEY_PFX = "cube.dimension.";
public static final String DIM_REFERS_SFX = ".refers";
+ public static final String CHAIN_NAME_SFX = ".chain.name";
+ public static final String CHAIN_REF_COLUMN_SFX = ".chain.column.name";
public static final String IS_JOIN_KEY_SFX = ".isjoinkey";
public static final String TABLE_COLUMN_SEPERATOR = ".";
public static final String INLINE_SIZE_SFX = ".inline.size";
@@ -81,6 +84,11 @@ public interface MetastoreConstants {
public static final String CLASS_SFX = ".class";
public static final String METASTORE_ENABLE_CACHING = "cube.metastore.enable.cache";
+ // join chain constants
+ public static final String JOIN_CHAIN_KEY = "cube.joinchain.";
+ public static final String NUM_CHAINS_SFX = ".numchains";
+ public static final String FULL_CHAIN_KEY = ".fullchain.";
+
// storage constants
public static final String STORAGE_ENTITY_PFX = "storage.";
public static final String STORAGE_PFX = "cube.storagetable.";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index bcdfddb..bd95497 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -106,6 +106,14 @@ public class MetastoreUtil implements MetastoreConstants {
return getDimensionKeyPrefix(dimName) + DIM_REFERS_SFX;
}
+ public static final String getDimRefChainNameKey(String dimName) {
+ return getDimensionKeyPrefix(dimName) + CHAIN_NAME_SFX;
+ }
+
+ public static final String getDimRefChainColumnKey(String dimName) {
+ return getDimensionKeyPrefix(dimName) + CHAIN_REF_COLUMN_SFX;
+ }
+
public static String getDimUseAsJoinKey(String dimName) {
return getDimensionKeyPrefix(dimName) + IS_JOIN_KEY_SFX;
}
@@ -114,7 +122,7 @@ public class MetastoreUtil implements MetastoreConstants {
return tableName.toLowerCase() + TABLE_COLUMN_SEPERATOR + columnName.toLowerCase();
}
- public static final String getDimensionDestReference(List<TableReference> references) {
+ public static final String getReferencesString(List<TableReference> references) {
String toks[] = new String[references.size()];
for (int i = 0; i < references.size(); i++) {
@@ -160,6 +168,29 @@ public class MetastoreUtil implements MetastoreConstants {
return getColumnKeyPrefix(colName) + TYPE_SFX;
}
+ ////////////////////////////
+ // Join chain properties //
+ ////////////////////////////
+ public static String getJoinChainKey(String colName) {
+ return JOIN_CHAIN_KEY + colName.toLowerCase();
+ }
+
+ public static String getJoinChainNumChainsKey(String colName) {
+ return getJoinChainKey(colName) + NUM_CHAINS_SFX;
+ }
+
+ public static String getJoinChainFullChainKey(String colName, int index) {
+ return getJoinChainKey(colName) + FULL_CHAIN_KEY + index;
+ }
+ public static String getJoinChainDescriptionKey(String colName) {
+ return getJoinChainKey(colName) + DESC_SFX;
+ }
+
+ public static String getJoinChainDisplayKey(String colName) {
+ return getJoinChainKey(colName) + DISPLAY_SFX;
+ }
+
+
// ////////////////////////
// Dimension table properties //
// ////////////////////////
@@ -241,6 +272,10 @@ public class MetastoreUtil implements MetastoreConstants {
return getCubePrefix(cubeName) + TIMED_DIMENSIONS_LIST_SFX;
}
+ public static final String getCubeJoinChainListKey(String cubeName) {
+ return getCubePrefix(cubeName) + JOIN_CHAIN_LIST_SFX;
+ }
+
public static final String getParentCubeNameKey(String cubeName) {
return getCubePrefix(cubeName) + PARENT_CUBE_SFX;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
index afb4a22..51a9528 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ReferencedDimAtrribute.java
@@ -24,14 +24,22 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+@EqualsAndHashCode(callSuper=true)
+@ToString
public class ReferencedDimAtrribute extends BaseDimAttribute {
- private final List<TableReference> references = new ArrayList<TableReference>();
+ @Getter private final List<TableReference> references = new ArrayList<TableReference>();
// boolean whether to say the key is only a denormalized variable kept or can
// be used in join resolution as well
private Boolean isJoinKey = true;
+ @Getter private String chainName = null;
+ @Getter private String refColumn = null;
public ReferencedDimAtrribute(FieldSchema column, String displayString, TableReference reference) {
this(column, displayString, reference, null, null, null);
@@ -65,12 +73,16 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
this.isJoinKey = isJoinKey;
}
- public void addReference(TableReference reference) {
- references.add(reference);
+ public ReferencedDimAtrribute(FieldSchema column, String displayString, String chainName, String refColumn,
+ Date startTime, Date endTime, Double cost) {
+ super(column, displayString, startTime, endTime, cost);
+ this.chainName = chainName.toLowerCase();
+ this.refColumn = refColumn.toLowerCase();
+ this.isJoinKey = false;
}
- public List<TableReference> getReferences() {
- return references;
+ public void addReference(TableReference reference) {
+ references.add(reference);
}
public boolean removeReference(TableReference ref) {
@@ -84,9 +96,14 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
@Override
public void addProperties(Map<String, String> props) {
super.addProperties(props);
- props
- .put(MetastoreUtil.getDimensionSrcReferenceKey(getName()), MetastoreUtil.getDimensionDestReference(references));
- props.put(MetastoreUtil.getDimUseAsJoinKey(getName()), isJoinKey.toString());
+ if (chainName != null) {
+ props.put(MetastoreUtil.getDimRefChainNameKey(getName()), chainName);
+ props.put(MetastoreUtil.getDimRefChainColumnKey(getName()), refColumn);
+ } else {
+ props.put(MetastoreUtil.getDimensionSrcReferenceKey(getName()),
+ MetastoreUtil.getReferencesString(references));
+ props.put(MetastoreUtil.getDimUseAsJoinKey(getName()), isJoinKey.toString());
+ }
}
/**
@@ -97,45 +114,21 @@ public class ReferencedDimAtrribute extends BaseDimAttribute {
*/
public ReferencedDimAtrribute(String name, Map<String, String> props) {
super(name, props);
- String refListStr = props.get(MetastoreUtil.getDimensionSrcReferenceKey(getName()));
- String refListDims[] = StringUtils.split(refListStr, ",");
- for (String refDimRaw : refListDims) {
- references.add(new TableReference(refDimRaw));
- }
- String isJoinKeyStr = props.get(MetastoreUtil.getDimUseAsJoinKey(name));
- if (isJoinKeyStr != null) {
- isJoinKey = Boolean.parseBoolean(isJoinKeyStr);
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((getReferences() == null) ? 0 : getReferences().hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!super.equals(obj)) {
- return false;
- }
- ReferencedDimAtrribute other = (ReferencedDimAtrribute) obj;
- if (this.getReferences() == null) {
- if (other.getReferences() != null) {
- return false;
+ String chName = props.get(MetastoreUtil.getDimRefChainNameKey(getName()));
+ if (!StringUtils.isBlank(chName)) {
+ this.chainName = chName;
+ this.refColumn = props.get(MetastoreUtil.getDimRefChainColumnKey(getName()));
+ this.isJoinKey = false;
+ } else {
+ String refListStr = props.get(MetastoreUtil.getDimensionSrcReferenceKey(getName()));
+ String refListDims[] = StringUtils.split(refListStr, ",");
+ for (String refDimRaw : refListDims) {
+ references.add(new TableReference(refDimRaw));
+ }
+ String isJoinKeyStr = props.get(MetastoreUtil.getDimUseAsJoinKey(name));
+ if (isJoinKeyStr != null) {
+ isJoinKey = Boolean.parseBoolean(isJoinKeyStr);
}
- } else if (!this.getReferences().equals(other.getReferences())) {
- return false;
}
- return true;
- }
-
- @Override
- public String toString() {
- String str = super.toString();
- str += "references:" + getReferences();
- return str;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TableReference.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TableReference.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TableReference.java
index 995e9cb..7f2dc6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TableReference.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TableReference.java
@@ -26,8 +26,8 @@ public class TableReference {
}
public TableReference(String destTable, String destColumn) {
- this.destTable = destTable;
- this.destColumn = destColumn;
+ this.destTable = destTable.toLowerCase();
+ this.destColumn = destColumn.toLowerCase();
}
public TableReference(String reference) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 7128854..ed4f93a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -81,6 +81,13 @@ class CandidateTableResolver implements ContextRewriter {
LOG.info("Populating optional dim:" + dim);
populateDimTables(dim, cubeql, true);
}
+ if (cubeql.getAutoJoinCtx() != null) {
+ // Before checking for candidate table columns, prune join paths containing non existing columns
+ // in populated candidate tables
+ cubeql.getAutoJoinCtx().pruneAllPaths(cubeql.getCube(), cubeql.getCandidateFactTables(), null);
+ cubeql.getAutoJoinCtx().pruneAllPathsForCandidateDims(cubeql.getCandidateDimTables());
+ cubeql.getAutoJoinCtx().refreshJoinPathColumns();
+ }
checkForSourceReachabilityForDenormCandidates(cubeql);
// check for joined columns and denorm columns on refered tables
resolveCandidateFactTablesForJoins(cubeql);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index d52f9a9..bc15890 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -109,8 +109,7 @@ class JoinResolver implements ContextRewriter {
private String joinTypeCfg;
// Map of a joined table to its columns which are part of any of the join
- // paths. This is used
- // in candidate table resolver
+ // paths. This is used in candidate table resolver
Map<Dimension, Map<AbstractCubeTable, List<String>>> joinPathColumns =
new HashMap<Dimension, Map<AbstractCubeTable, List<String>>>();
@@ -141,6 +140,11 @@ class JoinResolver implements ContextRewriter {
jp.initColumnsForTable();
}
}
+ refreshJoinPathColumns();
+ }
+
+ public void refreshJoinPathColumns() {
+ joinPathColumns.clear();
for (Map.Entry<Dimension, List<SchemaGraph.JoinPath>> joinPathEntry : allPaths.entrySet()) {
List<SchemaGraph.JoinPath> joinPaths = joinPathEntry.getValue();
Map<AbstractCubeTable, List<String>> dimReachablePaths = joinPathColumns.get(joinPathEntry.getKey());
@@ -400,6 +404,42 @@ class JoinResolver implements ContextRewriter {
pruneAllPaths(dimsToQuery);
}
+ /**
+ * Prunes allPaths by removing paths which contain columns that are not present in any candidate dims.
+ *
+ * @param candidateDims
+ */
+ public void pruneAllPathsForCandidateDims(Map<Dimension, Set<CandidateDim>> candidateDims) {
+ Map<Dimension, Set<String>> dimColumns = new HashMap<Dimension, Set<String>>();
+ // populate all columns present in candidate dims for each dimension
+ for (Map.Entry<Dimension, Set<CandidateDim>> entry : candidateDims.entrySet()) {
+ Dimension dim = entry.getKey();
+ Set<String> allColumns = new HashSet<String>();
+ for (CandidateDim cdim : entry.getValue()) {
+ allColumns.addAll(cdim.getColumns());
+ }
+ dimColumns.put(dim, allColumns);
+ }
+ for (List<SchemaGraph.JoinPath> paths : allPaths.values()) {
+ for (int i = 0; i < paths.size(); i++) {
+ SchemaGraph.JoinPath jp = paths.get(i);
+ for (AbstractCubeTable refTable : jp.getAllTables()) {
+ List<String> cols = jp.getColumnsForTable(refTable);
+ if (refTable instanceof Dimension) {
+ if (cols != null && !dimColumns.get(refTable).containsAll(cols)) {
+ // This path requires some columns from the cube which are not present in any candidate dim
+ // Remove this path
+ LOG.info("Removing join path:" + jp + " as columns :" + cols + " dont exist");
+ paths.remove(i);
+ i--;
+ }
+ }
+ }
+ }
+ pruneEmptyPaths(allPaths);
+ }
+ }
+
private void pruneEmptyPaths(Map<Dimension, List<SchemaGraph.JoinPath>> allPaths) {
Iterator<Map.Entry<Dimension, List<SchemaGraph.JoinPath>>> iter = allPaths.entrySet().iterator();
while (iter.hasNext()) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 09dd5b9..e32ae7d 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -81,6 +81,7 @@ public class TestCubeMetastoreClient {
private static Set<CubeDimAttribute> stateAttrs = new HashSet<CubeDimAttribute>();
private static Set<CubeDimAttribute> countryAttrs = new HashSet<CubeDimAttribute>();
private static Set<ExprColumn> cubeExpressions = new HashSet<ExprColumn>();
+ private static Set<JoinChain> joinChains = new HashSet<JoinChain>();
private static Set<ExprColumn> dimExpressions = new HashSet<ExprColumn>();
/**
@@ -205,7 +206,29 @@ public class TestCubeMetastoreClient {
cubeDimensions.add(new InlineDimAttribute(new FieldSchema("region", "string", "region dim"), regions));
cubeDimensions.add(new InlineDimAttribute(new FieldSchema("regionstart", "string", "region dim"),
"Region with starttime", now, null, 100.0, regions));
- cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, new HashMap<String, String>(), 0.0);
+ JoinChain zipCity = new JoinChain("cityFromZip", "Zip City", "zip city desc");
+ List<TableReference> chain = new ArrayList<TableReference>();
+ chain.add(new TableReference(cubeName, "zipcode"));
+ chain.add(new TableReference("zipdim", "zipcode"));
+ chain.add(new TableReference("zipdim", "cityid"));
+ chain.add(new TableReference("citydim", "id"));
+ zipCity.addPath(chain);
+ List<TableReference> chain2 = new ArrayList<TableReference>();
+ chain2.add(new TableReference(cubeName, "zipcode2"));
+ chain2.add(new TableReference("zipdim", "zipcode"));
+ chain2.add(new TableReference("zipdim", "cityid"));
+ chain2.add(new TableReference("citydim", "id"));
+ zipCity.addPath(chain2);
+ joinChains.add(zipCity);
+ JoinChain cityChain = new JoinChain("city", "Cube City", "cube city desc");
+ chain = new ArrayList<TableReference>();
+ chain.add(new TableReference(cubeName, "cityid"));
+ chain.add(new TableReference("citydim", "id"));
+ cityChain.addPath(chain);
+ joinChains.add(cityChain);
+ cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("zipcityname", "string", "zip city name"),
+ "Zip city name", "cityFromZip", "name", null, null, null));
+ cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, new HashMap<String, String>(), 0.0);
measures = new HashSet<String>();
measures.add("msr1");
measures.add("msr2");
@@ -395,7 +418,7 @@ public class TestCubeMetastoreClient {
@Test(priority = 1)
public void testCube() throws Exception {
- client.createCube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, new HashMap<String, String>());
+ client.createCube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, new HashMap<String, String>());
Assert.assertTrue(client.tableExists(cubeName));
Table cubeTbl = client.getHiveTable(cubeName);
Assert.assertTrue(client.isCube(cubeTbl));
@@ -431,6 +454,35 @@ public class TestCubeMetastoreClient {
Assert.assertEquals(cube2.getExpressionByName("booleancut").getDisplayString(), "Boolean Cut");
Assert.assertTrue(cube2.allFieldsQueriable());
+ Assert.assertTrue(cube2.getJoinChainNames().contains("cityfromzip"));
+ Assert.assertTrue(cube2.getJoinChainNames().contains("city"));
+ Assert.assertFalse(cube2.getJoinChains().isEmpty());
+ Assert.assertEquals(cube2.getJoinChains().size(), 2);
+ JoinChain zipchain = cube2.getChainByName("cityfromzip");
+ Assert.assertEquals(zipchain.getDisplayString(), "Zip City");
+ Assert.assertEquals(zipchain.getDescription(), "zip city desc");
+ Assert.assertEquals(zipchain.getPaths().size(), 2);
+ Assert.assertEquals(zipchain.getPaths().get(0).size(), 4);
+ Assert.assertEquals(zipchain.getPaths().get(0).get(0).toString(), "testmetastorecube.zipcode");
+ Assert.assertEquals(zipchain.getPaths().get(0).get(1).toString(), "zipdim.zipcode");
+ Assert.assertEquals(zipchain.getPaths().get(0).get(2).toString(), "zipdim.cityid");
+ Assert.assertEquals(zipchain.getPaths().get(0).get(3).toString(), "citydim.id");
+ Assert.assertEquals(zipchain.getPaths().get(1).size(), 4);
+ Assert.assertEquals(zipchain.getPaths().get(1).get(0).toString(), "testmetastorecube.zipcode2");
+ Assert.assertEquals(zipchain.getPaths().get(1).get(1).toString(), "zipdim.zipcode");
+ Assert.assertEquals(zipchain.getPaths().get(1).get(2).toString(), "zipdim.cityid");
+ Assert.assertEquals(zipchain.getPaths().get(1).get(3).toString(), "citydim.id");
+ JoinChain citychain = cube2.getChainByName("city");
+ Assert.assertEquals(citychain.getDisplayString(), "Cube City");
+ Assert.assertEquals(citychain.getDescription(), "cube city desc");
+ Assert.assertEquals(citychain.getPaths().size(), 1);
+ Assert.assertEquals(citychain.getPaths().get(0).size(), 2);
+ Assert.assertEquals(citychain.getPaths().get(0).get(0).toString(), "testmetastorecube.cityid");
+ Assert.assertEquals(citychain.getPaths().get(0).get(1).toString(), "citydim.id");
+ Assert.assertNotNull(cube2.getDimAttributeByName("zipcityname"));
+ Assert.assertEquals(((ReferencedDimAtrribute)cube2.getDimAttributeByName("zipcityname")).getChainName(), "cityfromzip");
+ Assert.assertEquals(((ReferencedDimAtrribute)cube2.getDimAttributeByName("zipcityname")).getRefColumn(), "name");
+
client.createDerivedCube(cubeName, derivedCubeName, measures, dimensions, new HashMap<String, String>(), 0L);
Assert.assertTrue(client.tableExists(derivedCubeName));
Table derivedTbl = client.getHiveTable(derivedCubeName);
@@ -500,6 +552,14 @@ public class TestCubeMetastoreClient {
toAlter.addTimedDimension("zt");
toAlter.removeTimedDimension("dt");
+ JoinChain cityChain = new JoinChain("city", "Cube City", "cube city desc modified");
+ List<TableReference> chain = new ArrayList<TableReference>();
+ chain.add(new TableReference(cubeName, "cityid"));
+ chain.add(new TableReference("citydim", "id"));
+ cityChain.addPath(chain);
+ toAlter.alterJoinChain(cityChain);
+ toAlter.removeJoinChain("cityFromZip");
+
Assert.assertNotNull(toAlter.getMeasureByName("testAddMsr1"));
Assert.assertNotNull(toAlter.getMeasureByName("msr3"));
Assert.assertEquals(toAlter.getMeasureByName("msr3").getDisplayString(), "Measure3Altered");
@@ -531,6 +591,8 @@ public class TestCubeMetastoreClient {
Assert.assertNotNull(altered.getDimAttributeByName("dim1"));
Assert.assertEquals(altered.getDimAttributeByName("dim1").getDescription(), "basedim altered");
Assert.assertNull(altered.getDimAttributeByName("location2"));
+ Assert.assertNull(altered.getChainByName("cityFromZip"));
+ Assert.assertEquals(altered.getChainByName("city").getDescription(), "cube city desc modified");
toAlter.alterMeasure(new ColumnMeasure(new FieldSchema("testAddMsr1", "double", "testAddMeasure")));
client.alterCube(cubeName, toAlter);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index a985eda..19deea6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -1300,8 +1300,8 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
// Add referenced tables to visited list
if (graph.get(node) != null) {
for (SchemaGraph.TableRelationship edge : graph.get(node)) {
- if (!visited.contains(edge.getFromTable())) {
- toVisit.addLast(edge.getFromTable());
+ if (!visited.contains(edge.getToTable())) {
+ toVisit.addLast(edge.getToTable());
}
}
}
@@ -1353,4 +1353,4 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
release(sessionid);
return latestDate;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 3ae11db..7bbfd48 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -85,9 +85,16 @@ public class JAXBUtils {
}
}
+ Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>();
+ if (cube.getJoinchains() != null) {
+ for (XJoinchain xj : cube.getJoinchains().getChains()) {
+ joinchains.add(joinChainFromXJoinChain(xj));
+ }
+ }
+
Map<String, String> properties = mapFromXProperties(cube.getProperties());
double cubeWeight = cube.getWeight() == null ? 0d : cube.getWeight();
- return new Cube(cube.getName(), measures, dims, expressions, properties, cubeWeight);
+ return new Cube(cube.getName(), measures, dims, expressions, joinchains, properties, cubeWeight);
}
}
@@ -133,6 +140,14 @@ public class JAXBUtils {
xdmList.add(xDimAttrFromHiveDimAttr(cd));
}
xc.setDimAttributes(xdm);
+
+ XJoinchains xjc = XCF.createXJoinchains();
+ List<XJoinchain> chainSet = xjc.getChains();
+ for (JoinChain jc : c.getJoinChains()) {
+ chainSet.add(xJoinChainFromJoinChain(jc));
+ }
+ xjc.getChains().addAll(chainSet);
+ xc.setJoinchains(xjc);
}
return xc;
}
@@ -166,6 +181,15 @@ public class JAXBUtils {
endDate,
xd.getCost()
);
+ } else if (xd.getChainrefcolumn() != null) {
+ hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType(), xd.getDescription()),
+ xd.getDisplayString(),
+ xd.getChainrefcolumn().getChainName(),
+ xd.getChainrefcolumn().getRefcol(),
+ startDate,
+ endDate,
+ xd.getCost()
+ );
} else {
hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType(), xd.getDescription()),
xd.getDisplayString(),
@@ -253,7 +277,14 @@ public class JAXBUtils {
if (cd instanceof ReferencedDimAtrribute) {
ReferencedDimAtrribute rd = (ReferencedDimAtrribute) cd;
List<TableReference> dimRefs = rd.getReferences();
- xd.setReferences(xTabReferenceFromHiveTabReference(dimRefs));
+ if (rd.getChainName() != null) {
+ XChaincolumn xcc = new XChaincolumn();
+ xcc.setChainName(rd.getChainName());
+ xcc.setRefcol(rd.getRefColumn());
+ xd.setChainrefcolumn(xcc);
+ } else {
+ xd.setReferences(xTabReferenceFromHiveTabReference(dimRefs));
+ }
xd.setType(rd.getType());
xd.setCost(rd.getCost());
} else if (cd instanceof BaseDimAttribute) {
@@ -264,6 +295,21 @@ public class JAXBUtils {
return xd;
}
+ /**
+ * Create XJoinChain from cube join chain
+ */
+ public static XJoinchain xJoinChainFromJoinChain(JoinChain jc) {
+ XJoinchain xjc = XCF.createXJoinchain();
+ xjc.setName(jc.getName());
+ xjc.setDescription(jc.getDescription());
+ xjc.setDisplayString(jc.getDisplayString());
+
+ for (List<TableReference> path : jc.getPaths()) {
+ xjc.getPaths().add(xTabReferenceFromHiveTabReference(path));
+ }
+ return xjc;
+ }
+
public static XTablereferences xTabReferenceFromHiveTabReference(List<TableReference> hiveRefs) {
XTablereferences xrefs = XCF.createXTablereferences();
List<XTablereference> xrefList = xrefs.getTableReferences();
@@ -301,6 +347,27 @@ public class JAXBUtils {
return cm;
}
+ /**
+ * Create cube's JoinChain from JAXB counterpart
+ *
+ * @param xm
+ *
+ * @return {@link JoinChain}
+ */
+ public static JoinChain joinChainFromXJoinChain(XJoinchain xj) {
+ JoinChain jc = new JoinChain(xj.getName(), xj.getDisplayString(), xj.getDescription());
+ for (int i = 0; i < xj.getPaths().size(); i++) {
+ XTablereferences xchain = xj.getPaths().get(i);
+ List<TableReference> chain = new ArrayList<TableReference>(xchain.getTableReferences().size());
+
+ for (XTablereference xRef : xchain.getTableReferences()) {
+ chain.add(new TableReference(xRef.getDestTable(), xRef.getDestColumn()));
+ }
+ jc.addPath(chain);
+ }
+ return jc;
+ }
+
public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws ParseException {
ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType(), xe.getDescription()),
xe.getDisplayString(),
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/204a387a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 56958b6..ab67911 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -56,6 +56,7 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature;
import static org.testng.Assert.*;
+import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -256,7 +257,6 @@ public class TestMetastoreService extends LensJerseyTest {
c.add(GregorianCalendar.DAY_OF_MONTH, 7);
final XMLGregorianCalendar endDate = DatatypeFactory.newInstance().newXMLGregorianCalendar(c);
-
XCube cube = cubeObjectFactory.createXCube();
cube.setName(cubeName);
cube.setWeight(100.0);
@@ -280,11 +280,21 @@ public class TestMetastoreService extends LensJerseyTest {
xd2.setEndTime(endDate);
xd2.setCost(5.0);
+ XDimAttribute xd3 = cubeObjectFactory.createXDimAttribute();
+ xd3.setName("testdim2col2");
+ xd3.setType("string");
+ xd3.setDescription("ref chained dimension");
+ xd3.setDisplayString("Chained Dimension");
+ XChaincolumn xcc = new XChaincolumn();
+ xcc.setChainName("chain1");
+ xcc.setRefcol("col2");
+ xd3.setChainrefcolumn(xcc);
+
xdims.getDimAttributes().add(xd1);
xdims.getDimAttributes().add(xd2);
+ xdims.getDimAttributes().add(xd3);
cube.setDimAttributes(xdims);
-
XMeasures measures = cubeObjectFactory.createXMeasures();
XMeasure xm1 = new XMeasure();
@@ -312,6 +322,25 @@ public class TestMetastoreService extends LensJerseyTest {
measures.getMeasures().add(xm2);
cube.setMeasures(measures);
+ XJoinchains joinchains = cubeObjectFactory.createXJoinchains();
+
+ XJoinchain xj1 = new XJoinchain();
+ xj1.setName("chain1");
+ xj1.setDescription("first chain");
+ xj1.setDisplayString("Chain-1");
+ XTablereferences path1 = cubeObjectFactory.createXTablereferences();
+ XTablereference link1 = new XTablereference();
+ link1.setDestTable(cubeName);
+ link1.setDestColumn("col1");
+ XTablereference link2 = new XTablereference();
+ link2.setDestTable("testdim");
+ link2.setDestColumn("col1");
+ path1.getTableReferences().add(link1);
+ path1.getTableReferences().add(link2);
+ xj1.getPaths().add(path1);
+ joinchains.getChains().add(xj1);
+ cube.setJoinchains(joinchains);
+
XExpressions expressions = cubeObjectFactory.createXExpressions();
XExprColumn xe1 = new XExprColumn();
@@ -552,15 +581,31 @@ public class TestMetastoreService extends LensJerseyTest {
assertFalse(actual.isDerived());
assertNull(actual.getParent());
Cube hcube = (Cube) JAXBUtils.hiveCubeFromXCube(actual, null);
- assertNotNull(hcube.getDimAttributeByName("dim1"));
assertEquals(hcube.getDimAttributeByName("dim1").getDescription(), "first dimension");
assertEquals(hcube.getDimAttributeByName("dim1").getDisplayString(), "Dimension1");
+ assertNotNull(hcube.getDimAttributeByName("testdim2col2"));
+ assertEquals(hcube.getDimAttributeByName("testdim2col2").getDisplayString(), "Chained Dimension");
+ assertEquals(hcube.getDimAttributeByName("testdim2col2").getDescription(), "ref chained dimension");
+ assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getType(), "string");
+ assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getChainName(), "chain1");
+ assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getRefColumn(), "col2");
assertNotNull(hcube.getMeasureByName("msr1"));
assertEquals(hcube.getMeasureByName("msr1").getDescription(), "first measure");
assertEquals(hcube.getMeasureByName("msr1").getDisplayString(), "Measure1");
assertNotNull(hcube.getExpressionByName("expr1"));
assertEquals(hcube.getExpressionByName("expr1").getDescription(), "first expression");
assertEquals(hcube.getExpressionByName("expr1").getDisplayString(), "Expression1");
+ Assert.assertFalse(hcube.getJoinChains().isEmpty());
+ Assert.assertEquals(hcube.getJoinChains().size(), 1);
+ Assert.assertTrue(hcube.getJoinChainNames().contains("chain1"));
+ JoinChain chain1 = hcube.getChainByName("chain1");
+ Assert.assertEquals(chain1.getDisplayString(), "Chain-1");
+ Assert.assertEquals(chain1.getDescription(), "first chain");
+ Assert.assertEquals(chain1.getPaths().size(), 1);
+ List<TableReference> links = chain1.getPaths().get(0);
+ Assert.assertEquals(links.size(), 2);
+ Assert.assertEquals(links.get(0).toString(), "testgetcube.col1");
+ Assert.assertEquals(links.get(1).toString(), "testdim.col1");
final XCube dcube = createDerivedCube("testGetDerivedCube", "testGetCube");
target = target().path("metastore").path("cubes");
@@ -693,7 +738,7 @@ public class TestMetastoreService extends LensJerseyTest {
.queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType<JAXBElement<XCube>>() {});
XCube actual = got.getValue();
assertEquals(actual.getWeight(), 200.0);
- assertEquals(actual.getDimAttributes().getDimAttributes().size(), 3);
+ assertEquals(actual.getDimAttributes().getDimAttributes().size(), 4);
assertEquals(actual.getMeasures().getMeasures().size(), 3);
CubeInterface hcube = JAXBUtils.hiveCubeFromXCube(actual, null);
@@ -1994,6 +2039,7 @@ public class TestMetastoreService extends LensJerseyTest {
"flattestcube.msr2",
"flattestcube.dim1",
"flattestcube.dim2",
+ "flattestcube.testdim2col2",
"flattestcube.cubetodim1ref",
"flattestcube.expr1",
"flattestdim1.dim1todim2ref",