You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/01/30 13:28:00 UTC
incubator-lens git commit: LENS-58 : Flattened columns should include
reachable fields through join chains (amareshwari
Repository: incubator-lens
Updated Branches:
refs/heads/master f78aef9c1 -> 3d6e714cf
LENS-58 : Flattened columns should include reachable fields through join chains (amareshwari
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/3d6e714c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/3d6e714c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/3d6e714c
Branch: refs/heads/master
Commit: 3d6e714cf2e59fac91d274d2ab92c21332bc9c0f
Parents: f78aef9
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Fri Jan 30 17:57:48 2015 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Fri Jan 30 17:57:48 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 69 +++--
.../metastore/CubeMetastoreServiceImpl.java | 172 +++++------
.../apache/lens/server/metastore/JAXBUtils.java | 34 ++-
.../server/metastore/MetastoreResource.java | 6 +-
.../server/metastore/TestMetastoreService.java | 303 ++++++++++---------
5 files changed, 306 insertions(+), 278 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3d6e714c/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index f75ea60..532feb6 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -114,7 +114,6 @@
<xs:attribute type="xs:dateTime" name="end_time"/>
<xs:attribute type="xs:double" name="cost"/>
<xs:attribute type="xs:string" name="unit"/>
- <xs:attribute type="xs:string" name="cube_table"/>
<xs:attribute type="xs:double" name="min"/>
<xs:attribute type="xs:double" name="max"/>
</xs:complexType>
@@ -152,7 +151,6 @@
<xs:attribute type="xs:string" name="description"/>
<xs:attribute type="xs:string" name="display_string"/>
<xs:attribute type="xs:string" name="expr"/>
- <xs:attribute type="xs:string" name="cube_table"/>
</xs:complexType>
<xs:complexType name="x_expressions">
@@ -174,7 +172,7 @@
</xs:annotation>
<xs:sequence>
<xs:element type="x_tablereferences" name="references"/>
- <xs:element type="x_chaincolumn" name="chainrefcolumn" maxOccurs="1" minOccurs="1"/>
+ <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="1" minOccurs="1"/>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="type" use="required"/>
@@ -184,12 +182,20 @@
<xs:attribute type="xs:dateTime" name="end_time"/>
<xs:attribute type="xs:double" name="cost"/>
<xs:attribute type="xs:string" name="expr"/>
- <xs:attribute type="xs:string" name="cube_table"/>
</xs:complexType>
- <xs:complexType name="x_chaincolumn">
- <xs:attribute type="xs:string" name="chainName" use="required"/>
- <xs:attribute type="xs:string" name="refcol" use="required"/>
+
+ <xs:complexType name="x_chain_column">
+ <xs:attribute type="xs:string" name="chain_name" use="required" />
+ <xs:attribute type="xs:string" name="ref_col" use="required" />
+ <xs:attribute type="xs:string" name="dest_table">
+ <xs:annotation>
+ <xs:documentation>
+ This will be the destination table of the chain. LENS will set and return, need not be set by end-user.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
</xs:complexType>
+
<xs:complexType name="x_tablereference">
<xs:attribute type="xs:string" name="dest_table" use="required"/>
<xs:attribute type="xs:string" name="dest_column" use="required"/>
@@ -199,13 +205,20 @@
<xs:element type="x_tablereference" name="table_references" maxOccurs="unbounded" minOccurs="1"/>
</xs:sequence>
</xs:complexType>
- <xs:complexType name="x_joinchain">
+ <xs:complexType name="x_join_chain">
<xs:sequence>
<xs:element type="x_tablereferences" name="paths" maxOccurs="unbounded" minOccurs="1"/>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="description"/>
<xs:attribute type="xs:string" name="display_string"/>
+ <xs:attribute type="xs:string" name="dest_table">
+ <xs:annotation>
+ <xs:documentation>
+ This will be the destination table of the chain. LENS will set and return, need not be set by end-user.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
</xs:complexType>
<xs:complexType name="x_joinchains">
<xs:annotation>
@@ -214,7 +227,7 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element type="x_joinchain" name="chains" maxOccurs="unbounded" minOccurs="1"/>
+ <xs:element type="x_join_chain" name="chains" maxOccurs="unbounded" minOccurs="1"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="x_dim_attributes">
@@ -448,14 +461,32 @@
<xs:attribute name="lastAccessTime" type="xs:int" use="required"/>
</xs:complexType>
- <xs:element name="flattened_columns">
- <xs:complexType>
- <xs:choice maxOccurs="unbounded">
- <xs:element name="measure" type="x_measure"/>
- <xs:element name="expression" type="x_expr_column"/>
- <xs:element name="dim_attribute" type="x_dim_attribute"/>
- </xs:choice>
- </xs:complexType>
- </xs:element>
-
+ <xs:element name="flattened_columns" type="flattened_columns" />
+ <xs:complexType name="flattened_columns">
+ <xs:annotation>
+ <xs:documentation>
+ LENS will use this type to return all flattened columns reachable from a cube or dimension.
+ This type is not expected to be created by end-users directly.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element name="flattened_column" type="flattened_column" maxOccurs="unbounded" minOccurs="0" />
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="flattened_column">
+ <xs:choice maxOccurs="1" minOccurs="1">
+ <xs:element name="measure" type="x_measure" />
+ <xs:element name="expression" type="x_expr_column" />
+ <xs:element name="dim_attribute" type="x_dim_attribute" />
+ </xs:choice>
+ <xs:attribute name="table_name" type="xs:string" use="required" />
+ <xs:attribute name="chain_name" type="xs:string">
+ <xs:annotation>
+ <xs:documentation>
+ For chained columns, the destination table of the chain will be returned in table_name and the
+ chain_name will be populated as well.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:complexType>
</xs:schema>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3d6e714c/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 9ef89f6..0275fba 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -1190,54 +1190,96 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
}
+ private void addAllMeasuresToFlattenedList(ObjectFactory objectFactory, CubeInterface cube,
+ List<FlattenedColumn> columnList) {
+ for (CubeMeasure msr : cube.getMeasures()) {
+ FlattenedColumn fcol = objectFactory.createFlattenedColumn();
+ fcol.setMeasure(JAXBUtils.xMeasureFromHiveMeasure(msr));
+ fcol.setTableName(cube.getName());
+ columnList.add(fcol);
+ }
+ }
+
+ private void addAllDirectAttributesToFlattenedListFromCube(ObjectFactory objectFactory, CubeInterface cube,
+ List<FlattenedColumn> columnList) {
+ AbstractBaseTable baseTbl = (AbstractBaseTable)(cube instanceof DerivedCube ?
+ ((DerivedCube)cube).getParent() : cube);
+ for (CubeDimAttribute dim : cube.getDimAttributes()) {
+ FlattenedColumn fcol = objectFactory.createFlattenedColumn();
+ fcol.setDimAttribute(JAXBUtils.xDimAttrFromHiveDimAttr(dim, baseTbl));
+ fcol.setTableName(cube.getName());
+ columnList.add(fcol);
+ }
+ }
+
+ private void addAllDirectAttributesToFlattenedListFromDimension(ObjectFactory objectFactory, Dimension dimension,
+ List<FlattenedColumn> columnList, String chainName) {
+ for (CubeDimAttribute cd : dimension.getAttributes()) {
+ FlattenedColumn fcol = objectFactory.createFlattenedColumn();
+ fcol.setDimAttribute(JAXBUtils.xDimAttrFromHiveDimAttr(cd, dimension));
+ fcol.setTableName(dimension.getName());
+ if (chainName != null) {
+ fcol.setChainName(chainName);
+ }
+ columnList.add(fcol);
+ }
+ }
+
+ private void addAllDirectExpressionsToFlattenedList(ObjectFactory objectFactory, AbstractBaseTable baseTbl,
+ List<FlattenedColumn> columnList, String chainName) {
+ for (ExprColumn expr : baseTbl.getExpressions()) {
+ FlattenedColumn fcol = objectFactory.createFlattenedColumn();
+ fcol.setExpression(JAXBUtils.xExprColumnFromHiveExprColumn(expr));
+ fcol.setTableName(baseTbl.getName());
+ if (chainName != null) {
+ fcol.setChainName(chainName);
+ }
+ columnList.add(fcol);
+ }
+ }
+
+ private void addAllChainedColsToFlattenedListFromCube(CubeMetastoreClient client, ObjectFactory objectFactory,
+ CubeInterface cube, List<FlattenedColumn> columnList) throws HiveException {
+ if (cube instanceof DerivedCube) {
+ return;
+ }
+ addAllChainedColsToFlattenedList(client, objectFactory, (AbstractBaseTable)cube, columnList);
+ }
+
+ private void addAllChainedColsToFlattenedList(CubeMetastoreClient client, ObjectFactory objectFactory,
+ AbstractBaseTable baseTbl, List<FlattenedColumn> columnList) throws HiveException {
+ for (JoinChain chain : baseTbl.getJoinChains()) {
+ Dimension dim = client.getDimension(chain.getDestTable());
+ addAllDirectAttributesToFlattenedListFromDimension(objectFactory, dim, columnList, chain.getName());
+ addAllDirectExpressionsToFlattenedList(objectFactory, dim, columnList, chain.getName());
+ }
+ }
+
@Override
public FlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName) throws LensException {
try {
acquire(sessionHandle);
CubeMetastoreClient client = getClient(sessionHandle);
- boolean isCube = false;
- AbstractCubeTable cubeTbl = null;
+ ObjectFactory objectFactory = new ObjectFactory();
+ FlattenedColumns flattenedColumns = objectFactory.createFlattenedColumns();
+ List<FlattenedColumn> columnList = flattenedColumns.getFlattenedColumn();
// check if the table is a cube or dimension
if (client.isCube(tableName)) {
- isCube = true;
CubeInterface cube = client.getCube(tableName);
- if (cube instanceof Cube) {
- cubeTbl = (Cube) cube;
- } else if (cube instanceof DerivedCube) {
- cubeTbl = (DerivedCube) cube;
- }
+ addAllMeasuresToFlattenedList(objectFactory, cube, columnList);
+ addAllDirectAttributesToFlattenedListFromCube(objectFactory, cube, columnList);
+ addAllDirectExpressionsToFlattenedList(objectFactory, (AbstractBaseTable)cube, columnList, null);
+ addAllChainedColsToFlattenedListFromCube(client, objectFactory, cube, columnList);
} else if (client.isDimension(tableName)) {
- cubeTbl = client.getDimension(tableName);
+ Dimension dimension = client.getDimension(tableName);
+ addAllDirectAttributesToFlattenedListFromDimension(objectFactory, dimension, columnList, null);
+ addAllDirectExpressionsToFlattenedList(objectFactory, (AbstractBaseTable)dimension, columnList, null);
+ addAllChainedColsToFlattenedList(client, objectFactory, dimension, columnList);
} else {
throw new BadRequestException("Can't get reachable columns. '"
+ tableName + "' is neither a cube nor a dimension");
}
-
- Map<String, CubeColumn> columnMap = getFlattenedColumnView(client, cubeTbl, isCube);
- // Convert this to the JAXB collection
- ObjectFactory objectFactory = new ObjectFactory();
- FlattenedColumns flattenedColumns = objectFactory.createFlattenedColumns();
- List<Object> columnList = flattenedColumns.getMeasureOrExpressionOrDimAttribute();
-
- for (Map.Entry<String, CubeColumn> entry : columnMap.entrySet()) {
- String colName = entry.getKey();
- CubeColumn column = entry.getValue();
- String table = colName.substring(0, colName.indexOf('.'));
- if (column instanceof CubeMeasure) {
- XMeasure xmeasure = JAXBUtils.xMeasureFromHiveMeasure((CubeMeasure) column);
- xmeasure.setCubeTable(table);
- columnList.add(xmeasure);
- } else if (column instanceof CubeDimAttribute) {
- XDimAttribute xDimAttribute = JAXBUtils.xDimAttrFromHiveDimAttr((CubeDimAttribute) column);
- xDimAttribute.setCubeTable(table);
- columnList.add(xDimAttribute);
- } else if (column instanceof ExprColumn) {
- XExprColumn xExprColumn = JAXBUtils.xExprColumnFromHiveExprColumn((ExprColumn) column);
- xExprColumn.setCubeTable(table);
- columnList.add(xExprColumn);
- }
- }
return flattenedColumns;
} catch (LensException exc) {
throw exc;
@@ -1248,68 +1290,6 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
}
- private Map<String, CubeColumn> getFlattenedColumnView(CubeMetastoreClient client,
- AbstractCubeTable table,
- boolean isCube) throws HiveException {
- SchemaGraph schemaGraph = client.getSchemaGraph();
- Map<AbstractCubeTable, Set<SchemaGraph.TableRelationship>> graph =
- isCube ? schemaGraph.getCubeGraph((CubeInterface) table) : schemaGraph.getDimOnlyGraph();
- Map<String, CubeColumn> columnMap = new LinkedHashMap<String, CubeColumn>();
-
- // Do a BFS over the schema graph
- LinkedList<AbstractCubeTable> toVisit = new LinkedList<AbstractCubeTable>();
- Set<AbstractCubeTable> visited = new HashSet<AbstractCubeTable>();
- toVisit.add(table);
-
- while (!toVisit.isEmpty()) {
- AbstractCubeTable node = toVisit.removeFirst();
- visited.add(node);
- String nodeName = node.getName();
- if (node instanceof CubeInterface) {
- Cube cube = null;
-
- if (node instanceof Cube) {
- cube = (Cube) node;
- } else if (node instanceof DerivedCube) {
- cube = ((DerivedCube) node).getParent();
- } else {
- continue;
- }
-
- // Add columns of the cube
- for (CubeMeasure measure : cube.getMeasures()) {
- columnMap.put(nodeName + "." + measure.getName(), measure);
- }
-
- for (CubeDimAttribute dimAttribute : cube.getDimAttributes()) {
- columnMap.put(nodeName + "." + dimAttribute.getName(), dimAttribute);
- }
-
- for (ExprColumn expression : cube.getExpressions()) {
- columnMap.put(nodeName + "." + expression.getName(), expression);
- }
- } else if (node instanceof Dimension) {
- Dimension dim = (Dimension) node;
- for (CubeDimAttribute dimAttribute : dim.getAttributes()) {
- columnMap.put(nodeName + "." + dimAttribute.getName(), dimAttribute);
- }
- } else {
- LOG.warn("Neither cube nor dimension " + node.getName());
- }
-
- // Add referenced tables to visited list
- if (graph.get(node) != null) {
- for (SchemaGraph.TableRelationship edge : graph.get(node)) {
- if (!visited.contains(edge.getToTable())) {
- toVisit.addLast(edge.getToTable());
- }
- }
- }
- } // end bfs
- // columnMap now contains flattened view
- return columnMap;
- }
-
@Override
public Date getLatestDateOfCube(LensSessionHandle sessionid, String cubeName, String timeDimension)
throws LensException, HiveException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3d6e714c/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 2a9566f..e62dfed 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -87,7 +87,7 @@ public class JAXBUtils {
Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>();
if (cube.getJoinchains() != null) {
- for (XJoinchain xj : cube.getJoinchains().getChains()) {
+ for (XJoinChain xj : cube.getJoinchains().getChains()) {
joinchains.add(joinChainFromXJoinChain(xj));
}
}
@@ -137,12 +137,12 @@ public class JAXBUtils {
XDimAttributes xdm = XCF.createXDimAttributes();
List<XDimAttribute> xdmList = xdm.getDimAttributes();
for (CubeDimAttribute cd : c.getDimAttributes()) {
- xdmList.add(xDimAttrFromHiveDimAttr(cd));
+ xdmList.add(xDimAttrFromHiveDimAttr(cd, (Cube)c));
}
xc.setDimAttributes(xdm);
XJoinchains xjc = XCF.createXJoinchains();
- List<XJoinchain> chainSet = xjc.getChains();
+ List<XJoinChain> chainSet = xjc.getChains();
for (JoinChain jc : c.getJoinChains()) {
chainSet.add(xJoinChainFromJoinChain(jc));
}
@@ -181,11 +181,11 @@ public class JAXBUtils {
endDate,
xd.getCost()
);
- } else if (xd.getChainrefcolumn() != null) {
+ } else if (xd.getChainRefColumn() != null) {
hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType(), xd.getDescription()),
xd.getDisplayString(),
- xd.getChainrefcolumn().getChainName(),
- xd.getChainrefcolumn().getRefcol(),
+ xd.getChainRefColumn().getChainName(),
+ xd.getChainRefColumn().getRefCol(),
startDate,
endDate,
xd.getCost()
@@ -268,7 +268,7 @@ public class JAXBUtils {
/**
* Create XDimension from hive ql cube dimension
*/
- public static XDimAttribute xDimAttrFromHiveDimAttr(CubeDimAttribute cd) {
+ public static XDimAttribute xDimAttrFromHiveDimAttr(CubeDimAttribute cd, AbstractBaseTable baseTable) {
XDimAttribute xd = XCF.createXDimAttribute();
xd.setName(cd.getName());
xd.setDescription(cd.getDescription());
@@ -278,10 +278,11 @@ public class JAXBUtils {
ReferencedDimAtrribute rd = (ReferencedDimAtrribute) cd;
List<TableReference> dimRefs = rd.getReferences();
if (rd.getChainName() != null) {
- XChaincolumn xcc = new XChaincolumn();
+ XChainColumn xcc = new XChainColumn();
xcc.setChainName(rd.getChainName());
- xcc.setRefcol(rd.getRefColumn());
- xd.setChainrefcolumn(xcc);
+ xcc.setRefCol(rd.getRefColumn());
+ xcc.setDestTable(baseTable.getChainByName(rd.getChainName()).getDestTable());
+ xd.setChainRefColumn(xcc);
} else {
xd.setReferences(xTabReferenceFromHiveTabReference(dimRefs));
}
@@ -298,11 +299,12 @@ public class JAXBUtils {
/**
* Create XJoinChain from cube join chain
*/
- public static XJoinchain xJoinChainFromJoinChain(JoinChain jc) {
- XJoinchain xjc = XCF.createXJoinchain();
+ public static XJoinChain xJoinChainFromJoinChain(JoinChain jc) {
+ XJoinChain xjc = XCF.createXJoinChain();
xjc.setName(jc.getName());
xjc.setDescription(jc.getDescription());
xjc.setDisplayString(jc.getDisplayString());
+ xjc.setDestTable(jc.getDestTable());
for (JoinChain.Path path : jc.getPaths()) {
xjc.getPaths().add(xTabReferenceFromHiveTabReference(path.getReferences()));
@@ -354,7 +356,7 @@ public class JAXBUtils {
*
* @return {@link JoinChain}
*/
- public static JoinChain joinChainFromXJoinChain(XJoinchain xj) {
+ public static JoinChain joinChainFromXJoinChain(XJoinChain xj) {
JoinChain jc = new JoinChain(xj.getName(), xj.getDisplayString(), xj.getDescription());
for (int i = 0; i < xj.getPaths().size(); i++) {
XTablereferences xchain = xj.getPaths().get(i);
@@ -786,7 +788,7 @@ public class JAXBUtils {
Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>();
if (dimension.getJoinchains() != null) {
- for (XJoinchain xj : dimension.getJoinchains().getChains()) {
+ for (XJoinChain xj : dimension.getJoinchains().getChains()) {
joinchains.add(joinChainFromXJoinChain(xj));
}
}
@@ -804,7 +806,7 @@ public class JAXBUtils {
XDimAttributes xdm = XCF.createXDimAttributes();
List<XDimAttribute> xdmList = xdm.getDimAttributes();
for (CubeDimAttribute cd : dimension.getAttributes()) {
- xdmList.add(xDimAttrFromHiveDimAttr(cd));
+ xdmList.add(xDimAttrFromHiveDimAttr(cd, dimension));
}
xd.setAttributes(xdm);
@@ -816,7 +818,7 @@ public class JAXBUtils {
xd.setExpressions(xexprs);
XJoinchains xjc = XCF.createXJoinchains();
- List<XJoinchain> chainSet = xjc.getChains();
+ List<XJoinChain> chainSet = xjc.getChains();
for (JoinChain jc : dimension.getJoinChains()) {
chainSet.add(xJoinChainFromJoinChain(jc));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3d6e714c/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 7194f6c..b156dc3 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -1444,17 +1444,17 @@ public class MetastoreResource {
*/
@GET
@Path("flattened/{tableName}")
- public FlattenedColumns getFlattenedColumns(
+ public JAXBElement<FlattenedColumns> getFlattenedColumns(
@QueryParam("sessionid") LensSessionHandle sessionid,
@PathParam("tableName") String tableName) {
checkSessionId(sessionid);
try {
- return getSvc().getFlattenedColumns(sessionid, tableName);
+ return xCubeObjectFactory.createFlattenedColumns(getSvc().getFlattenedColumns(sessionid, tableName));
} catch (LensException exc) {
throw new WebApplicationException(exc);
}
}
-
+
/**
* Get the latest available date upto which data is available for the base cubes, for the time dimension.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3d6e714c/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 8bf9f81..a861395 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -283,10 +283,10 @@ public class TestMetastoreService extends LensJerseyTest {
xd3.setType("string");
xd3.setDescription("ref chained dimension");
xd3.setDisplayString("Chained Dimension");
- XChaincolumn xcc = new XChaincolumn();
+ XChainColumn xcc = new XChainColumn();
xcc.setChainName("chain1");
- xcc.setRefcol("col2");
- xd3.setChainrefcolumn(xcc);
+ xcc.setRefCol("col2");
+ xd3.setChainRefColumn(xcc);
xdims.getDimAttributes().add(xd1);
xdims.getDimAttributes().add(xd2);
@@ -322,7 +322,7 @@ public class TestMetastoreService extends LensJerseyTest {
XJoinchains joinchains = cubeObjectFactory.createXJoinchains();
- XJoinchain xj1 = new XJoinchain();
+ XJoinChain xj1 = new XJoinChain();
xj1.setName("chain1");
xj1.setDescription("first chain");
xj1.setDisplayString("Chain-1");
@@ -337,6 +337,24 @@ public class TestMetastoreService extends LensJerseyTest {
path1.getTableReferences().add(link2);
xj1.getPaths().add(path1);
joinchains.getChains().add(xj1);
+
+ XJoinChain xj2 = new XJoinChain();
+ xj2.setName("dim2chain");
+ xj2.setDescription("testdim2 chain");
+ xj2.setDisplayString("Chain-2");
+ XTablereferences path = cubeObjectFactory.createXTablereferences();
+ path.getTableReferences().add(link1);
+ path.getTableReferences().add(link2);
+ XTablereference link3 = new XTablereference();
+ link3.setDestTable("testdim");
+ link3.setDestColumn("col2");
+ XTablereference link4 = new XTablereference();
+ link4.setDestTable("testdim2");
+ link4.setDestColumn("col1");
+ path.getTableReferences().add(link3);
+ path.getTableReferences().add(link4);
+ xj2.getPaths().add(path);
+ joinchains.getChains().add(xj2);
cube.setJoinchains(joinchains);
XExpressions expressions = cubeObjectFactory.createXExpressions();
@@ -575,6 +593,21 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(actual.getMeasures().getMeasures().size(), cube.getMeasures().getMeasures().size());
assertEquals(actual.getDimAttributes().getDimAttributes().size(), cube.getDimAttributes().getDimAttributes().size());
assertEquals(actual.getExpressions().getExpressions().size(), cube.getExpressions().getExpressions().size());
+ Map<String, XJoinChain> chains = new HashMap<String, XJoinChain>();
+ for (XJoinChain xjc : actual.getJoinchains().getChains()) {
+ chains.put(xjc.getName(), xjc);
+ }
+ assertEquals(chains.get("chain1").getDestTable(), "testdim");
+ assertEquals(chains.get("dim2chain").getDestTable(), "testdim2");
+ boolean chainValidated = false;
+ for (XDimAttribute attr : actual.getDimAttributes().getDimAttributes()) {
+ if (attr.getName().equalsIgnoreCase("testdim2col2")) {
+ assertEquals(attr.getChainRefColumn().getDestTable(), "testdim");
+ chainValidated = true;
+ break;
+ }
+ }
+ assertTrue(chainValidated);
assertEquals(actual.getWeight(), 100.0d);
assertFalse(actual.isDerived());
assertNull(actual.getParent());
@@ -594,7 +627,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(hcube.getExpressionByName("expr1").getDescription(), "first expression");
assertEquals(hcube.getExpressionByName("expr1").getDisplayString(), "Expression1");
Assert.assertFalse(hcube.getJoinChains().isEmpty());
- Assert.assertEquals(hcube.getJoinChains().size(), 1);
+ Assert.assertEquals(hcube.getJoinChains().size(), 2);
Assert.assertTrue(hcube.getJoinChainNames().contains("chain1"));
JoinChain chain1 = hcube.getChainByName("chain1");
Assert.assertEquals(chain1.getDisplayString(), "Chain-1");
@@ -1006,6 +1039,44 @@ public class TestMetastoreService extends LensJerseyTest {
return dimension;
}
+ private void createdChainedDimensions() throws Exception {
+ XDimension dimension = createDimension("testdim");
+ XDimension dimension2 = createDimension("testdim2");
+
+ XJoinchains joinchains = cubeObjectFactory.createXJoinchains();
+
+ XJoinChain xj1 = new XJoinChain();
+ xj1.setName("chain1");
+ xj1.setDescription("first chain");
+ xj1.setDisplayString("Chain-1");
+ XTablereferences path1 = cubeObjectFactory.createXTablereferences();
+ XTablereference link1 = new XTablereference();
+ link1.setDestTable("testdim");
+ link1.setDestColumn("col1");
+ XTablereference link2 = new XTablereference();
+ link2.setDestTable("testdim2");
+ link2.setDestColumn("col1");
+ path1.getTableReferences().add(link1);
+ path1.getTableReferences().add(link2);
+ xj1.getPaths().add(path1);
+ joinchains.getChains().add(xj1);
+ dimension.setJoinchains(joinchains);
+
+ final WebTarget target = target().path("metastore").path("dimensions");
+
+ // create
+ APIResult result = target.queryParam("sessionid", lensSessionId).request(
+ mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dimension)), APIResult.class);
+ assertNotNull(result);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+
+ // create
+ result = target.queryParam("sessionid", lensSessionId).request(
+ mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dimension2)), APIResult.class);
+ assertNotNull(result);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+
+ }
@Test
public void testDimension() throws Exception {
final String DB = dbPFX + "test_dimension";
@@ -1013,36 +1084,10 @@ public class TestMetastoreService extends LensJerseyTest {
createDatabase(DB);
setCurrentDatabase(DB);
try {
- XDimension dimension = createDimension("testdim");
- XDimension dimension2 = createDimension("testdim2");
-
- XJoinchains joinchains = cubeObjectFactory.createXJoinchains();
-
- XJoinchain xj1 = new XJoinchain();
- xj1.setName("chain1");
- xj1.setDescription("first chain");
- xj1.setDisplayString("Chain-1");
- XTablereferences path1 = cubeObjectFactory.createXTablereferences();
- XTablereference link1 = new XTablereference();
- link1.setDestTable("testdim");
- link1.setDestColumn("col1");
- XTablereference link2 = new XTablereference();
- link2.setDestTable("testdim2");
- link2.setDestColumn("col1");
- path1.getTableReferences().add(link1);
- path1.getTableReferences().add(link2);
- xj1.getPaths().add(path1);
- joinchains.getChains().add(xj1);
- dimension.setJoinchains(joinchains);
+ createdChainedDimensions();
final WebTarget target = target().path("metastore").path("dimensions");
- // create
- APIResult result = target.queryParam("sessionid", lensSessionId).request(
- mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dimension)), APIResult.class);
- assertNotNull(result);
- assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
-
// getall
StringList dimensions = target.queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class);
boolean foundDim = false;
@@ -1069,7 +1114,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(testDim.getJoinchains().getChains().get(0).getDescription(), "first chain");
assertEquals(testDim.getJoinchains().getChains().get(0).getDisplayString(), "Chain-1");
- Dimension dim = JAXBUtils.dimensionFromXDimension(dimension);
+ Dimension dim = JAXBUtils.dimensionFromXDimension(testDim);
assertNotNull(dim.getAttributeByName("col1"));
assertEquals(dim.getAttributeByName("col1").getDescription(), "first column");
assertEquals(dim.getAttributeByName("col1").getDisplayString(), "Column1");
@@ -1084,35 +1129,34 @@ public class TestMetastoreService extends LensJerseyTest {
XProperty prop = cubeObjectFactory.createXProperty();
prop.setName("dim.prop2.name");
prop.setValue("dim.prop2.value");
- dimension.getProperties().getProperties().add(prop);
+ testDim.getProperties().getProperties().add(prop);
- dimension.getAttributes().getDimAttributes().remove(1);
+ testDim.getAttributes().getDimAttributes().remove(1);
XDimAttribute xd1 = cubeObjectFactory.createXDimAttribute();
xd1.setName("col3");
xd1.setType("string");
- dimension.getAttributes().getDimAttributes().add(xd1);
- dimension.setWeight(200.0);
+ testDim.getAttributes().getDimAttributes().add(xd1);
+ testDim.setWeight(200.0);
- result = target.path("testdim")
+ APIResult result = target.path("testdim")
.queryParam("sessionid", lensSessionId)
- .request(mediaType).put(Entity.xml(cubeObjectFactory.createXDimension(dimension)), APIResult.class);
+ .request(mediaType).put(Entity.xml(cubeObjectFactory.createXDimension(testDim)), APIResult.class);
assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
- testDim = target.path("testdim").queryParam("sessionid", lensSessionId).request(mediaType).get(XDimension.class);
- assertEquals(testDim.getName(), "testdim");
- assertTrue(testDim.getProperties().getProperties().size() >= 2);
- assertTrue(JAXBUtils.mapFromXProperties(testDim.getProperties()).containsKey("dim.prop2.name"));
- assertEquals(JAXBUtils.mapFromXProperties(testDim.getProperties()).get("dim.prop2.name"), "dim.prop2.value");
- assertTrue(JAXBUtils.mapFromXProperties(testDim.getProperties()).containsKey("dimension.foo"));
- assertEquals(JAXBUtils.mapFromXProperties(testDim.getProperties()).get("dimension.foo"), "dim.bar");
- assertEquals(testDim.getWeight(), 200.0);
- assertEquals(testDim.getAttributes().getDimAttributes().size(), 2);
-
- dim = JAXBUtils.dimensionFromXDimension(testDim);
+ XDimension altered = target.path("testdim").queryParam("sessionid", lensSessionId).request(mediaType).get(XDimension.class);
+ assertEquals(altered.getName(), "testdim");
+ assertTrue(altered.getProperties().getProperties().size() >= 2);
+ assertTrue(JAXBUtils.mapFromXProperties(altered.getProperties()).containsKey("dim.prop2.name"));
+ assertEquals(JAXBUtils.mapFromXProperties(altered.getProperties()).get("dim.prop2.name"), "dim.prop2.value");
+ assertTrue(JAXBUtils.mapFromXProperties(altered.getProperties()).containsKey("dimension.foo"));
+ assertEquals(JAXBUtils.mapFromXProperties(altered.getProperties()).get("dimension.foo"), "dim.bar");
+ assertEquals(altered.getWeight(), 200.0);
+ assertEquals(altered.getAttributes().getDimAttributes().size(), 2);
+
+ dim = JAXBUtils.dimensionFromXDimension(altered);
System.out.println("Attributes:" + dim.getAttributes());
assertNotNull(dim.getAttributeByName("col3"));
- assertNull(dim.getAttributeByName("col2"));
- assertNotNull(dim.getAttributeByName("col1"));
+ assertTrue(dim.getAttributeByName("col2") == null || dim.getAttributeByName("col1") == null);
// drop the dimension
result = target.path("testdim")
@@ -2054,63 +2098,33 @@ public class TestMetastoreService extends LensJerseyTest {
}
}
+ private void populateActualTablesAndCols(List<FlattenedColumn> columns, Set<String> tables, Set<String> colSet) {
+ for (FlattenedColumn colObject : columns) {
+ String colStr;
+ tables.add(colObject.getTableName());
+ if (colObject.getMeasure() != null) {
+ colStr = colObject.getTableName() + "." + colObject.getMeasure().getName();
+ } else if (colObject.getDimAttribute() != null) {
+ colStr = (colObject.getChainName() != null ? colObject.getChainName() + "-" : "") + colObject.getTableName()
+ + "." + colObject.getDimAttribute().getName();
+ } else { // it will be expression
+ colStr = (colObject.getChainName() != null ? colObject.getChainName() + "-" : "") + colObject.getTableName()
+ + "." + colObject.getExpression().getName();
+ }
+ colSet.add(colStr);
+ }
+ }
+
@Test
public void testFlattenedView() throws Exception {
- final String DB = dbPFX + "test_native_tables";
+ final String DB = dbPFX + "test_flattened_view";
String prevDb = getCurrentDatabase();
createDatabase(DB);
setCurrentDatabase(DB);
try {
- XCube flatTestCube = createTestCube("flatTestCube");
- XDimAttributes dimAttrs = flatTestCube.getDimAttributes();
-
- GregorianCalendar c = new GregorianCalendar();
- c.setTime(new Date());
- final XMLGregorianCalendar startDate = DatatypeFactory.newInstance().newXMLGregorianCalendar(c);
-
- // Add link from cube to dim1
- XDimAttribute cubeToDim1Ref = cubeObjectFactory.createXDimAttribute();
- cubeToDim1Ref.setName("cubeToDim1Ref");
- cubeToDim1Ref.setType("string");
- cubeToDim1Ref.setDescription("test ref cube to dim1");
- cubeToDim1Ref.setDisplayString("cubeToDim1Ref");
- cubeToDim1Ref.setStartTime(startDate);
- // Don't set endtime on this dim to validate null handling on server side
- cubeToDim1Ref.setCost(10.0);
- XTablereferences xTablereference = cubeObjectFactory.createXTablereferences();
- cubeToDim1Ref.setReferences(xTablereference);
- List<XTablereference> refDimRefs = cubeToDim1Ref.getReferences().getTableReferences();
- XTablereference r = cubeObjectFactory.createXTablereference();
- r.setDestTable("flatTestDim1");
- r.setDestColumn("col1");
- refDimRefs.add(r);
- dimAttrs.getDimAttributes().add(cubeToDim1Ref);
-
- // Link from dim1 to dim2
- XDimension dim1 = createDimension("flatTestDim1");
- XDimAttributes dim1Attrs = dim1.getAttributes();
- XDimAttribute dim1ToDim2Ref = cubeObjectFactory.createXDimAttribute();
- dim1ToDim2Ref.setName("dim1ToDim2Ref");
- dim1ToDim2Ref.setType("string");
- dim1ToDim2Ref.setDescription("test ref dim1 to dim2");
- dim1ToDim2Ref.setDisplayString("dim1ToDim2Ref");
- dim1ToDim2Ref.setStartTime(startDate);
- // Don't set endtime on this dim to validate null handling on server side
- dim1ToDim2Ref.setCost(10.0);
- XTablereferences xTablereferences2 = cubeObjectFactory.createXTablereferences();
- dim1ToDim2Ref.setReferences(xTablereferences2);
- List<XTablereference> refs = dim1ToDim2Ref.getReferences().getTableReferences();
- XTablereference r2 = cubeObjectFactory.createXTablereference();
- r2.setDestTable("flatTestDim2");
- r2.setDestColumn("col1");
- refs.add(r2);
- dim1Attrs.getDimAttributes().add(dim1ToDim2Ref);
-
- XDimension dim2 = createDimension("flatTestDim2");
-
// Create the tables
-
+ XCube flatTestCube = createTestCube("flatTestCube");
// Create cube
final WebTarget cubeTarget = target().path("metastore").path("cubes");
APIResult result =
@@ -2119,73 +2133,74 @@ public class TestMetastoreService extends LensJerseyTest {
assertNotNull(result);
assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
- // Dim1
- final WebTarget dimTarget = target().path("metastore").path("dimensions");
- result = dimTarget.queryParam("sessionid", lensSessionId).request(
- mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dim1)), APIResult.class);
- assertNotNull(result);
- assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
-
- // Dim2
- result = dimTarget.queryParam("sessionid", lensSessionId).request(
- mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dim2)), APIResult.class);
- assertNotNull(result);
- assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ // create chained dimensions - testdim and testdim2
+ createdChainedDimensions();
// Now test flattened view
final WebTarget flatCubeTarget = target().path("metastore").path("flattened").path("flattestcube");
FlattenedColumns flattenedColumns = null;
- try {
- flattenedColumns =
- flatCubeTarget.queryParam("sessionid", lensSessionId).request().get(FlattenedColumns.class);
- } catch (Exception exc) {
- exc.printStackTrace();
- throw exc;
- }
+ JAXBElement<FlattenedColumns> actualElement = flatCubeTarget.queryParam("sessionid", lensSessionId).request()
+ .get(new GenericType<JAXBElement<FlattenedColumns>>() {});
+ flattenedColumns = actualElement.getValue();
assertNotNull(flattenedColumns);
- List<Object> columns = flattenedColumns.getMeasureOrExpressionOrDimAttribute();
+ List<FlattenedColumn> columns = flattenedColumns.getFlattenedColumn();
assertNotNull(columns);
assertTrue(!columns.isEmpty());
- int i = 0;
Set<String> tables = new HashSet<String>();
Set<String> colSet = new HashSet<String>();
- for (Object colObject : columns) {
- assertTrue(colObject instanceof XMeasure || colObject instanceof XDimAttribute || colObject instanceof XExprColumn);
- if (colObject instanceof XMeasure) {
- tables.add(((XMeasure) colObject).getCubeTable());
- colSet.add(((XMeasure) colObject).getCubeTable() + "." + ((XMeasure) colObject).getName() );
- } else if (colObject instanceof XDimAttribute) {
- tables.add(((XDimAttribute) colObject).getCubeTable());
- colSet.add(((XDimAttribute) colObject).getCubeTable() + "." + ((XDimAttribute) colObject).getName() );
- } else {
- tables.add(((XExprColumn) colObject).getCubeTable());
- colSet.add(((XExprColumn) colObject).getCubeTable() + "." + ((XExprColumn) colObject).getName() );
- }
- }
+ populateActualTablesAndCols(columns, tables, colSet);
- assertEquals(tables, new HashSet<String>(Arrays.asList("flattestcube", "flattestdim1", "flattestdim2")));
- assertEquals(colSet,new HashSet<String>(Arrays.asList(
+ assertEquals(tables, new HashSet<String>(Arrays.asList("flattestcube", "testdim", "testdim2")));
+ assertEquals(colSet, new HashSet<String>(Arrays.asList(
"flattestcube.msr1",
"flattestcube.msr2",
"flattestcube.dim1",
"flattestcube.dim2",
"flattestcube.testdim2col2",
- "flattestcube.cubetodim1ref",
"flattestcube.expr1",
- "flattestdim1.dim1todim2ref",
- "flattestdim1.col2",
- "flattestdim1.col1",
- "flattestdim2.col2",
- "flattestdim2.col1")));
+ "chain1-testdim.col2",
+ "chain1-testdim.col1",
+ "chain1-testdim.dimexpr",
+ "dim2chain-testdim2.col2",
+ "dim2chain-testdim2.col1",
+ "dim2chain-testdim2.dimexpr"
+ )));
+
+ // Now test flattened view for dimension
+ final WebTarget flatDimTarget = target().path("metastore").path("flattened").path("testdim");
+ actualElement = flatDimTarget.queryParam("sessionid", lensSessionId).request()
+ .get(new GenericType<JAXBElement<FlattenedColumns>>() {});
+ flattenedColumns = actualElement.getValue();
+ assertNotNull(flattenedColumns);
+
+ columns = flattenedColumns.getFlattenedColumn();
+ assertNotNull(columns);
+ assertTrue(!columns.isEmpty());
+
+ tables = new HashSet<String>();
+ colSet = new HashSet<String>();
+ populateActualTablesAndCols(columns, tables, colSet);
+
+ assertEquals(tables, new HashSet<String>(Arrays.asList("testdim", "testdim2")));
+ assertEquals(colSet, new HashSet<String>(Arrays.asList(
+ "testdim.col2",
+ "testdim.col1",
+ "testdim.dimexpr",
+ "chain1-testdim2.col2",
+ "chain1-testdim2.col1",
+ "chain1-testdim2.dimexpr"
+ )));
+
} finally {
dropDatabase(DB);
setCurrentDatabase(prevDb);
}
}
- private void createTestFactAndStorageTable(final String cubeName,final String[] storages, final String tableName, final String[] timePartColNames) {
+ private void createTestFactAndStorageTable(final String cubeName,final String[] storages, final String tableName,
+ final String[] timePartColNames) {
// Create a fact table object linked to cubeName
String[] updatePeriods = {"HOURLY"};