You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:47:32 UTC
[08/30] incubator-lens git commit: LENS-287: Fixes the exception that
was coming while describing cube(Amareshwari Sriramadasu via prongs)
LENS-287: Fixes the exception that was coming while describing cube(Amareshwari Sriramadasu via prongs)
Conflicts:
lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d27bf4ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d27bf4ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d27bf4ca
Branch: refs/heads/master
Commit: d27bf4ca31d30965223693b0866eee7024f70c5e
Parents: 1670cc1
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Wed Feb 11 11:32:25 2015 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Wed Feb 11 11:46:00 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 62 +++++++--------
.../apache/lens/server/metastore/JAXBUtils.java | 67 ++++++++--------
.../server/metastore/TestMetastoreService.java | 81 +++++++++++---------
3 files changed, 107 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d27bf4ca/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 568d1c9..6336b9e 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -249,13 +249,14 @@
</xs:documentation>
</xs:annotation>
<xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="x_column_type" name="type" use="required">
+ <xs:attribute type="xs:string" name="type" use="required">
<xs:annotation>
<xs:documentation>
- The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN,
- TINYINT,
- SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY,
- MAP, STRUCT
+ The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN, TINYINT,
+ SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
+ STRUCT, UNION
+ See hive represenation for specifying complex types -
+ https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
</xs:documentation>
</xs:annotation>
</xs:attribute>
@@ -317,7 +318,17 @@
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="x_column_type" name="type" use="required" />
+ <xs:attribute type="xs:string" name="type" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN,TINYINT,
+ SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
+ STRUCT, UNION
+ See hive represenation for specifying complex types -
+ https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
<xs:attribute type="xs:string" name="description" />
<xs:attribute type="xs:string" name="display_string">
<xs:annotation>
@@ -557,37 +568,20 @@
</xs:documentation>
</xs:annotation>
<xs:attribute name="name" type="xs:string" use="required" />
- <xs:attribute name="type" type="x_column_type" use="required" />
+ <xs:attribute type="xs:string" name="type" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN, TINYINT,
+ SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
+ STRUCT, UNION
+ See hive represenation for specifying complex types -
+ https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
<xs:attribute name="comment" type="xs:string" />
</xs:complexType>
- <xs:simpleType name="x_column_type">
- <xs:annotation>
- <xs:documentation>
- Column Types allowed.
- </xs:documentation>
- </xs:annotation>
- <xs:restriction base="xs:string">
- <xs:enumeration value="BOOLEAN" />
- <xs:enumeration value="TINYINT" />
- <xs:enumeration value="SMALLINT" />
- <xs:enumeration value="INT" />
- <xs:enumeration value="BIGINT" />
- <xs:enumeration value="FLOAT" />
- <xs:enumeration value="DOUBLE" />
- <xs:enumeration value="DECIMAL" />
- <xs:enumeration value="STRING" />
- <xs:enumeration value="CHAR" />
- <xs:enumeration value="VARCHAR" />
- <xs:enumeration value="DATE" />
- <xs:enumeration value="TIMESTAMP" />
- <xs:enumeration value="BINARY" />
- <xs:enumeration value="ARRAY" />
- <xs:enumeration value="MAP" />
- <xs:enumeration value="STRUCT" />
- </xs:restriction>
- </xs:simpleType>
-
<xs:simpleType name="x_measure_type">
<xs:annotation>
<xs:documentation>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d27bf4ca/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 329d899..7357d0c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -165,33 +165,32 @@ public class JAXBUtils {
for (XTableReference xRef : xd.getRefSpec().getTableReferences().getTableReference()) {
dimRefs.add(new TableReference(xRef.getTable(), xRef.getColumn()));
}
-
- hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().name().toLowerCase(),
- xd.getDescription()),
- xd.getDisplayString(),
- dimRefs,
- startDate,
- endDate,
- null
- );
+ hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
+ xd.getDescription()),
+ xd.getDisplayString(),
+ dimRefs,
+ startDate,
+ endDate,
+ null
+ );
} else if (xd.getRefSpec() != null && xd.getRefSpec().getChainRefColumn() != null) {
- hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().name().toLowerCase(),
- xd.getDescription()),
- xd.getDisplayString(),
- xd.getRefSpec().getChainRefColumn().getChainName(),
- xd.getRefSpec().getChainRefColumn().getRefCol(),
- startDate,
- endDate,
- null
- );
+ hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
+ xd.getDescription()),
+ xd.getDisplayString(),
+ xd.getRefSpec().getChainRefColumn().getChainName(),
+ xd.getRefSpec().getChainRefColumn().getRefCol(),
+ startDate,
+ endDate,
+ null
+ );
} else {
- hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType().name().toLowerCase(),
- xd.getDescription()),
- xd.getDisplayString(),
- startDate,
- endDate,
- null
- );
+ hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
+ xd.getDescription()),
+ xd.getDisplayString(),
+ startDate,
+ endDate,
+ null
+ );
}
return hiveDim;
@@ -252,7 +251,7 @@ public class JAXBUtils {
XExprColumn xe = XCF.createXExprColumn();
xe.setName(ec.getName());
- xe.setType(XColumnType.valueOf(ec.getType().toUpperCase()));
+ xe.setType(ec.getType());
xe.setDescription(ec.getDescription());
xe.setDisplayString(ec.getDisplayString());
xe.setExpr(ec.getExpr());
@@ -282,10 +281,10 @@ public class JAXBUtils {
refspec.getTableReferences().getTableReference().addAll(xTabReferencesFromHiveTabReferences(dimRefs));
}
xd.setRefSpec(refspec);
- xd.setType(XColumnType.valueOf(rd.getType().toUpperCase()));
+ xd.setType(rd.getType());
} else if (cd instanceof BaseDimAttribute) {
BaseDimAttribute bd = (BaseDimAttribute) cd;
- xd.setType(XColumnType.valueOf(bd.getType().toUpperCase()));
+ xd.setType(bd.getType());
}
return xd;
}
@@ -382,10 +381,10 @@ public class JAXBUtils {
}
public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws ParseException {
- ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().name().toLowerCase(),
- xe.getDescription()),
- xe.getDisplayString(),
- xe.getExpr());
+ ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(),
+ xe.getDescription()),
+ xe.getDisplayString(),
+ xe.getExpr());
return ec;
}
@@ -427,7 +426,7 @@ public class JAXBUtils {
return null;
}
- return new FieldSchema(c.getName(), c.getType().name().toLowerCase(), c.getComment());
+ return new FieldSchema(c.getName(), c.getType().toLowerCase(), c.getComment());
}
public static XColumn columnFromFieldSchema(FieldSchema fs) {
@@ -436,7 +435,7 @@ public class JAXBUtils {
}
XColumn c = XCF.createXColumn();
c.setName(fs.getName());
- c.setType(XColumnType.valueOf(fs.getType().toUpperCase()));
+ c.setType(fs.getType());
c.setComment(fs.getComment());
return c;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d27bf4ca/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index e29227c..92b85ff 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -262,7 +262,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd1 = cubeObjectFactory.createXDimAttribute();
xd1.setName("dim1");
- xd1.setType(XColumnType.STRING);
+ xd1.setType("STRING");
xd1.setDescription("first dimension");
xd1.setDisplayString("Dimension1");
// Don't set endtime on this dim to validate null handling on server side
@@ -270,7 +270,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd2 = cubeObjectFactory.createXDimAttribute();
xd2.setName("dim2");
- xd2.setType(XColumnType.INT);
+ xd2.setType("INT");
xd2.setDescription("second dimension");
xd2.setDisplayString("Dimension2");
// Don't set start time on this dim to validate null handling on server side
@@ -278,7 +278,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd3 = cubeObjectFactory.createXDimAttribute();
xd3.setName("testdim2col2");
- xd3.setType(XColumnType.STRING);
+ xd3.setType("STRING");
xd3.setDescription("ref chained dimension");
xd3.setDisplayString("Chained Dimension");
XChainColumn xcc = new XChainColumn();
@@ -287,9 +287,17 @@ public class TestMetastoreService extends LensJerseyTest {
xd3.setRefSpec(cubeObjectFactory.createXDimAttributeRefSpec());
xd3.getRefSpec().setChainRefColumn(xcc);
+ // add attribute with complex type
+ XDimAttribute xd4 = cubeObjectFactory.createXDimAttribute();
+ xd4.setName("dim4");
+ xd4.setType("struct<a:INT,b:array<string>,c:map<int,array<struct<x:int,y:array<int>>>");
+ xd4.setDescription("complex attribute");
+ xd4.setDisplayString("Complex Attribute");
+
cube.getDimAttributes().getDimAttribute().add(xd1);
cube.getDimAttributes().getDimAttribute().add(xd2);
cube.getDimAttributes().getDimAttribute().add(xd3);
+ cube.getDimAttributes().getDimAttribute().add(xd4);
XMeasure xm1 = new XMeasure();
xm1.setName("msr1");
@@ -356,7 +364,7 @@ public class TestMetastoreService extends LensJerseyTest {
XExprColumn xe1 = new XExprColumn();
xe1.setName("expr1");
- xe1.setType(XColumnType.DOUBLE);
+ xe1.setType("DOUBLE");
xe1.setDescription("first expression");
xe1.setDisplayString("Expression1");
xe1.setExpr("msr1/1000");
@@ -601,9 +609,11 @@ public class TestMetastoreService extends LensJerseyTest {
assertNotNull(hcube.getDimAttributeByName("testdim2col2"));
assertEquals(hcube.getDimAttributeByName("testdim2col2").getDisplayString(), "Chained Dimension");
assertEquals(hcube.getDimAttributeByName("testdim2col2").getDescription(), "ref chained dimension");
- assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getType(), "string");
- assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getChainName(), "chain1");
- assertEquals(((ReferencedDimAtrribute)hcube.getDimAttributeByName("testdim2col2")).getRefColumn(), "col2");
+ assertEquals(((BaseDimAttribute)hcube.getDimAttributeByName("dim4")).getType(),
+ "struct<a:int,b:array<string>,c:map<int,array<struct<x:int,y:array<int>>>");
+ assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getType(), "string");
+ assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getChainName(), "chain1");
+ assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getRefColumn(), "col2");
assertNotNull(hcube.getMeasureByName("msr1"));
assertEquals(hcube.getMeasureByName("msr1").getDescription(), "first measure");
assertEquals(hcube.getMeasureByName("msr1").getDisplayString(), "Measure1");
@@ -728,7 +738,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd2 = cubeObjectFactory.createXDimAttribute();
xd2.setName("dim3");
- xd2.setType(XColumnType.STRING);
+ xd2.setType("STRING");
cube.getDimAttributes().getDimAttribute().add(xd2);
XProperty xp = new XProperty();
@@ -745,8 +755,8 @@ public class TestMetastoreService extends LensJerseyTest {
JAXBElement<XCube> got =
target.path(cubeName)
.queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType<JAXBElement<XCube>>() {});
- XBaseCube actual = (XBaseCube)got.getValue();
- assertEquals(actual.getDimAttributes().getDimAttribute().size(), 4);
+ XBaseCube actual = (XBaseCube) got.getValue();
+ assertEquals(actual.getDimAttributes().getDimAttribute().size(), 5);
assertEquals(actual.getMeasures().getMeasure().size(), 3);
CubeInterface hcube = JAXBUtils.hiveCubeFromXCube(actual, null);
@@ -872,7 +882,7 @@ public class TestMetastoreService extends LensJerseyTest {
for (String timePartColName : timePartColNames) {
XColumn partCol = cubeObjectFactory.createXColumn();
partCol.setName(timePartColName);
- partCol.setType(XColumnType.STRING);
+ partCol.setType("STRING");
partCol.setComment("partition column");
xs1.getPartCols().getColumn().add(partCol);
xs1.getTimePartCols().add(timePartColName);
@@ -909,12 +919,12 @@ public class TestMetastoreService extends LensJerseyTest {
XColumn c1 = cubeObjectFactory.createXColumn();
c1.setName("col1");
- c1.setType(XColumnType.STRING);
+ c1.setType("STRING");
c1.setComment("Fisrt column");
dt.getColumns().getColumn().add(c1);
XColumn c2 = cubeObjectFactory.createXColumn();
c2.setName("col2");
- c2.setType(XColumnType.INT);
+ c2.setType("INT");
c2.setComment("Second column");
dt.getColumns().getColumn().add(c2);
@@ -960,7 +970,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd1 = cubeObjectFactory.createXDimAttribute();
xd1.setName("col1");
- xd1.setType(XColumnType.STRING);
+ xd1.setType("STRING");
xd1.setDescription("first column");
xd1.setDisplayString("Column1");
// Don't set endtime on this dim to validate null handling on server side
@@ -968,7 +978,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimAttribute xd2 = cubeObjectFactory.createXDimAttribute();
xd2.setName("col2");
- xd2.setType(XColumnType.INT);
+ xd2.setType("INT");
xd2.setDescription("second column");
xd2.setDisplayString("Column2");
// Don't set start time on this dim to validate null handling on server side
@@ -980,7 +990,7 @@ public class TestMetastoreService extends LensJerseyTest {
XExprColumn xe1 = new XExprColumn();
xe1.setName("dimexpr");
- xe1.setType(XColumnType.STRING);
+ xe1.setType("STRING");
xe1.setDescription("dimension expression");
xe1.setDisplayString("Dim Expression");
xe1.setExpr("substr(col1, 3)");
@@ -1090,7 +1100,7 @@ public class TestMetastoreService extends LensJerseyTest {
testDim.getAttributes().getDimAttribute().remove(1);
XDimAttribute xd1 = cubeObjectFactory.createXDimAttribute();
xd1.setName("col3");
- xd1.setType(XColumnType.STRING);
+ xd1.setType("STRING");
testDim.getAttributes().getDimAttribute().add(xd1);
APIResult result = target.path("testdim")
@@ -1208,7 +1218,7 @@ public class TestMetastoreService extends LensJerseyTest {
// Add a column
XColumn c = cubeObjectFactory.createXColumn();
c.setName("col3");
- c.setType(XColumnType.STRING);
+ c.setType("STRING");
c.setComment("Added column");
dt2.getColumns().getColumn().add(c);
@@ -1229,8 +1239,8 @@ public class TestMetastoreService extends LensJerseyTest {
List<XColumn> colList = dt3.getColumns().getColumn();
boolean foundCol = false;
for (XColumn col : colList) {
- if (col.getName().equals("col3") && col.getType().equals(XColumnType.STRING) &&
- "Added column".equalsIgnoreCase(col.getComment())) {
+ if (col.getName().equals("col3") && col.getType().equals("string")
+ && "Added column".equalsIgnoreCase(col.getComment())) {
foundCol = true;
break;
}
@@ -1417,13 +1427,13 @@ public class TestMetastoreService extends LensJerseyTest {
XColumn c1 = cubeObjectFactory.createXColumn();
c1.setName("c1");
- c1.setType(XColumnType.STRING);
+ c1.setType("STRING");
c1.setComment("col1");
f.getColumns().getColumn().add(c1);
XColumn c2 = cubeObjectFactory.createXColumn();
c2.setName("c2");
- c2.setType(XColumnType.STRING);
+ c2.setType("STRING");
c2.setComment("col2");
f.getColumns().getColumn().add(c2);
@@ -2086,19 +2096,20 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(tables, new HashSet<String>(Arrays.asList("flattestcube", "testdim", "testdim2")));
assertEquals(colSet, new HashSet<String>(Arrays.asList(
- "flattestcube.msr1",
- "flattestcube.msr2",
- "flattestcube.dim1",
- "flattestcube.dim2",
- "flattestcube.testdim2col2",
- "flattestcube.expr1",
- "chain1-testdim.col2",
- "chain1-testdim.col1",
- "chain1-testdim.dimexpr",
- "dim2chain-testdim2.col2",
- "dim2chain-testdim2.col1",
- "dim2chain-testdim2.dimexpr"
- )));
+ "flattestcube.msr1",
+ "flattestcube.msr2",
+ "flattestcube.dim1",
+ "flattestcube.dim2",
+ "flattestcube.testdim2col2",
+ "flattestcube.dim4",
+ "flattestcube.expr1",
+ "chain1-testdim.col2",
+ "chain1-testdim.col1",
+ "chain1-testdim.dimexpr",
+ "dim2chain-testdim2.col2",
+ "dim2chain-testdim2.col1",
+ "dim2chain-testdim2.dimexpr"
+ )));
// Now test flattened view for dimension
final WebTarget flatDimTarget = target().path("metastore").path("flattened").path("testdim");