You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/03/31 09:48:24 UTC
[16/16] lens git commit: LENS-1389: Back Merge with master
LENS-1389: Back Merge with master
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8868b063
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8868b063
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8868b063
Branch: refs/heads/lens-1381
Commit: 8868b063838050dd8746cefa491f143533e145c9
Parents: d45c538 d1b43d6
Author: Rajat Khandelwal <ra...@gmail.com>
Authored: Fri Mar 31 15:16:10 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Mar 31 15:16:10 2017 +0530
----------------------------------------------------------------------
contrib/clients/python/lens/client/main.py | 7 +++-
contrib/clients/python/lens/client/query.py | 43 +++++++++++++++-----
contrib/clients/python/setup.py | 2 +
contrib/clients/python/test/test_lensclient.py | 13 ++++--
.../lens/cube/metadata/CubeMetastoreClient.java | 22 +++++-----
.../cube/parse/DenormalizationResolver.java | 1 +
.../lens/cube/parse/ExpressionResolver.java | 2 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 2 -
.../metastore/CubeMetastoreServiceImpl.java | 2 -
tools/conf/server/lens-site.xml | 6 +++
10 files changed, 69 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --cc lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 7608a43,087c203..b445447
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@@ -307,74 -290,18 +307,74 @@@ public class CubeMetastoreClient
}
}
+ public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
+ Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
+ Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
+ throws LensException {
+ CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight,
+ properties, storageUpdatePeriodMap);
+ createCubeTable(factTable, storageTableDescs);
+ // do a get to update cache
+ getCubeFact(factName);
+
+ }
+ public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException {
+ if (entity instanceof XStorage) {
+ createStorage((XStorage) entity);
+ } else if (entity instanceof XCube) {
+ createCube((XCube)entity);
+ } else if (entity instanceof XDimension) {
+ createDimension((XDimension) entity);
+ } else if (entity instanceof XFactTable) {
+ createCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XDimensionTable) {
+ createCubeDimensionTable((XDimensionTable) entity);
+ } else if (entity instanceof XSegmentation) {
+ createSegmentation((XSegmentation) entity);
+ } else {
+ throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass());
+ }
+ }
+
+ public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity)
+ throws LensException, HiveException {
+ if (entity instanceof XStorage) {
+ alterStorage((XStorage) entity);
+ } else if (entity instanceof XCube) {
+ alterCube((XCube)entity);
+ } else if (entity instanceof XDimension) {
+ alterDimension((XDimension) entity);
+ } else if (entity instanceof XFactTable) {
+ alterCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XDimensionTable) {
+ alterCubeDimensionTable((XDimensionTable) entity);
+ } else if (entity instanceof XSegmentation) {
+ alterSegmentation((XSegmentation) entity);
+ } else {
+ throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass());
+ }
+ }
+
+
+ public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) {
+ Map<String, String> props = new HashMap<String, String>();
+ props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties()));
+ props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns()));
+ return props;
+ }
+ public void createCubeFactTable(XFactTable fact) throws LensException {
+ createCubeFactTable(fact.getCubeName(),
+ fact.getName(),
+ JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()),
+ JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()),
+ fact.getWeight(),
+ addFactColStartTimePropertyToFactProperties(fact),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
+ JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables()));
+ }
- public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
- Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
- Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
- throws LensException {
- CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight,
- properties, storageUpdatePeriodMap);
- createCubeTable(factTable, storageTableDescs);
- // do a get to update cache
- getCubeFact(factName);
-
- }
+
+
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
* storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --cc lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 76e5f23,cb26878..e5cf916
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@@ -138,9 -161,10 +138,10 @@@ public class DenormalizationResolver im
return null;
}
- public Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension,
- CandidateDim> dimsToQuery, boolean replaceFact) throws LensException {
+ Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql,
+ StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException {
Set<Dimension> refTbls = new HashSet<>();
- log.info("Doing denorm changes for fact :{}", cfact);
++ log.info("Doing denorm changes for fact :{}", sc);
if (!tableToRefCols.isEmpty()) {
// pick referenced columns for fact
http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --cc lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 860db28,9b29083..62d7386
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@@ -525,16 -511,1143 +525,14 @@@ public class CubeTestSetup
return expected.toString();
}
- private Set<ExprColumn> exprs;
- Set<ExprColumn> exprs;
-
- private void createCube(CubeMetastoreClient client) throws HiveException, ParseException, LensException {
- cubeMeasures = new HashSet<CubeMeasure>();
- Map<String, String> tags = new HashMap<>();
- tags.put(MetastoreConstants.MEASURE_DATACOMPLETENESS_TAG, "tag1");
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null,
- null, null, null, null, tags));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr2", "float", "second measure"), "Measure2", null, "SUM",
- "RS"));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr21", "float", "second measure"), "Measure22", null, "SUM",
- "RS"));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr22", "float", "second measure"), "Measure22", null, "SUM",
- "RS"));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr3", "double", "third measure"), "Measure3", null, "MAX",
- null));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null, "COUNT",
- null));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr9", "bigint", "ninth measure"), null, null, null, null,
- null, null, null, null, null, tags));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("noAggrMsr", "bigint", "measure without a default aggregate"),
- "No aggregateMsr", null, null, null));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"),
- "New measure", null, null, null, NOW, null, 100.0));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "fifteenth measure"), "Measure15", null, "SUM",
- "RS"));
-
- cubeDimensions = new HashSet<CubeDimAttribute>();
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("d_time", "timestamp", "d time")));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("processing_time", "timestamp", "processing time")));
- List<CubeDimAttribute> locationHierarchy = new ArrayList<CubeDimAttribute>();
- locationHierarchy.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "zip")));
- locationHierarchy.add(new BaseDimAttribute(new FieldSchema("cityid", "int", "city")));
- locationHierarchy.add(new BaseDimAttribute(new FieldSchema("stateid", "int", "state")));
- locationHierarchy.add(new BaseDimAttribute(new FieldSchema("countryid", "int", "country")));
- List<String> regions = Arrays.asList("APAC", "EMEA", "USA");
- locationHierarchy.add(new BaseDimAttribute(new FieldSchema("regionname", "string", "region"), "regionname", null,
- null, null, null, regions));
-
- cubeDimensions.add(new HierarchicalDimAttribute("location", "Location hierarchy", locationHierarchy));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "string", "basedim")));
- // Added for ambiguity test
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("ambigdim1", "string", "used in testColumnAmbiguity")));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2", "int", "ref dim"), "dim2 refer",
- "dim2chain", "id", null, null, 0.0));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cdim2", "int", "ref dim"), "Dim2 refer", NOW, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer",
- null, null, 10.0));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("unreachableName", "string", ""), "urdim name",
- "unreachableDim_chain", "name", null, null, 10.0));
- // denormalized reference
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big1", "bigint", "ref dim"), "dim2 refer",
- "dim2chain", "bigid1", null, null, 0.0));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("dim2big2", "bigint", "ref dim"), "dim2 refer",
- "dim2chain", "bigid2", null, null, 0.0));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim2bignew", "bigint", "ref dim"), "Dim2 refer",
- NOW, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id", "int", "ref dim"),
- "Timedim reference", null, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id", "int", "ref dim"),
- "Timedim reference", null, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_hour_id2", "int", "ref dim")));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("test_time_dim_day_id2", "int", "ref dim")));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("testDim3id", "string", "direct id to testdim3"),
- "dim3 refer", "dim3chain", "id", null, null, 0.0));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"),
- "city name", "cubecity", "name", null, null, 0.0));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"),
- "state name", "cubestate", "name", null, null, 0.0));
- List<ChainRefCol> references = new ArrayList<>();
- references.add(new ChainRefCol("timedatechain1", "full_date"));
- references.add(new ChainRefCol("timehourchain1", "full_hour"));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim", "date", "ref dim"),
- "Timedim full date", references, null, null, null, null));
- List<ChainRefCol> chainRefs = new ArrayList<>();
- chainRefs.add(new ChainRefCol("timehourchain2", "full_hour"));
- chainRefs.add(new ChainRefCol("timedatechain2", "full_date"));
- cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim2", "date", "chained dim"),
- "Timedim full date", chainRefs, null, null, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid1", "int", "id to city"),
- "City1", null, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"),
- "City2", null, null, null));
- cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"),
- "CityState", null, null, null));
-
- Map<String, JoinChain> joinChains = new HashMap<>();
- addCubeChains(joinChains, TEST_CUBE_NAME);
-
- exprs = new HashSet<ExprColumn>();
- exprs.add(new ExprColumn(new FieldSchema("avgmsr", "double", "avg measure"), "Avg Msr", "avg(msr1 + msr2)"));
- exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2expr", "double", "measure2"), "Msr2", "msr2)"));
- exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2qualifiedexpr", "double", "testcube.measure2"),
- "Msr2", "testcube.msr2"));
- exprs.add(new ExprColumn(new FieldSchema("singlecoldim1expr", "string", "dim1"), "dim1", "dim1)"));
- exprs.add(new ExprColumn(new FieldSchema("singlecoldim1qualifiedexpr", "string", "testcube.dim1"),
- "dim1", "testcube.dim1"));
- exprs.add(new ExprColumn(new FieldSchema("singlecolchainid", "string", "dim3chain.id"),
- "dim3chainid", "dim3chain.id)"));
- exprs.add(new ExprColumn(new FieldSchema("singlecolchainrefexpr", "string", "testcube.testDim3id"),
- "dim3chainid", "testcube.testDim3id"));
- exprs.add(new ExprColumn(new FieldSchema("singlecolchainfield", "string", "cubecity.name"),
- "cubecityname", "cubecity.name"));
- exprs.add(new ExprColumn(new FieldSchema("summsrs", "double", "sum measures"), "Sum Msrs",
- "(1000 + sum(msr1) + sum(msr2))/100"));
- exprs.add(new ExprColumn(new FieldSchema("msr5", "double", "materialized in some facts"), "Fifth Msr",
- "msr2 + msr3"));
- exprs.add(new ExprColumn(new FieldSchema("msr8", "double", "measure expression"), "Sixth Msr",
- "msr2 + msr3"));
- exprs.add(new ExprColumn(new FieldSchema("msr7", "double", "measure expression"), "Seventh Msr",
- "case when sum(msr2) = 0 then 0 else sum(case when cityid='x' then msr21 else msr22 end)/sum(msr2) end"));
- exprs.add(new ExprColumn(new FieldSchema("equalsums", "double", "sums are equals"), "equalsums",
- new ExprSpec("msr3 + msr4", null, null), new ExprSpec("(msr3 + msr2)/100", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("roundedmsr1", "double", "rounded measure1"), "Rounded msr1",
- "round(msr1/1000)"));
- exprs.add(new ExprColumn(new FieldSchema("roundedmsr2", "double", "rounded measure2"), "Rounded msr2",
- "round(msr2/1000)"));
- exprs.add(new ExprColumn(new FieldSchema("flooredmsr12", "double", "floored measure12"), "Floored msr12",
- "floor(msr12)"));
- exprs.add(new ExprColumn(new FieldSchema("nestedexpr", "double", "nested expr"), "Nested expr",
- new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null),
- new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end",
- null, null)));
- exprs.add(new ExprColumn(new FieldSchema("msr2expr", "double", "nested expr"), "Nested expr",
- new ExprSpec("case when cityStateName = 'xyz' then msr2 else 0 end", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("nestedExprWithTimes", "double", "nested expr"), "Nested expr",
- new ExprSpec("avg(roundedmsr2)", null, null), new ExprSpec("avg(equalsums)", null, null),
- new ExprSpec("case when substrexpr = 'xyz' then avg(msr5) when substrexpr = 'abc' then avg(msr4)/100 end",
- NOW, null), new ExprSpec("avg(newmeasure)", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("msr6", "bigint", "sixth measure"), "Measure6",
- "sum(msr2) + max(msr3)/ count(msr4)"));
- exprs.add(new ExprColumn(new FieldSchema("booleancut", "boolean", "a boolean expression"), "Boolean cut",
- "(dim1 != 'x' AND dim2 != 10)"));
- exprs.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a sub-string expression"), "Substr expr",
- new ExprSpec("substr(dim1, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("substrexprdim2", "string", "a sub-string expression"), "Substr expr",
- new ExprSpec("substr(dim2, 3))", null, null), new ExprSpec("substr(ascii(dim2chain.name), 3)", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("indiasubstr", "boolean", "nested sub string expression"), "Nested expr",
- "substrexpr = 'INDIA'"));
- exprs.add(new ExprColumn(new FieldSchema("refexpr", "string", "expression which facts and dimensions"),
- "Expr with cube and dim fields", "concat(dim1, \":\", citydim.name)"));
- exprs.add(new ExprColumn(new FieldSchema("nocolexpr", "string", "expression which non existing colun"),
- "No col expr", "myfun(nonexist)"));
- exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"),
- "new measure expr", "myfun(newmeasure)"));
- exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State",
- new ExprSpec("concat(cityname, \":\", statename_cube)", null, null),
- new ExprSpec("substr(concatedcitystate, 10)", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State",
- new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK),
- new ExprSpec("substr(concatedcitystate, 10)", null, null)));
- exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State",
- "concat('CityState:', cubecity.statename)"));
- exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state",
- "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'"));
- exprs.add(new ExprColumn(new FieldSchema("cubeStateName", "String", "statename from cubestate"), "CubeState Name",
- "substr(cubestate.name, 5)"));
- exprs.add(new ExprColumn(new FieldSchema("substrdim2big1", "String", "substr of dim2big1"), "dim2big1 substr",
- "substr(dim2big1, 5)"));
- exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii cityname"), "ascii cityname substr",
- "ascii(cityname)"));
- exprs.add(new ExprColumn(new FieldSchema("countofdistinctcityid", "int", "Count of Distinct CityId"),
- "Count of Distinct CityId Expr", "count(distinct(cityid))"));
- exprs.add(new ExprColumn(new FieldSchema("notnullcityid", "int", "Not null cityid"),
- "Not null cityid Expr", "case when cityid is null then 0 else cityid end"));
-
- Map<String, String> cubeProperties = new HashMap<String, String>();
- cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(TEST_CUBE_NAME),
- "d_time,pt,it,et,test_time_dim,test_time_dim2");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "pt", "pt");
- cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "test_time_dim+[-10 days,10 days]");
-
- client.createCube(TEST_CUBE_NAME, cubeMeasures, cubeDimensions, exprs, Sets.newHashSet(joinChains.values()),
- cubeProperties);
-
- Set<String> measures = new HashSet<String>();
- measures.add("msr1");
- measures.add("msr2");
- measures.add("msr3");
- measures.add("msr9");
- Set<String> dimensions = new HashSet<String>();
- dimensions.add("dim1");
- dimensions.add("dim2");
- dimensions.add("dim2big1");
- dimensions.add("dim2big2");
- dimensions.add("dim2bignew");
- // Try creating derived cube with non existant dim/measures
- try{
- client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME,
- Sets.newHashSet("random_measure"), Sets.newHashSet("random_dim_attribute"),
- new HashMap<String, String>(), 5L);
- } catch(LensException e) {
- assertTrue(e.getMessage().contains("random_measure"));
- assertTrue(e.getMessage().contains("random_dim_attribute"));
- assertTrue(e.getMessage().contains("not present"));
- }
- client.createDerivedCube(TEST_CUBE_NAME, DERIVED_CUBE_NAME,
- measures, dimensions, new HashMap<String, String>(), 5L);
- }
-
- private void addCubeChains(Map<String, JoinChain> joinChains, final String cubeName) {
- joinChains.put("timehourchain1", new JoinChain("timehourchain1", "time chain", "time dim thru hour dim") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "test_time_dim_hour_id"));
- add(new TableReference("hourdim", "id"));
- }
- });
- }
- });
- joinChains.put("timedatechain1", new JoinChain("timedatechain1", "time chain", "time dim thru date dim") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "test_time_dim_day_id"));
- add(new TableReference("daydim", "id"));
- }
- });
- }
- });
- joinChains.put("timehourchain2", new JoinChain("timehourchain2", "time chain", "time dim thru hour dim") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "test_time_dim_hour_id2"));
- add(new TableReference("hourdim", "id"));
- }
- });
- }
- });
- joinChains.put("timedatechain2", new JoinChain("timedatechain2", "time chain", "time dim thru date dim") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "test_time_dim_day_id2"));
- add(new TableReference("daydim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeCity", new JoinChain("cubeCity", "cube-city", "city thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "cityid"));
- add(new TableReference("citydim", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2"));
- add(new TableReference("testdim2", "id"));
- add(new TableReference("testdim2", "cityid"));
- add(new TableReference("citydim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeCity1", new JoinChain("cubeCity1", "cube-city", "city thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "cityid1"));
- add(new TableReference("citydim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeCity2", new JoinChain("cubeCity2", "cube-city", "city thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "cityid2"));
- add(new TableReference("citydim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeState", new JoinChain("cubeState", "cube-state", "state thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "stateid"));
- add(new TableReference("statedim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeZip", new JoinChain("cubeZip", "cube-zip", "Zipcode thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "zipcode"));
- add(new TableReference("zipdim", "code"));
- }
- });
- }
- });
- joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "countryid"));
- add(new TableReference("countrydim", "id"));
- }
- });
- }
- });
- joinChains.put("dim2chain", new JoinChain("dim2chain", "cube-testdim2", "testdim2 thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2"));
- add(new TableReference("testdim2", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big1"));
- add(new TableReference("testdim2", "bigid1"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big2"));
- add(new TableReference("testdim2", "bigid2"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2bignew"));
- add(new TableReference("testdim2", "bigidnew"));
- }
- });
- }
- });
- joinChains.put("dim3chain", new JoinChain("dim3chain", "cube-testdim3", "cyclicdim thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2"));
- add(new TableReference("testdim2", "id"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big1"));
- add(new TableReference("testdim2", "bigid1"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big2"));
- add(new TableReference("testdim2", "bigid2"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2bignew"));
- add(new TableReference("testdim2", "bigidnew"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- }
- });
- joinChains.put("dim4chain", new JoinChain("dim4chain", "cube-testdim3", "cyclicdim thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2"));
- add(new TableReference("testdim2", "id"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big1"));
- add(new TableReference("testdim2", "bigid1"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2big2"));
- add(new TableReference("testdim2", "bigid2"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "dim2bignew"));
- add(new TableReference("testdim2", "bigidnew"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- }
- });
- joinChains.put("cdimChain", new JoinChain("cdimChain", "cube-cyclicdim", "cyclicdim thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "cdim2"));
- add(new TableReference("cycledim1", "id"));
- }
- });
- }
- });
- joinChains.put("unreachableDim_chain", new JoinChain("unreachableDim_chain", "cube-unreachableDim",
- "unreachableDim thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "urdimid"));
- add(new TableReference("unreachableDim", "id"));
- }
- });
- }
- });
- joinChains.put("cubeCountry", new JoinChain("cubeCountry", "cube-country", "country thru cube") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference(cubeName, "countryid"));
- add(new TableReference("countrydim", "id"));
- }
- });
- }
- });
- }
- private void createBaseAndDerivedCubes(CubeMetastoreClient client)
- throws HiveException, ParseException, LensException {
- Set<CubeMeasure> cubeMeasures2 = new HashSet<>(cubeMeasures);
- Set<CubeDimAttribute> cubeDimensions2 = new HashSet<>(cubeDimensions);
- cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr11", "int", "first measure")));
- cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr12", "float", "second measure"), "Measure2", null, "SUM",
- "RS"));
- cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr13", "double", "third measure"), "Measure3", null, "MAX",
- null));
- cubeMeasures2.add(new ColumnMeasure(new FieldSchema("msr14", "bigint", "fourth measure"), "Measure4", null,
- "COUNT", null));
- cubeMeasures2.add(new ColumnMeasure(new FieldSchema("directMsr", "bigint", "fifth measure"), "Direct Measure",
- null, "SUM", null));
-
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim11", "string", "basedim")));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim12", "int", "ref dim"), "Dim2 refer",
- "dim2chain", "id", null, null, null)); // used as key in the chains
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim22", "int", "ref dim"), "Dim2 refer",
- "dim2chain", "id", null, null, null)); // not used as key in the chains
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim13", "string", "basedim")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("userid", "int", "userid")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("xuserid", "int", "userid")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("yuserid", "int", "userid")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_in_past", "int", "user_id_added_in_past")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_added_far_future", "int",
- "user_id_added_far_future")));
- cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("user_id_deprecated", "int", "user_id_deprecated")));
-
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("xsports", "array<string>", ""),
- "xuser sports", "xusersports", "name", null, null, null));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("ysports", "array<string>", ""),
- "yuser sports", "yusersports", "name", null, null, null));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sports", "array<string>", ""),
- "user sports", "usersports", "name", null, null, null));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("sportids", "array<int>", ""),
- "user sports", "userInterestIds", "sport_id", null, null, null));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("statecountry", "string", ""),
- "state country", "cubestatecountry", "name", null, null, null));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("citycountry", "string", ""),
- "city country", "cubecitystatecountry", "name", null, null, null));
- List<ChainRefCol> refCols = new ArrayList<>();
- refCols.add(new ChainRefCol("cubeState", "countrycapital"));
- refCols.add(new ChainRefCol("cubeCityStateCountry", "capital"));
- cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("cubeCountryCapital", "String", "ref dim"),
- "Country capital", refCols, null, null, null, null));
- Map<String, String> cubeProperties = new HashMap<>();
- cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(BASE_CUBE_NAME),
- "d_time,pt,it,et,test_time_dim,test_time_dim2");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim", "ttd");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "test_time_dim2", "ttd2");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "d_time", "dt");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "it", "it");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "et", "et");
- cubeProperties.put(MetastoreConstants.TIMEDIM_TO_PART_MAPPING_PFX + "processing_time", "pt");
- cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "d_time", "processing_time+[-5 days,5 days]");
- cubeProperties.put(MetastoreConstants.TIMEDIM_RELATION + "processing_time", "test_time_dim+[-5 days,5 days]");
- cubeProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "false");
-
- Map<String, JoinChain> joinChainMap = new HashMap<>();
- addCubeChains(joinChainMap, "basecube");
- // update new paths
- joinChainMap.get("dim2chain").addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "dim12"));
- add(new TableReference("testdim2", "id"));
- }
- });
- joinChainMap.get("dim3chain").addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "dim12"));
- add(new TableReference("testdim2", "id"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- }
- });
- joinChainMap.get("dim4chain").addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "dim12"));
- add(new TableReference("testdim2", "id"));
- add(new TableReference("testdim2", "testdim3id"));
- add(new TableReference("testdim3", "id"));
- add(new TableReference("testdim3", "testdim4id"));
- add(new TableReference("testdim4", "id"));
- }
- });
- Set<JoinChain> joinChains = Sets.newHashSet(joinChainMap.values());
- joinChains.add(new JoinChain("cityState", "city-state", "state thru city") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "cityid"));
- add(new TableReference("citydim", "id"));
- add(new TableReference("citydim", "stateid"));
- add(new TableReference("statedim", "id"));
- }
- });
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "cityid"));
- add(new TableReference("citydim", "id"));
- add(new TableReference("citydim", "statename"));
- add(new TableReference("statedim", "name"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("cityZip", "city-zip", "zip thru city") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "cityid"));
- add(new TableReference("citydim", "id"));
- add(new TableReference("citydim", "zipcode"));
- add(new TableReference("zipdim", "code"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("cubeStateCountry", "cube-state-country", "country through state") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "stateid"));
- add(new TableReference("statedim", "id"));
- add(new TableReference("statedim", "countryid"));
- add(new TableReference("countrydim", "id"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("cubeCityStateCountry", "cube-city-state-country", "country through state thru city") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "cityid"));
- add(new TableReference("citydim", "id"));
- add(new TableReference("citydim", "stateid"));
- add(new TableReference("statedim", "id"));
- add(new TableReference("statedim", "countryid"));
- add(new TableReference("countrydim", "id"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("userchain", "user-chain", "user chain") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "userid"));
- add(new TableReference("userdim", "id"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("user_id_added_far_future_chain", "user_id_added_far_future_chain",
- "user_id_added_far_future_chain") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "user_id_added_far_future"));
- add(new TableReference("userdim", "user_id_added_far_future"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("userSports", "user-sports", "user sports") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "userid"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("user_interests", "user_id", true));
- add(new TableReference("user_interests", "sport_id"));
- add(new TableReference("sports", "id"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("userInterestIds", "user-interestsIds", "user interest ids") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "userid"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("user_interests", "user_id", true));
- }
- });
- }
- });
- joinChains.add(new JoinChain("xuserSports", "xuser-sports", "xuser sports") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "xuserid"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("user_interests", "user_id", true));
- add(new TableReference("user_interests", "sport_id"));
- add(new TableReference("sports", "id"));
- }
- });
- }
- });
- joinChains.add(new JoinChain("yuserSports", "user-sports", "user sports") {
- {
- addPath(new ArrayList<TableReference>() {
- {
- add(new TableReference("basecube", "yuserid"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("userdim", "id"));
- add(new TableReference("user_interests", "user_id", true));
- add(new TableReference("user_interests", "sport_id"));
- add(new TableReference("sports", "id"));
- }
- });
- }
- });
-
- // add ref dim through chain
- cubeDimensions2.add(
- new ReferencedDimAttribute(new FieldSchema("cityStateCapital", "string", "State's capital thru city"),
- "State's capital thru city", "cityState", "capital", null, null, null));
- Set<ExprColumn> baseExprs = new HashSet<>(exprs);
- baseExprs.add(new ExprColumn(new FieldSchema("substrsprorts", "String", "substr of sports"), "substr sports",
- "substr(sports, 10)"));
- baseExprs.add(new ExprColumn(new FieldSchema("xsports_abbr", "array<string>", ""),
- "xuser sports", "substr(xsports, 3)"));
- baseExprs.add(new ExprColumn(new FieldSchema("ysports_abbr", "array<string>", ""),
- "yuser sports", "substr(ysports, 3)"));
- baseExprs.add(new ExprColumn(new FieldSchema("sports_abbr", "array<string>", ""),
- "user sports", "substr(sports, 3)"));
- baseExprs.add(new ExprColumn(new FieldSchema("sportids_abbr", "array<string>", ""),
- "user sports", "case when sportids == 1 then 'CKT' when sportids == 2 then 'FTB' else 'NON' end"));
- baseExprs.add(new ExprColumn(new FieldSchema("directMsrExpr", "bigint", ""),
- "Direct Measure", new ExprSpec("directMsr + 0", null, null), new ExprSpec("msr13 + msr14", null, null)));
- client.createCube(BASE_CUBE_NAME, cubeMeasures2, cubeDimensions2, baseExprs, joinChains, cubeProperties);
-
- Map<String, String> derivedProperties = new HashMap<>();
- derivedProperties.put(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE, "true");
- Set<String> measures = new HashSet<>();
- measures.add("msr1");
- measures.add("msr9");
- measures.add("msr11");
- Set<String> dimensions = new HashSet<>();
- dimensions.add("dim1");
- dimensions.add("dim11");
- dimensions.add("d_time");
- client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME1, measures, dimensions, derivedProperties, 5L);
-
- measures = new HashSet<>();
- measures.add("msr2");
- measures.add("msr12");
- measures.add("msr13");
- measures.add("msr14");
- measures.add("directmsr");
- dimensions = new HashSet<>();
- dimensions.add("cityid");
- dimensions.add("stateid");
- dimensions.add("userid");
- dimensions.add("xuserid");
- dimensions.add("yuserid");
- dimensions.add("dim1");
- dimensions.add("dim2");
- dimensions.add("dim2big1");
- dimensions.add("dim2big2");
- dimensions.add("dim2bignew");
- dimensions.add("dim11");
- dimensions.add("dim13");
- dimensions.add("dim12");
- dimensions.add("dim22");
- dimensions.add("d_time");
- dimensions.add("test_time_dim");
- dimensions.add("test_time_dim2");
- dimensions.add("test_time_dim_hour_id");
- dimensions.add("test_time_dim_day_id");
- dimensions.add("test_time_dim_hour_id2");
- dimensions.add("test_time_dim_day_id2");
- client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME2, measures, dimensions, derivedProperties, 10L);
- measures = new HashSet<>();
- measures.add("msr3");
- measures.add("msr13");
- dimensions = new HashSet<>();
- dimensions.add("dim1");
- dimensions.add("location");
- dimensions.add("d_time");
- dimensions.add("test_time_dim");
- dimensions.add("test_time_dim2");
- dimensions.add("test_time_dim_hour_id");
- dimensions.add("test_time_dim_day_id");
- dimensions.add("test_time_dim_hour_id2");
- dimensions.add("test_time_dim_day_id2");
- client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME3, measures, dimensions, derivedProperties, 20L);
-
- // create base cube facts
- createBaseCubeFacts(client);
- }
-
- private void createBaseCubeFacts(CubeMetastoreClient client) throws HiveException, LensException {
-
- Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
- Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>();
- updates.add(MINUTELY);
- updates.add(HOURLY);
- updates.add(DAILY);
- updates.add(MONTHLY);
- updates.add(QUARTERLY);
- updates.add(YEARLY);
-
- ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>();
- List<String> timePartCols = new ArrayList<String>();
- partCols.add(TestCubeMetastoreClient.getDatePartition());
- timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey());
-
- StorageTableDesc s1 = new StorageTableDesc();
- s1.setInputFormat(TextInputFormat.class.getCanonicalName());
- s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- s1.setPartCols(partCols);
- s1.setTimePartCols(timePartCols);
-
- StorageTableDesc s2 = new StorageTableDesc();
- s2.setInputFormat(TextInputFormat.class.getCanonicalName());
- s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- ArrayList<FieldSchema> s2PartCols = new ArrayList<FieldSchema>();
- s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition"));
- s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition"));
- s2.setPartCols(s2PartCols);
- s2.setTimePartCols(Arrays.asList("ttd", "ttd2"));
-
- storageAggregatePeriods.put(c1, updates);
- storageAggregatePeriods.put(c2, updates);
- storageAggregatePeriods.put(c3, updates);
- storageAggregatePeriods.put(c4, updates);
-
- Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c1, s1);
- storageTables.put(c4, s2);
- storageTables.put(c2, s1);
- storageTables.put(c3, s1);
-
- String factName = "testFact1_BASE";
- List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size());
- for (CubeMeasure measure : cubeMeasures) {
- factColumns.add(measure.getColumn());
- }
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("zipcode", "int", "zip"));
- factColumns.add(new FieldSchema("cityid", "int", "city id"));
- factColumns.add(new FieldSchema("stateid", "int", "state id"));
- factColumns.add(new FieldSchema("userid", "int", "user id"));
- factColumns.add(new FieldSchema("xuserid", "int", "user id"));
- factColumns.add(new FieldSchema("yuserid", "int", "user id"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
- factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id"));
-
- // create cube fact with materialized expressions
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
- factValidityProperties, storageTables);
-
- factName = "testFact5_BASE";
- factColumns = new ArrayList<>(cubeMeasures.size());
- for (CubeMeasure measure : cubeMeasures) {
- factColumns.add(measure.getColumn());
- }
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim"));
-
- // create cube fact
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L,
- factValidityProperties, storageTables);
-
- // create fact only with extra measures
- factName = "testFact2_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr12", "float", "second measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
- factColumns.add(new FieldSchema("dim2", "int", "dim2 id"));
- factColumns.add(new FieldSchema("userid", "int", "user id"));
- factColumns.add(new FieldSchema("xuserid", "int", "user id"));
- factColumns.add(new FieldSchema("yuserid", "int", "user id"));
- // create cube fact
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
- factValidityProperties, storageTables);
- Map<String, String> properties = Maps.newHashMap(factValidityProperties);
- properties.put(MetastoreConstants.FACT_ABSOLUTE_END_TIME, DateUtil.relativeToAbsolute("now.day - 2 days"));
- properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days"));
- client.createCubeFactTable(BASE_CUBE_NAME, "testfact_deprecated", factColumns, storageAggregatePeriods, 5L,
- properties, storageTables);
-
- // create fact only with extra measures
- factName = "testFact3_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr13", "double", "third measure"));
- factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
-
- // create cube fact
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
- factValidityProperties, storageTables);
-
- // create fact with materialized expression
- factName = "testFact6_BASE";
- factColumns = new ArrayList<>();
- factColumns.add(new FieldSchema("msr13", "double", "third measure"));
- factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("booleancut", "boolean", "expr dim"));
-
- // create cube fact
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 150L,
- factValidityProperties, storageTables);
-
- // create raw fact only with extra measures
- factName = "testFact2_RAW_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr11", "int", "first measure"));
- factColumns.add(new FieldSchema("msr12", "float", "second measure"));
- factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
- factColumns.add(new FieldSchema("dim13", "string", "base dim"));
- factColumns.add(new FieldSchema("dim12", "string", "base dim"));
- factColumns.add(new FieldSchema("dim22", "string", "base dim"));
- factColumns.add(new FieldSchema("cityid", "int", "city id"));
-
- storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
- updates = new HashSet<UpdatePeriod>();
- updates.add(HOURLY);
- storageAggregatePeriods.put(c1, updates);
-
- storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c1, s1);
-
- // create cube fact
- properties.clear();
- properties.putAll(factValidityProperties);
- properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false");
- properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2");
-
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
- storageTables);
-
- // create raw fact only with extra measures
- factName = "testFact3_RAW_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr13", "double", "third measure"));
- factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
- factColumns.add(new FieldSchema("dim12", "string", "base dim"));
-
- storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
- updates = new HashSet<UpdatePeriod>();
- updates.add(HOURLY);
- storageAggregatePeriods.put(c1, updates);
-
- storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c1, s1);
- properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01");
- properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01");
- properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01");
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
- storageTables);
-
- factName = "testFact4_RAW_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr13", "double", "third measure"));
- factColumns.add(new FieldSchema("msr14", "bigint", "fourth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("user_id_added_in_past", "int", "user id"));
- factColumns.add(new FieldSchema("user_id_added_far_future", "int", "user id"));
- factColumns.add(new FieldSchema("user_id_deprecated", "int", "user id"));
-
- storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c1, s1);
- properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_in_past"), "2016-01-01");
- properties.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("user_id_deprecated"), "2016-01-01");
- properties.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("user_id_added_far_future"), "2099-01-01");
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
- storageTables);
-
- factName = "testFact5_RAW_BASE";
- factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr9", "bigint", "ninth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
-
- properties.clear();
- properties.putAll(factValidityProperties);
- properties.put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, "false");
- properties.put(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG, "f2");
- client.createCubeFactTable(BASE_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
- storageTables);
-
- CubeFactTable fact = client.getFactTable(factName);
- // Add all hourly partitions for two days
- Calendar cal = Calendar.getInstance();
- cal.setTime(TWODAYS_BACK);
- Date temp = cal.getTime();
- while (!(temp.after(NOW))) {
- Map<String, Date> timeParts = new HashMap<String, Date>();
- timeParts.put("dt", temp);
- StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
- client.addPartition(sPartSpec, c1, CubeTableType.FACT);
- cal.add(HOUR_OF_DAY, 1);
- temp = cal.getTime();
- }
- }
-
- private void createCubeContinuousFact(CubeMetastoreClient client) throws Exception {
- // create continuous raw fact only with extra measures
- String factName = "testFact_CONTINUOUS";
- List<FieldSchema> factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr11", "double", "third measure"));
- factColumns.add(new FieldSchema("msr15", "int", "fifteenth measure"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
- factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
- factColumns.add(new FieldSchema("dim1", "string", "base dim"));
- factColumns.add(new FieldSchema("dim11", "string", "base dim"));
- factColumns.add(new FieldSchema("dim12", "string", "base dim"));
-
- Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
- Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>();
- updates.add(CONTINUOUS);
- storageAggregatePeriods.put(c0, updates);
-
- StorageTableDesc s0 = new StorageTableDesc();
- s0.setInputFormat(TextInputFormat.class.getCanonicalName());
- s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
-
- Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c0, s0);
- Map<String, String> properties = Maps.newHashMap(factValidityProperties);
- properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days"));
-
- client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
- storageTables);
- }
--
- private void createCubeFact(CubeMetastoreClient client) throws Exception {
+ private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exception {
String factName = "testFact";
- List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size());
- for (CubeMeasure measure : cubeMeasures) {
- if (!measure.getColumn().getName().equals("msr15")) { //do not add msr15
- factColumns.add(measure.getColumn());
- }
- }
- factColumns.add(new FieldSchema("msr5", "double", "msr5"));
-
- // add dimensions of the cube
- factColumns.add(new FieldSchema("zipcode", "int", "zip"));
- factColumns.add(new FieldSchema("cityid", "int", "city id"));
- factColumns.add(new FieldSchema("cityid1", "int", "city id"));
- factColumns.add(new FieldSchema("stateid", "int", "city id"));
- factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id"));
- factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id"));
- factColumns.add(new FieldSchema("ambigdim1", "string", "used in" + " testColumnAmbiguity"));
-
- Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
- Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>();
- updates.add(MINUTELY);
- updates.add(HOURLY);
- updates.add(DAILY);
- updates.add(MONTHLY);
- updates.add(QUARTERLY);
- updates.add(YEARLY);
-
- ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>();
- List<String> timePartCols = new ArrayList<String>();
- partCols.add(TestCubeMetastoreClient.getDatePartition());
- timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey());
-
- StorageTableDesc s1 = new StorageTableDesc();
- s1.setInputFormat(TextInputFormat.class.getCanonicalName());
- s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- s1.setPartCols(partCols);
- s1.setTimePartCols(timePartCols);
-
- StorageTableDesc s2 = new StorageTableDesc();
- s2.setInputFormat(TextInputFormat.class.getCanonicalName());
- s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- ArrayList<FieldSchema> s2PartCols = new ArrayList<FieldSchema>();
- s2PartCols.add(new FieldSchema("ttd", serdeConstants.STRING_TYPE_NAME, "test date partition"));
- s2PartCols.add(new FieldSchema("ttd2", serdeConstants.STRING_TYPE_NAME, "test date partition"));
- s2.setPartCols(s2PartCols);
- s2.setTimePartCols(Arrays.asList("ttd", "ttd2"));
-
- StorageTableDesc s3 = new StorageTableDesc();
- s3.setInputFormat(TextInputFormat.class.getCanonicalName());
- s3.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- s3.setPartCols(partCols);
- s3.setTimePartCols(timePartCols);
- s3.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 90 days");
- s3.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now.day - 10 days");
-
- StorageTableDesc s5 = new StorageTableDesc();
- s5.setInputFormat(TextInputFormat.class.getCanonicalName());
- s5.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- s5.setPartCols(partCols);
- s5.setTimePartCols(timePartCols);
- s5.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "now.day - 10 days");
-
- storageAggregatePeriods.put(c1, updates);
- storageAggregatePeriods.put(c2, updates);
- storageAggregatePeriods.put(c3, updates);
- storageAggregatePeriods.put(c4, updates);
- storageAggregatePeriods.put(c5, updates);
-
- Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
- storageTables.put(c1, s1);
- storageTables.put(c4, s2);
- storageTables.put(c2, s1);
- storageTables.put(c3, s3);
- storageTables.put(c5, s5);
-
- //add storage with continuous update period
- updates.add(CONTINUOUS);
- storageAggregatePeriods.put(c0, updates);
- StorageTableDesc s0 = new StorageTableDesc();
- s0.setInputFormat(TextInputFormat.class.getCanonicalName());
- s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
- storageTables.put(c0, s0);
-
- // create cube fact
- client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
- factValidityProperties, storageTables);
+
client.getTimelines(factName, c1, null, null);
client.getTimelines(factName, c4, null, null);
+
client.clearHiveTableCache();
+
CubeFactTable fact = client.getFactTable(factName);
Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1)));
assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
http://git-wip-us.apache.org/repos/asf/lens/blob/8868b063/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --cc lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index abaae5b,24660e1..194b380
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@@ -226,10 -232,24 +226,8 @@@ public class CubeMetastoreServiceImpl e
*/
@Override
public void createDimensionTable(LensSessionHandle sessionid, XDimensionTable xDimTable) throws LensException {
-- String dimTblName = xDimTable.getTableName();
- List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns());
- Map<String, UpdatePeriod> updatePeriodMap =
- JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables());
-
- Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties());
- Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables(
- xDimTable.getStorageTables());
--
try (SessionContext ignored = new SessionContext(sessionid)){
- log.info("# Columns: " + columns);
- getClient(sessionid).createCubeDimensionTable(xDimTable.getDimensionName(),
- dimTblName,
- columns,
- xDimTable.getWeight(),
- updatePeriodMap,
- properties,
- storageDesc);
+ getClient(sessionid).createCubeDimensionTable(xDimTable);
log.info("Dimension Table created " + xDimTable.getTableName());
}
}