You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2017/06/06 08:30:17 UTC
[3/3] lens git commit: LENS-1412 : Add capability to define virtual
facts in a cube
LENS-1412 : Add capability to define virtual facts in a cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5de45e0f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5de45e0f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5de45e0f
Branch: refs/heads/master
Commit: 5de45e0f8212c2e03eaf9886b720e13868a1c42a
Parents: 34500f1
Author: Rajitha R <ra...@gmail.com>
Authored: Tue Jun 6 13:59:58 2017 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jun 6 13:59:58 2017 +0530
----------------------------------------------------------------------
.../lens/api/metastore/SchemaTraverser.java | 1 +
lens-api/src/main/resources/cube-0.1.xsd | 176 +++----
lens-api/src/main/resources/lens-errors.conf | 6 +
.../lens/cli/commands/LensFactCommands.java | 6 +-
.../lens/cli/commands/LensSchemaCommands.java | 18 +-
.../apache/lens/cli/TestLensFactCommands.java | 132 +++++-
.../apache/lens/cli/TestLensSchemaCommands.java | 2 +-
.../schema/cubes/base/virtual-cube.xml | 30 ++
.../resources/schema/facts/virtual_fact.xml | 28 ++
.../java/org/apache/lens/client/LensClient.java | 2 +-
.../apache/lens/client/LensMetadataClient.java | 18 +-
.../lens/cube/error/LensCubeErrorCode.java | 3 +-
.../lens/cube/metadata/AbstractCubeTable.java | 22 +-
.../lens/cube/metadata/CubeFactTable.java | 52 ++-
.../lens/cube/metadata/CubeMetastoreClient.java | 463 ++++++++++++++-----
.../cube/metadata/CubeVirtualFactTable.java | 186 ++++++++
.../apache/lens/cube/metadata/FactTable.java | 166 +++++++
.../apache/lens/cube/metadata/JAXBUtils.java | 24 +-
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../lens/cube/metadata/MetastoreUtil.java | 26 +-
.../apache/lens/cube/metadata/Segmentation.java | 13 +-
.../org/apache/lens/cube/parse/Candidate.java | 8 +-
.../lens/cube/parse/CandidateTableResolver.java | 11 +-
.../lens/cube/parse/StorageCandidate.java | 55 +--
.../cube/parse/StorageCandidateHQLContext.java | 12 +-
.../lens/cube/parse/StorageTableResolver.java | 5 +-
.../lens/cube/metadata/CubeFactTableTest.java | 12 +-
.../cube/metadata/TestCubeMetastoreClient.java | 123 ++++-
.../apache/lens/cube/parse/CubeTestSetup.java | 40 +-
.../lens/cube/parse/TestCubeRewriter.java | 16 +
.../parse/TestCubeSegmentationRewriter.java | 15 +-
.../lens/cube/parse/TestUnionQueries.java | 2 +-
.../resources/schema/cubes/base/virtualcube.xml | 36 ++
.../test/resources/schema/facts/virtualfact.xml | 27 ++
lens-driver-es/pom.xml | 3 +-
lens-examples/pom.xml | 3 +-
.../src/test/resources/yaml/fact1.yaml | 4 +-
.../src/test/resources/yaml/fact2.yaml | 4 +-
.../src/test/resources/yaml/rawfact.yaml | 4 +-
.../yaml/sales-aggr-continuous-fact.yaml | 4 +-
.../test/resources/yaml/sales-aggr-fact1.yaml | 4 +-
.../test/resources/yaml/sales-aggr-fact2.yaml | 4 +-
.../src/test/resources/yaml/sales-raw-fact.yaml | 4 +-
.../api/metastore/CubeMetastoreService.java | 8 +-
lens-server/pom.xml | 2 +-
.../metastore/CubeMetastoreServiceImpl.java | 24 +-
.../server/metastore/MetastoreResource.java | 17 +-
.../lens/server/common/RestAPITestUtil.java | 7 +-
.../server/metastore/TestMetastoreService.java | 274 ++++++++++-
src/site/apt/user/cli.apt | 3 +-
tools/scripts/generate-site-public.sh | 2 +-
51 files changed, 1646 insertions(+), 463 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
index 9564443..09c848a 100644
--- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
+++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
@@ -42,6 +42,7 @@ public class SchemaTraverser implements Runnable {
types.put("cubes/derived", XDerivedCube.class);
types.put("dimensions", XDimension.class);
types.put("facts", XFactTable.class);
+ types.put("facts/virtual", XVirtualFactTable.class);
types.put("dimtables", XDimensionTable.class);
types.put("dimensiontables", XDimensionTable.class);
types.put("dimensiontables", XDimensionTable.class);
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 8158e6d..4d10d24 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -168,76 +168,6 @@
</xs:complexContent>
</xs:complexType>
- <xs:element name="x_virtual_fact_table" type="x_virtual_fact_table"/>
- <xs:complexType name="x_virtual_fact_table">
- <xs:annotation>
- <xs:documentation>
- Virtual fact extends fact. It can override the cube of its source fact. It can have different
- properties associated with it than its source.
- Properties that can be set for a virtual fact are :
- - Filters which would be added in query rewriting
- </xs:documentation>
- </xs:annotation>
- <xs:sequence>
- <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0">
- <xs:annotation>
- <xs:documentation>
- Properties that can be set for a virtual fact are
- 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string would be added as an additional
- filter when the query is being constructed in the cube query writing phase.
- 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this,
- this fact is not a candidate. Time format can be as you would specify in the time_range_in clause.
- i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]]
- 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current time.
- Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just
- a time difference e.g. "-90 days". Or can be specified in relative syntax.
- e.g. now.year or now.day - 6 hour etc.
- 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of
- the fact will be valid here. Format same as absolute start time.
- 5. cube.fact.relative.end.time: You can specify the end date for fact table
- relative to current date e.g. fact table is valid for next 90days starting from today.
- This can be specified as just a time difference e.g. "+90 days"
-
- </xs:documentation>
- </xs:annotation>
- </xs:element>
- </xs:sequence>
- <xs:attribute type="xs:string" name="source_fact_name" use="required">
- <xs:annotation>
- <xs:documentation>
- The Source fact name over which the Virtual fact is defined.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="cube_name" use="required">
- <xs:annotation>
- <xs:documentation>
- The base cube's name to which the Virtual fact is associated.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute name="name" type="xs:string" use="required">
- <xs:annotation>
- <xs:documentation>
- The virtual fact table name.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute name="weight" use="optional" >
- <xs:annotation>
- <xs:documentation>
- The weight of the fact table. LENS will use this attribute to decide the lightest table to query when there
- are more than one eligible tables. If not defined, the source fact weight would be picked up.
- </xs:documentation>
- </xs:annotation>
- <xs:simpleType>
- <xs:restriction base="xs:double">
- <xs:minInclusive value="0"></xs:minInclusive>
- </xs:restriction>
- </xs:simpleType>
- </xs:attribute>
- </xs:complexType>
-
<xs:element name="x_derived_cube" type="x_derived_cube"/>
<xs:complexType name="x_derived_cube">
<xs:annotation>
@@ -1287,9 +1217,99 @@
</xs:sequence>
</xs:complexType>
+ <xs:element name="x_fact" type="x_fact"/>
+
+ <xs:complexType name="x_fact" abstract="true">
+ <xs:annotation>
+ <xs:documentation>
+ XFact can either be a Cube Fact for which the user would give the full specification of the
+ columns and storages
+ or can be a Virtual fact, for which the user would specify only the Source fact and filters if any.
+ Virtual fact derives its columns and storage specific details all from the source Fact. It can however have
+ different properties associated with it than the source.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="description"/>
+ <xs:attribute type="xs:string" name="cube_name" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The cube's name to which the fact is associated.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:complexType>
+
+ <xs:element name="x_virtual_fact_table" type="x_virtual_fact_table"/>
+ <xs:complexType name="x_virtual_fact_table">
+ <xs:complexContent>
+ <xs:extension base="x_fact">
+ <xs:annotation>
+ <xs:documentation>
+ Virtual fact extends fact. It can override the cube of its source fact. It can have different
+ properties associated with it than its source.
+ Properties that can be set for a virtual fact are :
+ - Filters which would be added in query rewriting
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ Properties that can be set for a virtual fact are :
+ 1. cube.fact.query.where.filter : filter string that needs to be added in WHERE clause. This string
+ would be added as an additional
+ filter when the query is being constructed in the cube query writing phase.
+ 2. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this,
+ this fact is not a candidate. Time format can be as you would specify in the time_range_in clause.
+ i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]]
+ 3. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current
+ time.
+ Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just
+ a time difference e.g. "-90 days". Or can be specified in relative syntax.
+ e.g. now.year or now.day - 6 hour etc.
+ 4. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data
+ of
+ the fact will be valid here. Format same as absolute start time.
+ 5. cube.fact.relative.end.time: You can specify the end date for fact table
+ relative to current date e.g. fact table is valid for next 90days starting from today.
+ This can be specified as just a time difference e.g. "+90 days"
+
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
+ </xs:sequence>
+
+ <xs:attribute type="xs:string" name="source_fact_name" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The Source fact name over which the Virtual fact is defined.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="weight" use="optional">
+ <xs:annotation>
+ <xs:documentation>
+ The weight of the fact table. LENS will use this attribute to decide the lightest table to query when
+ there
+ are more than one eligible tables. If not defined, the source fact weight would be picked up.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="xs:double">
+ <xs:minInclusive value="0"></xs:minInclusive>
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
+ </xs:complexType>
+
<xs:element name="x_fact_table" type="x_fact_table"/>
<xs:complexType name="x_fact_table">
+ <xs:complexContent>
+ <xs:extension base="x_fact">
<xs:annotation>
<xs:documentation>
Fact table that is associated to a base cube. The columns in the fact table will be a subset of
@@ -1334,20 +1354,6 @@
</xs:element>
<xs:element name="storage_tables" type="x_storage_tables" maxOccurs="1" minOccurs="0"/>
</xs:sequence>
- <xs:attribute name="name" type="xs:string" use="required">
- <xs:annotation>
- <xs:documentation>
- The fact table name.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute name="cube_name" type="xs:string" use="required">
- <xs:annotation>
- <xs:documentation>
- The base cube's name to which the fact_table is associated.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
<xs:attribute name="weight" use="required" >
<xs:annotation>
<xs:documentation>
@@ -1361,6 +1367,8 @@
</xs:restriction>
</xs:simpleType>
</xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
</xs:complexType>
<xs:element name="x_segmentation" type="x_segmentation"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/resources/lens-errors.conf
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf
index 43de1e9..fafd655 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -384,6 +384,12 @@ lensCubeErrorsForMetastore = [
errorMsg = "Partition filter can not be null or empty"
}
+ {
+ errorCode = 3106
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "Fact %s not of type %s"
+ }
+
]
lensDriverErrors = [
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index a01d6c0..a872998 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.util.List;
import org.apache.lens.api.APIResult;
-import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.metastore.XFact;
import org.apache.lens.api.metastore.XPartition;
import org.apache.lens.api.metastore.XStorageTableElement;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
@@ -39,7 +39,7 @@ import lombok.NonNull;
@Component
@UserDocumentation(title = "Commands for Facts Management",
description = "These command provide CRUD for facts, associated storages, and fact partitions")
-public class LensFactCommands extends LogicalTableCrudCommand<XFactTable> {
+public class LensFactCommands extends LogicalTableCrudCommand<XFact> {
/**
* Show facts.
@@ -352,7 +352,7 @@ public class LensFactCommands extends LogicalTableCrudCommand<XFactTable> {
}
@Override
- protected XFactTable doRead(String name) {
+ protected XFact doRead(String name) {
return getClient().getFactTable(name);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index 20f313a..aca1cf9 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -24,14 +24,7 @@ import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.lens.api.metastore.SchemaTraverser;
-import org.apache.lens.api.metastore.XBaseCube;
-import org.apache.lens.api.metastore.XDerivedCube;
-import org.apache.lens.api.metastore.XDimension;
-import org.apache.lens.api.metastore.XDimensionTable;
-import org.apache.lens.api.metastore.XFactTable;
-import org.apache.lens.api.metastore.XSegmentation;
-import org.apache.lens.api.metastore.XStorage;
+import org.apache.lens.api.metastore.*;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
import org.springframework.beans.factory.annotation.Autowired;
@@ -81,7 +74,12 @@ public class LensSchemaCommands implements CommandMarker {
+ "|\n"
+ "|-- facts\n"
+ " |-- fact1.xml\n"
- + " |-- fact2.xml\n\n\n"
+ + " |-- fact2.xml\n"
+ + "| |\n"
+ + "| |-- virtual\n"
+ + "| | |-- virtual_fact1.xml\n"
+ + "| | |-- virtual_fact2.xml\n"
+ + "| |\n\n\n"
+ "If your cubes are divided between base and derived cubes,\nit makes sense to seperate into two directories, "
+ "since derived cubes can't be created unless base cube exists.\nIn the other case you can keep them in the cubes "
+ "directory itself.\nFor dimtables, you can keep your schema files in a directory named either dimtables or "
@@ -114,6 +112,8 @@ public class LensSchemaCommands implements CommandMarker {
UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s");
CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s");
UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s");
+ CREATE_COMMAND_MAP.put(XVirtualFactTable.class, "create fact --path %s");
+ UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --name %s --path %s");
CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s");
UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s");
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index d31e25c..927b439 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -59,11 +59,21 @@ public class TestLensFactCommands extends LensCliApplicationTest {
@Test
public void testFactCommands() throws IOException, URISyntaxException {
createSampleCube();
+ createVirtualCube();
+
+ //test base fact
addFact1Table();
updateFact1Table();
testFactStorageActions();
testFactPartitionActions();
+
+ //test virtual fact
+ addVirtualFactTable();
+ updateVirtualFactTable();
+ dropVirtualFactTable();
+
dropFact1Table();
+ dropVirtualCube();
dropSampleCube();
}
@@ -76,10 +86,25 @@ public class TestLensFactCommands extends LensCliApplicationTest {
assertTrue(cubeList.contains("sample_cube"), cubeList);
}
+ private void createVirtualCube() throws URISyntaxException {
+ URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("schema/cubes/base/virtual-cube.xml");
+ String cubeList = getCubeCommand().showCubes();
+ assertFalse(cubeList.contains("virtualcube"), cubeList);
+ getCubeCommand().createCube(new File(cubeSpec.toURI()));
+ cubeList = getCubeCommand().showCubes();
+ assertTrue(cubeList.contains("virtualcube"), cubeList);
+ }
+
+
private void dropSampleCube() {
getCubeCommand().dropCube("sample_cube");
}
+
+ private void dropVirtualCube() {
+ getCubeCommand().dropCube("virtualcube");
+ }
+
private static LensFactCommands getCommand() {
if (command == null) {
LensClient client = new LensClient();
@@ -115,9 +140,9 @@ public class TestLensFactCommands extends LensCliApplicationTest {
*/
public static void addFact1Table() throws IOException {
LensFactCommands command = getCommand();
- String factList = command.showFacts(null);
+ String factList = command.showFacts("sample_cube");
assertEquals(command.showFacts("sample_cube"), "No fact found for sample_cube");
- assertEquals(factList, "No fact found", "Fact tables should not be found");
+ assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found");
// add local storage before adding fact table
TestLensStorageCommands.addLocalStorage(FACT_LOCAL);
URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/fact1.xml");
@@ -126,7 +151,7 @@ public class TestLensFactCommands extends LensCliApplicationTest {
} catch (Exception e) {
fail("Unable to create fact table" + e.getMessage());
}
- factList = command.showFacts(null);
+ factList = command.showFacts("sample_cube");
assertEquals(command.showFacts("sample_cube"), factList);
try {
assertEquals(command.showFacts("blah"), factList);
@@ -144,6 +169,40 @@ public class TestLensFactCommands extends LensCliApplicationTest {
}
/**
+ * Adds the virtual fact table.
+ *
+ * @throws IOException
+ */
+ public static void addVirtualFactTable() throws IOException {
+ LensFactCommands command = getCommand();
+ String factList = command.showFacts("virtualcube");
+ assertEquals(command.showFacts("virtualcube"), "No fact found for virtualcube");
+ assertEquals(factList, "No fact found for virtualcube", "Fact tables should not be found");
+ // add local storage before adding fact table
+ URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml");
+ try {
+ command.createFact(new File(factSpec.toURI()));
+ } catch (Exception e) {
+ fail("Unable to create virtual fact table" + e.getMessage());
+ }
+ factList = command.showFacts("virtualcube");
+ assertEquals(command.showFacts("virtualcube"), factList);
+ try {
+ assertEquals(command.showFacts("blah"), factList);
+ fail();
+ } catch (NotFoundException e) {
+ log.info("blah is not a table", e);
+ }
+ try {
+ assertEquals(command.showFacts("virtualfact"), factList);
+ fail();
+ } catch (NotFoundException e) {
+ log.info("virtualfact is a table, but not a cube table", e);
+ }
+ assertEquals("virtualfact", factList, "Virtualfact table should be found");
+ }
+
+ /**
* Update fact1 table.
*/
public static void updateFact1Table() {
@@ -196,6 +255,54 @@ public class TestLensFactCommands extends LensCliApplicationTest {
}
+
+ /**
+ * Update virtual fact table.
+ */
+ public static void updateVirtualFactTable() {
+ try {
+ LensFactCommands command = getCommand();
+ URL factSpec = TestLensFactCommands.class.getClassLoader().getResource("schema/facts/virtual_fact.xml");
+ StringBuilder sb = new StringBuilder();
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(factSpec.getFile()));
+ String s;
+ while ((s = bufferedReader.readLine()) != null) {
+ sb.append(s).append("\n");
+ }
+
+ bufferedReader.close();
+
+ String xmlContent = sb.toString();
+
+ xmlContent = xmlContent.replace("<property name=\"virtualfact.prop\" value=\"f1\"/>\n",
+ "<property name=\"virtualfact.prop\" value=\"f1\"/>\n"
+ + "<property name=\"virtualfact.prop1\" value=\"f2\"/>\n");
+
+ File newFile = new File("target/local-virtualfact.xml");
+ Writer writer = new OutputStreamWriter(new FileOutputStream(newFile));
+ writer.write(xmlContent);
+ writer.close();
+
+ String desc = command.describeFactTable("virtualfact");
+ log.debug(desc);
+ String propString = "virtualfact.prop: f1";
+ String propString1 = "virtualfact.prop1: f2";
+
+ command.updateFactTable("virtualfact", new File("target/local-virtualfact.xml"));
+ desc = command.describeFactTable("virtualfact");
+ log.debug(desc);
+ assertTrue(desc.contains(propString), "The sample property value is not set");
+ assertTrue(desc.contains(propString1), "The sample property value is not set");
+
+ newFile.delete();
+
+ } catch (Throwable t) {
+ log.error("Updating of the virtualfact table failed with ", t);
+ fail("Updating of the virtualfact table failed with " + t.getMessage());
+ }
+
+ }
+
/**
* Test fact storage actions.
*/
@@ -336,11 +443,24 @@ public class TestLensFactCommands extends LensCliApplicationTest {
*/
public static void dropFact1Table() {
LensFactCommands command = getCommand();
- String factList = command.showFacts(null);
+ String factList = command.showFacts("sample_cube");
assertEquals("fact1", factList, "Fact1 table should be found");
command.dropFact("fact1", false);
- factList = command.showFacts(null);
- assertEquals(factList, "No fact found", "Fact tables should not be found");
+ factList = command.showFacts("sample_cube");
+ assertEquals(factList, "No fact found for sample_cube", "Fact tables should not be found");
TestLensStorageCommands.dropStorage(FACT_LOCAL);
}
+
+
+ /**
+ * Drop virtualfact table.
+ */
+ public static void dropVirtualFactTable() {
+ LensFactCommands command = getCommand();
+ String factList = command.showFacts("virtualcube");
+ assertEquals("virtualfact", factList, "Virtualfact table should be found");
+ command.dropFact("virtualfact", false);
+ factList = command.showFacts("virtualcube");
+ assertEquals(factList, "No fact found for virtualcube", "Virtual Fact tables should not be found");
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java
index ca6db2c..4a23532 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensSchemaCommands.java
@@ -37,7 +37,7 @@ public class TestLensSchemaCommands extends LensCLITest {
assertTrue(((String) execute("show databases")).contains(dbName));
execute("show storages", "local");
execute("show dimensions", "test_detail\ntest_dim");
- execute("show cubes", "sample_cube\ncube_with_no_weight_facts");
+ execute("show cubes", "virtualcube\nsample_cube\ncube_with_no_weight_facts");
assertTrue(((String) execute("show dimtables")).contains("dim_table"));
assertTrue(((String) execute("show facts")).contains("fact1"));
execute("show segmentations", "seg1");
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml
new file mode 100644
index 0000000..8c57cee
--- /dev/null
+++ b/lens-cli/src/test/resources/schema/cubes/base/virtual-cube.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="virtualcube" xmlns="uri:lens:cube:0.1">
+ <properties>
+ <property name="sample_cube.prop" value="sample" />
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt" />
+ </properties>
+ <measures>
+ <measure name="measure1" _type="BIGINT" />
+ </measures>
+</x_base_cube>
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cli/src/test/resources/schema/facts/virtual_fact.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/schema/facts/virtual_fact.xml b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml
new file mode 100644
index 0000000..81e2676
--- /dev/null
+++ b/lens-cli/src/test/resources/schema/facts/virtual_fact.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_virtual_fact_table source_fact_name="fact1" cube_name="virtualcube" name="virtualfact" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/>
+ <property name="virtualfact.prop" value="f1"/>
+ </properties>
+</x_virtual_fact_table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 4f8da06..1de99fe 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -514,7 +514,7 @@ public class LensClient implements AutoCloseable {
return mc.updateDimension(dimName, dimSpec);
}
- public XFactTable getFactTable(String factName) {
+ public XFact getFactTable(String factName) {
return mc.getFactTable(factName);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index f077c9c..8a05952 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -425,12 +425,12 @@ public class LensMetadataClient {
.delete());
}
- public XFactTable getFactTable(String factTableName) {
+ public XFact getFactTable(String factTableName) {
WebTarget target = getMetastoreWebTarget();
- JAXBElement<XFactTable> table = target.path("facts").path(factTableName)
+ JAXBElement<XFact> table = target.path("facts").path(factTableName)
.queryParam("sessionid", this.connection.getSessionHandle())
.request(MediaType.APPLICATION_XML)
- .get(new GenericType<JAXBElement<XFactTable>>() {
+ .get(new GenericType<JAXBElement<XFact>>() {
});
return table.getValue();
}
@@ -445,17 +445,17 @@ public class LensMetadataClient {
return seg.getValue();
}
- public APIResult createFactTable(XFactTable f) {
+ public APIResult createFactTable(XFact f) {
WebTarget target = getMetastoreWebTarget();
return translate(target.path("facts")
.queryParam("sessionid", this.connection.getSessionHandle())
.request(MediaType.APPLICATION_XML)
- .post(Entity.xml(new GenericEntity<JAXBElement<XFactTable>>(objFact.createXFactTable(f)){})));
+ .post(Entity.xml(new GenericEntity<JAXBElement<XFact>>(objFact.createXFact(f)){})));
}
public APIResult createFactTable(String factSpec) {
try {
- return createFactTable(this.<XFactTable>readFromXML(factSpec));
+ return createFactTable(this.<XFact>readFromXML(factSpec));
} catch (JAXBException | IOException e) {
return failureAPIResult(e);
}
@@ -478,17 +478,17 @@ public class LensMetadataClient {
}
}
- public APIResult updateFactTable(String factName, XFactTable table) {
+ public APIResult updateFactTable(String factName, XFact table) {
WebTarget target = getMetastoreWebTarget();
return translate(target.path("facts").path(factName)
.queryParam("sessionid", this.connection.getSessionHandle())
.request(MediaType.APPLICATION_XML_TYPE)
- .put(Entity.xml(new GenericEntity<JAXBElement<XFactTable>>(objFact.createXFactTable(table)){})));
+ .put(Entity.xml(new GenericEntity<JAXBElement<XFact>>(objFact.createXFact(table)){})));
}
public APIResult updateFactTable(String factName, String table) {
try {
- return updateFactTable(factName, this.<XFactTable>readFromXML(table));
+ return updateFactTable(factName, this.<XFact>readFromXML(table));
} catch (JAXBException | IOException e) {
return failureAPIResult(e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index 32b9db3..ed076e2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -65,7 +65,8 @@ public enum LensCubeErrorCode {
TIMELINE_ABSENT(3102, 100),
EXPRESSION_NOT_PARSABLE(3103, 1500),
ENTITY_NOT_FOUND(3104, 1500),
- NO_PARTITION_FILTER(3105, 1500);
+ NO_PARTITION_FILTER(3105, 1500),
+ ENTITY_TYPE_NOT_AS_EXPECTED(3106, 1500);
public LensErrorInfo getLensErrorInfo() {
return this.errorInfo;
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
index 67aaff8..624b294 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
@@ -20,9 +20,6 @@ package org.apache.lens.cube.metadata;
import java.util.*;
-import org.apache.lens.server.api.error.LensException;
-
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -67,7 +64,7 @@ public abstract class AbstractCubeTable implements Named {
protected void addProperties() {
properties.put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name());
- properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight));
+ properties.put(MetastoreUtil.getCubeTableWeightKey(name), String.valueOf(weight()));
}
public String getName() {
@@ -182,23 +179,6 @@ public abstract class AbstractCubeTable implements Named {
return true;
}
- public Date getDateFromProperty(String propKey, boolean relative, boolean start) {
- String prop = getProperties().get(propKey);
- try {
- if (StringUtils.isNotBlank(prop)) {
- if (relative) {
- return DateUtil.resolveRelativeDate(prop, now());
- } else {
- return DateUtil.resolveAbsoluteDate(prop);
- }
- }
- } catch (LensException e) {
- log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop);
- }
- return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE;
- }
-
-
@Override
public String toString() {
return getName();
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index 88bc1fc..c57a9c1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -34,7 +34,7 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class CubeFactTable extends AbstractCubeTable {
+public class CubeFactTable extends AbstractCubeTable implements FactTable {
@Getter
// Map<StorageName, Map<update_period, storage_table_prefix>>
private final Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap;
@@ -44,7 +44,7 @@ public class CubeFactTable extends AbstractCubeTable {
public CubeFactTable(Table hiveTable) {
super(hiveTable);
this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties());
- this.cubeName = getCubeName(getName(), getProperties());
+ this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName()));
this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties());
}
@@ -77,7 +77,7 @@ public class CubeFactTable extends AbstractCubeTable {
}
public boolean hasColumn(String column) {
- List<String> validColumns = getValidColumns();
+ Set<String> validColumns = getValidColumns();
if (validColumns != null) {
return validColumns.contains(column);
} else {
@@ -88,7 +88,7 @@ public class CubeFactTable extends AbstractCubeTable {
@Override
protected void addProperties() {
super.addProperties();
- addCubeNames(getName(), getProperties(), cubeName);
+ this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName);
addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods);
addStorageTableProperties(getName(), getProperties(), storagePrefixUpdatePeriodMap);
}
@@ -115,10 +115,6 @@ public class CubeFactTable extends AbstractCubeTable {
}
}
- private static void addCubeNames(String factName, Map<String, String> props, String cubeName) {
- props.put(MetastoreUtil.getFactCubeNameKey(factName), cubeName);
- }
-
private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) {
Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>();
for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) {
@@ -156,10 +152,6 @@ public class CubeFactTable extends AbstractCubeTable {
return storageUpdatePeriods;
}
- static String getCubeName(String factName, Map<String, String> props) {
- return props.get(MetastoreUtil.getFactCubeNameKey(factName));
- }
-
public Map<String, Set<UpdatePeriod>> getUpdatePeriods() {
return storageUpdatePeriods;
}
@@ -276,10 +268,11 @@ public class CubeFactTable extends AbstractCubeTable {
*
* @return
*/
- public List<String> getValidColumns() {
+ public Set<String> getValidColumns() {
String validColsStr =
MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName()));
- return validColsStr == null ? null : Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','));
+ return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(),
+ ',')));
}
/**
@@ -374,7 +367,7 @@ public class CubeFactTable extends AbstractCubeTable {
*/
public void alterCubeName(String cubeName) {
this.cubeName = cubeName;
- addCubeNames(getName(), getProperties(), cubeName);
+ this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName);
}
public String getDataCompletenessTag() {
@@ -390,12 +383,28 @@ public class CubeFactTable extends AbstractCubeTable {
getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, Boolean.toString(isAggregated));
}
+ @Override
+ public boolean isVirtualFact() {
+ return false;
+ }
+
+ @Override
+ public String getSourceFactName() {
+ return this.getName();
+ }
+
+ public String getTablePrefix(String storage, UpdatePeriod updatePeriod) {
+ return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod);
+ }
+
public Date getAbsoluteStartTime() {
- return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME, false, true);
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME),
+ false, true);
}
public Date getRelativeStartTime() {
- return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_START_TIME, true, true);
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME),
+ true, true);
}
public Date getStartTime() {
@@ -403,18 +412,17 @@ public class CubeFactTable extends AbstractCubeTable {
}
public Date getAbsoluteEndTime() {
- return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_END_TIME, false, false);
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME),
+ false, false);
}
public Date getRelativeEndTime() {
- return getDateFromProperty(MetastoreConstants.FACT_RELATIVE_END_TIME, true, false);
+ return MetastoreUtil.getDateFromProperty(this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME),
+ true, false);
}
public Date getEndTime() {
return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime()));
}
- public String getTablePrefix(String storage, UpdatePeriod updatePeriod) {
- return storagePrefixUpdatePeriodMap.get(storage).get(updatePeriod);
- }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index b5c4c89..749e44c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -28,17 +28,7 @@ import java.text.ParseException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lens.api.metastore.XCube;
-import org.apache.lens.api.metastore.XDerivedCube;
-import org.apache.lens.api.metastore.XDimension;
-import org.apache.lens.api.metastore.XDimensionTable;
-import org.apache.lens.api.metastore.XFactTable;
-import org.apache.lens.api.metastore.XSegmentation;
-import org.apache.lens.api.metastore.XStorage;
-import org.apache.lens.api.metastore.XStorageTableElement;
-import org.apache.lens.api.metastore.XUpdatePeriod;
-import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor;
-import org.apache.lens.api.metastore.XUpdatePeriods;
+import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.Storage.LatestInfo;
import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
@@ -67,9 +57,11 @@ import org.jvnet.jaxb2_commons.lang.Equals;
import org.jvnet.jaxb2_commons.lang.HashCode;
import org.jvnet.jaxb2_commons.lang.ToString;
+import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import lombok.extern.slf4j.Slf4j;
/**
@@ -98,7 +90,9 @@ public class CubeMetastoreClient {
private final Map<String, CubeDimensionTable> allDimTables = Maps.newConcurrentMap();
private volatile boolean allDimTablesPopulated = false;
// map from fact name to fact table
- private final Map<String, CubeFactTable> allFactTables = Maps.newConcurrentMap();
+ private final Map<String, FactTable> allFactTables = Maps.newConcurrentMap();
+ // map from fact name to all virtual fact tables, any changes to facts must reflect in all of its virtual facts
+ private final Map<String, List<String>> factToVirtualFactMapping = Maps.newConcurrentMap();
private volatile boolean allFactTablesPopulated = false;
//map from segmentation name to segmentation
private final Map<String, Segmentation> allSegmentations = Maps.newConcurrentMap();
@@ -133,8 +127,8 @@ public class CubeMetastoreClient {
}
/** extract storage name from fact and storage table name. String operation */
- private String extractStorageName(CubeFactTable fact, String storageTableName) throws LensException {
- int ind = storageTableName.lastIndexOf(fact.getName());
+ private String extractStorageName(FactTable fact, String storageTableName) throws LensException {
+ int ind = storageTableName.lastIndexOf(fact.getSourceFactName());
if (ind <= 0) {
throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName());
}
@@ -164,7 +158,7 @@ public class CubeMetastoreClient {
String partCol = cube.getPartitionColumnOfTimeDim(timeDimension);
Date max = new Date(Long.MIN_VALUE);
boolean updated = false;
- for (CubeFactTable fact : getAllFacts(cube)) {
+ for (FactTable fact : getAllFacts(cube)) {
for (String storage : fact.getStorages()) {
for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) {
PartitionTimeline timeline = partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol);
@@ -191,7 +185,7 @@ public class CubeMetastoreClient {
throws LensException, HiveException {
UpdatePeriod updatePeriod = updatePeriodStr == null ? null : UpdatePeriod.valueOf(updatePeriodStr.toUpperCase());
List<PartitionTimeline> ret = Lists.newArrayList();
- CubeFactTable fact = getCubeFact(factName);
+ CubeFactTable fact = getCubeFactTable(factName);
List<String> storageList = Lists.newArrayList();
if (storage != null) {
storageList.add(storage);
@@ -315,7 +309,7 @@ public class CubeMetastoreClient {
properties, storageUpdatePeriodMap);
createCubeTable(factTable, storageTableDescs);
// do a get to update cache
- getCubeFact(factName);
+ getFactTable(factName);
}
@@ -326,8 +320,8 @@ public class CubeMetastoreClient {
createCube((XCube)entity);
} else if (entity instanceof XDimension) {
createDimension((XDimension) entity);
- } else if (entity instanceof XFactTable) {
- createCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XFact) {
+ createFactTable((XFact) entity);
} else if (entity instanceof XDimensionTable) {
createCubeDimensionTable((XDimensionTable) entity);
} else if (entity instanceof XSegmentation) {
@@ -345,8 +339,8 @@ public class CubeMetastoreClient {
alterCube((XCube)entity);
} else if (entity instanceof XDimension) {
alterDimension((XDimension) entity);
- } else if (entity instanceof XFactTable) {
- alterCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XFact) {
+ alterCubeFactTable((XFact) entity);
} else if (entity instanceof XDimensionTable) {
alterCubeDimensionTable((XDimensionTable) entity);
} else if (entity instanceof XSegmentation) {
@@ -356,24 +350,45 @@ public class CubeMetastoreClient {
}
}
-
public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) {
Map<String, String> props = new HashMap<String, String>();
props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties()));
props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns()));
return props;
}
- public void createCubeFactTable(XFactTable fact) throws LensException {
- createCubeFactTable(fact.getCubeName(),
- fact.getName(),
- JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()),
- JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()),
- fact.getWeight(),
- addFactColStartTimePropertyToFactProperties(fact),
- JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
- JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables()));
+
+ public void createFactTable(XFact fact) throws LensException {
+
+ if (fact instanceof XVirtualFactTable) {
+ XVirtualFactTable xvf = (XVirtualFactTable) fact;
+ createVirtualFactTable(xvf.getCubeName(), xvf.getName(), xvf.getSourceFactName(),
+ xvf.getWeight(), JAXBUtils.mapFromXProperties(xvf.getProperties()));
+ } else {
+ XFactTable xf = (XFactTable) fact;
+ createCubeFactTable(fact.getCubeName(),
+ fact.getName(),
+ JAXBUtils.fieldSchemaListFromColumns(xf.getColumns()),
+ JAXBUtils.getFactUpdatePeriodsFromStorageTables(xf.getStorageTables()),
+ xf.getWeight(),
+ addFactColStartTimePropertyToFactProperties(xf),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()),
+ JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables()));
+ }
}
+ public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight,
+ Map<String, String> properties) throws LensException {
+ FactTable sourceFact = getFactTable(sourceFactName);
+
+ Optional<Double> optionalWeight = Optional.fromNullable(weight);
+
+ CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName,
+ optionalWeight, properties, sourceFact);
+ createCubeTable(factTable, null);
+ // do a get to update cache
+ getFactTable(virtualFactName);
+
+ }
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
@@ -442,7 +457,7 @@ public class CubeMetastoreClient {
private void loadTimeLines(String fact, String storage, String timeLineKey) throws LensException, HiveException {
Set<String> uniqueStorageTables = new HashSet<>();
Map<UpdatePeriod, String> updatePeriodTableName = new HashMap<>();
- for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
+ for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) {
String storageTableName = getStorageTableName(fact, storage, updatePeriod);
updatePeriodTableName.put(updatePeriod, storageTableName);
Table storageTable = getTable(storageTableName);
@@ -477,7 +492,7 @@ public class CubeMetastoreClient {
// Not found in table properties either, compute from all partitions of the fact-storage table.
// First make sure all combinations of update period and partition column have an entry even
// if no partitions exist
- if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(storage) != null) {
+ if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) {
log.info("loading from all partitions: {}", storageTableName);
Table storageTable = getTable(storageTableName);
for (String partCol : getTimePartColNamesOfTable(storageTable)) {
@@ -848,7 +863,7 @@ public class CubeMetastoreClient {
new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, properties);
createCubeTable(factTable, storageTableDescs);
// do a get to update cache
- getCubeFact(factName);
+ getFactTable(factName);
}
/**
@@ -1089,7 +1104,7 @@ public class CubeMetastoreClient {
public Date getStorageTableEndDate(String storageTable, String factTableName)
throws LensException {
List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey());
- endDates.add(getFactTable(factTableName).getEndTime());
+ endDates.add((getFactTable(factTableName)).getEndTime());
return Collections.min(endDates);
}
@@ -1408,11 +1423,11 @@ public class CubeMetastoreClient {
}
/** extract storage name and check in timeline cache for existance */
- public boolean factPartitionExists(CubeFactTable fact, FactPartition part, String storageTableName)
+ public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName)
throws HiveException, LensException {
String storage = extractStorageName(fact, storageTableName);
- return partitionTimelineCache.partitionTimeExists(fact.getName(), storage, part.getPeriod(), part.getPartCol(),
- part.getPartSpec());
+ return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage,
+ part.getPeriod(), part.getPartCol(), part.getPartSpec());
}
public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
@@ -1638,12 +1653,35 @@ public class CubeMetastoreClient {
boolean isFactTable(Table tbl) {
String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY);
- return CubeTableType.FACT.name().equals(tableType);
+ return CubeTableType.FACT.name().equals(tableType)
+ && tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) == null;
}
-
boolean isFactTableForCube(Table tbl, String cube) {
- return isFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters())
+ return isFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName()))
+ .equalsIgnoreCase(cube.toLowerCase());
+ }
+
+ /**
+ * Is the table name passed a virtual fact table?
+ *
+ * @param virtualTableName table name
+ * @return true if it is virtual fact, false otherwise
+ * @throws HiveException
+ */
+ public boolean isVirtualFactTable(String virtualTableName) throws LensException {
+ Table tbl = getTable(virtualTableName);
+ return isVirtualFactTable(tbl);
+ }
+
+ boolean isVirtualFactTable(Table tbl) {
+ String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY);
+ return CubeTableType.FACT.name().equals(tableType)
+ && tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())) != null;
+ }
+
+ boolean isVirtualFactTableForCube(Table tbl, String cube) {
+ return isVirtualFactTable(tbl) && tbl.getParameters().get(MetastoreUtil.getFactCubeNameKey(tbl.getTableName()))
.equalsIgnoreCase(cube.toLowerCase());
}
@@ -1724,58 +1762,57 @@ public class CubeMetastoreClient {
return CubeTableType.DIMENSION.name().equals(tableType);
}
- public XFactTable getXFactTable(String tableName) throws LensException {
+ public XFact getXFactTable(String tableName) throws LensException {
return getXFactTable(getFactTable(tableName));
}
- public XFactTable getXFactTable(CubeFactTable cft) throws LensException {
+ public XFact getXFactTable(FactTable ft) throws LensException {
- XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft);
- Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap();
- for (String storageName : cft.getStorages()) {
- Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName);
- // This map tells if there are different tables for different update period.
- Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName);
- Set<String> tableNames = new HashSet<>();
- for (UpdatePeriod updatePeriod : updatePeriods) {
- tableNames.add(updatePeriodToTableMap.get(updatePeriod));
- }
- if (tableNames.size() <= 1) {
- XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
- getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName)));
- tblElement.setStorageName(storageName);
- for (UpdatePeriod p : updatePeriods) {
- tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
+ XFact fact;
+ if (ft.isVirtualFact()) {
+ CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft;
+ XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft);
+ factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName());
+ fact = factTable;
+ } else {
+ CubeFactTable cft = (CubeFactTable) ft;
+ XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft);
+ Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap();
+ for (String storageName : cft.getStorages()) {
+ Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName);
+ // This map tells if there are different tables for different update period.
+ Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName);
+ Set<String> tableNames = new HashSet<>();
+ for (UpdatePeriod updatePeriod : updatePeriods) {
+ tableNames.add(updatePeriodToTableMap.get(updatePeriod));
}
- factTable.getStorageTables().getStorageTable().add(tblElement);
- } else {
- // Multiple storage tables.
- XStorageTableElement tblElement = new XStorageTableElement();
- tblElement.setStorageName(storageName);
- XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
- tblElement.setUpdatePeriods(xUpdatePeriods);
- for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
- XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
- updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
- this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(),
+ if (tableNames.size() <= 1) {
+ XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
+ getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName)));
+ tblElement.setStorageName(storageName);
+ for (UpdatePeriod p : updatePeriods) {
+ tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
+ }
+ factTable.getStorageTables().getStorageTable().add(tblElement);
+ } else {
+ // Multiple storage tables.
+ XStorageTableElement tblElement = new XStorageTableElement();
+ tblElement.setStorageName(storageName);
+ XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
+ tblElement.setUpdatePeriods(xUpdatePeriods);
+ for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
+ XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
+ updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
+ this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(),
(String) entry.getValue()))));
- updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name()));
- xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
+ updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
+ xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
+ }
+ factTable.getStorageTables().getStorageTable().add(tblElement);
}
- factTable.getStorageTables().getStorageTable().add(tblElement);
}
+ fact = factTable;
}
- return factTable;
- }
- /**
- * Get {@link CubeFactTable} object corresponding to the name
- *
- * @param tableName The cube fact name
- * @return Returns CubeFactTable if table name passed is a fact table, null otherwise
- * @throws LensException
- */
-
- public CubeFactTable getFactTable(String tableName) throws LensException {
- return new CubeFactTable(getTableWithTypeFailFast(tableName, CubeTableType.FACT));
+ return fact;
}
public Segmentation getSegmentationTable(String tableName) throws HiveException, LensException {
@@ -1955,17 +1992,32 @@ public class CubeMetastoreClient {
* @return Returns cube is table name passed is a cube
* @throws LensException if there is no cube by the name
*/
- public CubeFactTable getCubeFact(String tableName) throws LensException {
- return getCubeFact(tableName, true);
+ public FactTable getFactTable(String tableName) throws LensException {
+ return getFactTable(tableName, true);
}
- private CubeFactTable getCubeFact(String tableName, boolean throwException) throws LensException {
+ private FactTable getFactTable(String tableName, boolean throwException) throws LensException {
tableName = tableName.trim().toLowerCase();
- CubeFactTable fact = allFactTables.get(tableName);
+ FactTable fact = allFactTables.get(tableName);
if (fact == null) {
synchronized (allFactTables) {
if (!allFactTables.containsKey(tableName)) {
Table tbl = getTableWithType(tableName, CubeTableType.FACT, throwException);
- fact = tbl == null ? null : new CubeFactTable(tbl);
+ if (tbl != null){
+ String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName()));
+ if (sourceFactName != null) {
+ fact = new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFactName));
+ if (factToVirtualFactMapping.get(sourceFactName) != null) {
+ List<String> prevList = factToVirtualFactMapping.get(sourceFactName);
+ prevList.add(tableName);
+ }else{
+ List<String> newList = new ArrayList<>();
+ newList.add(tableName);
+ factToVirtualFactMapping.put(sourceFactName, newList);
+ }
+ } else {
+ fact = new CubeFactTable(tbl);
+ }
+ }
if (enableCaching && fact != null) {
allFactTables.put(tableName, fact);
}
@@ -1977,6 +2029,15 @@ public class CubeMetastoreClient {
return fact;
}
+ private FactTable getFactTable(Table tbl) throws LensException {
+ String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName()));
+ if (sourceFact != null) {
+ return new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFact));
+ } else {
+ return new CubeFactTable(tbl);
+ }
+ }
+
public Segmentation getSegmentation(String segName) throws LensException {
return getSegmentation(segName, true);
}
@@ -2123,12 +2184,12 @@ public class CubeMetastoreClient {
* @return List of Cube Fact Table objects
* @throws LensException
*/
- public Collection<CubeFactTable> getAllFacts() throws LensException {
+ public Collection<FactTable> getAllFacts() throws LensException {
if (!allFactTablesPopulated) {
- List<CubeFactTable> facts = new ArrayList<>();
+ List<FactTable> facts = new ArrayList<>();
try {
for (String table : getAllHiveTableNames()) {
- CubeFactTable fact = getCubeFact(table, false);
+ FactTable fact = getFactTable(table, false);
if (fact != null) {
facts.add(fact);
}
@@ -2144,6 +2205,38 @@ public class CubeMetastoreClient {
}
/**
+ * Get all facts in metastore (virtual facts optional)
+ * @param includeVirtualFacts set true for including virtual facts
+ * @return List of Cube Fact Table objects
+ * @throws LensException
+ */
+ public Collection<FactTable> getAllFacts(boolean includeVirtualFacts) throws LensException {
+ if (!allFactTablesPopulated) {
+ List<FactTable> facts = new ArrayList<>();
+ try {
+ for (String table : getAllHiveTableNames()) {
+ FactTable fact = getFactTable(table, false);
+ if (fact != null) {
+ if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact
+ if (includeVirtualFacts) {
+ facts.add(fact);
+ }
+ } else {
+ facts.add(fact);
+ }
+ }
+ }
+ } catch (HiveException e) {
+ throw new LensException("Could not get all fact tables", e);
+ }
+ allFactTablesPopulated = enableCaching;
+ return facts;
+ } else {
+ return allFactTables.values();
+ }
+ }
+
+ /**
* Get all segmentations in metastore
*
* @return List of segmentation objects
@@ -2169,8 +2262,6 @@ public class CubeMetastoreClient {
}
}
-
-
private Collection<String> getAllHiveTableNames() throws HiveException, LensException {
if (!allTablesPopulated) {
List<String> allTables = getClient().getAllTables();
@@ -2192,7 +2283,7 @@ public class CubeMetastoreClient {
* @return List of fact tables
* @throws LensException
*/
- public List<CubeFactTable> getAllFacts(CubeInterface cube) throws LensException {
+ public List<FactTable> getAllFacts(CubeInterface cube) throws LensException {
String cubeName = null;
if (cube != null) {
if (cube instanceof DerivedCube) {
@@ -2200,8 +2291,33 @@ public class CubeMetastoreClient {
}
cubeName = cube.getName();
}
- List<CubeFactTable> cubeFacts = new ArrayList<>();
- for (CubeFactTable fact : getAllFacts()) {
+ List<FactTable> cubeFacts = new ArrayList<>();
+ for (FactTable fact : getAllFacts()) {
+ if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) {
+ cubeFacts.add(fact);
+ }
+ }
+ return cubeFacts;
+ }
+
+ /**
+ * Get all facts of cube (optional virtual facts)
+ *
+ * @param cube Cube object
+ * @param includeVirtualFacts set true for virtual facts
+ * @return List of fact tables with optional virtual facts
+ * @throws LensException
+ */
+ public List<FactTable> getAllFacts(CubeInterface cube, boolean includeVirtualFacts) throws LensException {
+ String cubeName = null;
+ if (cube != null) {
+ if (cube instanceof DerivedCube) {
+ cube = ((DerivedCube) cube).getParent();
+ }
+ cubeName = cube.getName();
+ }
+ List<FactTable> cubeFacts = new ArrayList<>();
+ for (FactTable fact : getAllFacts(includeVirtualFacts)) {
if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) {
cubeFacts.add(fact);
}
@@ -2262,8 +2378,8 @@ public class CubeMetastoreClient {
return dimTables;
}
- public boolean partColExists(String fact, String storage, String partCol) throws LensException {
- for (String storageTable : getStorageTables(fact, storage)) {
+ public boolean partColExists(FactTable factTable, String storage, String partCol) throws LensException {
+ for (String storageTable : getStorageTables(factTable, storage)) {
for (FieldSchema f : getTable(storageTable).getPartCols()) {
if (f.getName().equalsIgnoreCase(partCol)) {
return true;
@@ -2278,20 +2394,21 @@ public class CubeMetastoreClient {
* Note: If each update period in the storage has a different storage table, this method will return N Storage Tables
* where N is the number of update periods in the storage (LENS-1386)
*
- * @param fact
+ * @param factTable
* @param storage
* @return
* @throws LensException
*/
- public Set<String> getStorageTables(String fact, String storage) throws LensException {
+ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException {
Set<String> uniqueStorageTables = new HashSet<>();
- for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) {
- uniqueStorageTables.add(getStorageTableName(fact, storage, updatePeriod));
+
+ for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) {
+ String factName = factTable.getSourceFactName();
+ uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod));
}
return uniqueStorageTables;
}
-
/**
*
* @param table table name
@@ -2431,7 +2548,7 @@ public class CubeMetastoreClient {
*/
public void dropFact(String factName, boolean cascade) throws LensException {
getTableWithTypeFailFast(factName, CubeTableType.FACT);
- CubeFactTable fact = getFactTable(factName);
+ FactTable fact = getFactTable(factName);
if (cascade) {
for (String storage : fact.getStorages()) {
dropStorageFromFact(factName, storage, false);
@@ -2439,8 +2556,48 @@ public class CubeMetastoreClient {
}
dropHiveTable(factName);
allFactTables.remove(factName.trim().toLowerCase());
+ if (fact.isVirtualFact()) {
+ String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName()));
+ if (factToVirtualFactMapping.get(sourceFactTable) != null
+ && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) {
+ factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName());
+ }
+ } else {
+ dropAllVirtualFactTables(factName);
+ }
}
+ private void dropAllVirtualFactTables(String cubeFactTableName) throws LensException {
+ if (enableCaching) {
+ cubeFactTableName = cubeFactTableName.trim().toLowerCase();
+ if (factToVirtualFactMapping.get(cubeFactTableName) != null) {
+ List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName);
+ factToVirtualFactMapping.remove(cubeFactTableName);
+ for (String vf : virtualFactTableNames) {
+ dropVirtualFact(vf.trim().toLowerCase());
+ }
+ }
+ }
+ }
+
+ /**
+ * Drop a virtual fact
+ *
+ * @param virtualFactName virtual fact name
+ * @throws LensException
+ */
+ public void dropVirtualFact(String virtualFactName) throws LensException {
+ virtualFactName = virtualFactName.trim().toLowerCase();
+ Table virtualTbl = getTableWithTypeFailFast(virtualFactName, CubeTableType.FACT);
+ String sourceFactTable = virtualTbl.getParameters().get(getSourceFactNameKey(virtualTbl.getTableName()));
+ if (factToVirtualFactMapping.get(sourceFactTable) != null
+ && factToVirtualFactMapping.get(sourceFactTable).contains(virtualFactName)) {
+ factToVirtualFactMapping.get(sourceFactTable).remove(virtualFactName);
+ }
+ dropHiveTable(virtualFactName);
+ allFactTables.remove(virtualFactName.trim().toLowerCase());
+
+ }
public void dropSegmentation(String segName) throws LensException {
getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION);
@@ -2456,7 +2613,7 @@ public class CubeMetastoreClient {
* @throws LensException
*/
public void dropStorageFromFact(String factName, String storage) throws LensException {
- CubeFactTable cft = getFactTable(factName);
+ CubeFactTable cft = getCubeFactTable(factName);
dropHiveTablesForStorage(factName, storage);
cft.dropStorage(storage);
alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft);
@@ -2464,7 +2621,7 @@ public class CubeMetastoreClient {
}
private void dropHiveTablesForStorage(String factName, String storage) throws LensException{
- CubeFactTable cft = getFactTable(factName);
+ CubeFactTable cft = getCubeFactTable(factName);
Set<String> droppedTables = new HashSet<>();
for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) {
UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey();
@@ -2480,7 +2637,7 @@ public class CubeMetastoreClient {
throws LensException {
dropHiveTablesForStorage(factName, storage);
if (updateFact) {
- CubeFactTable cft = getFactTable(factName);
+ CubeFactTable cft = getCubeFactTable(factName);
cft.dropStorage(storage);
alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft);
updateFactCache(factName);
@@ -2530,11 +2687,20 @@ public class CubeMetastoreClient {
dropHiveTable(dimTblName);
allDimTables.remove(dimTblName.trim().toLowerCase());
}
- public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException {
- alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact),
- JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
- JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns()));
+
+ public void alterCubeFactTable(XFact fact) throws LensException, HiveException {
+ if (fact instanceof XVirtualFactTable) {
+ XVirtualFactTable xvf = (XVirtualFactTable) fact;
+ alterCubeFactTable(xvf.getName(), JAXBUtils.cubeVirtualFactFromFactTable(xvf,
+ getFactTable(xvf.getSourceFactName())), null, new HashMap<>());
+ } else {
+ XFactTable xf = (XFactTable) fact;
+ alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(xf),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()),
+ JAXBUtils.columnStartAndEndTimeFromXColumns(xf.getColumns()));
+ }
}
+
/**
* Alter a cubefact with new definition and alter underlying storage tables as well.
*
@@ -2544,7 +2710,7 @@ public class CubeMetastoreClient {
*
* @throws HiveException
*/
- public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable,
+ public void alterCubeFactTable(String factTableName, FactTable cubeFactTable,
Map<String, StorageTableDesc> storageTableDescs,
Map<String, String> props)
throws HiveException, LensException {
@@ -2552,7 +2718,7 @@ public class CubeMetastoreClient {
if (!props.isEmpty()) {
cubeFactTable.getProperties().putAll(props);
}
- alterCubeTable(factTableName, factTbl, cubeFactTable);
+ alterCubeTable(factTableName, factTbl, (AbstractCubeTable) cubeFactTable);
if (storageTableDescs != null) {
// create/alter tables for each storage
for (Map.Entry<String, StorageTableDesc> entry : storageTableDescs.entrySet()) {
@@ -2560,11 +2726,24 @@ public class CubeMetastoreClient {
}
}
updateFactCache(factTableName);
+
+ updateAllVirtualFacts(getFactTable(factTableName));
}
public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException {
alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg));
}
+
+ /**
+ * Alter a virtual cube fact with new definition
+ * @param cubeVirtualFactTable cube virtual fact table
+ * @throws HiveException
+ */
+ public void alterVirtualCubeFactTable(CubeVirtualFactTable cubeVirtualFactTable)
+ throws HiveException, LensException {
+ alterCubeFactTable(cubeVirtualFactTable.getName(), cubeVirtualFactTable, null, new HashMap<>());
+ }
+
public void alterSegmentation(String segName, Segmentation seg)
throws HiveException, LensException {
getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION);
@@ -2583,7 +2762,52 @@ public class CubeMetastoreClient {
private void updateFactCache(String factTableName) throws LensException {
if (enableCaching) {
- allFactTables.put(factTableName.trim().toLowerCase(), new CubeFactTable(refreshTable(factTableName)));
+ Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT);
+ FactTable refreshedTable;
+ if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){
+ String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName));
+ refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName),
+ getCubeFactTable(sourceFactName));
+ }else {
+ refreshedTable = new CubeFactTable(refreshTable(factTableName));
+ }
+ allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable);
+ }
+ }
+
+ public CubeFactTable getCubeFactTable(String factName) throws LensException {
+ FactTable factTable = getFactTable(factName);
+ if (factTable instanceof CubeFactTable) {
+ return (CubeFactTable) factTable;
+ } else {
+ throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(),
+ factName, "Fact"));
+ }
+ }
+
+ public CubeVirtualFactTable getCubeVirtualFactTable(String factName) throws LensException {
+ FactTable factTable = getFactTable(factName);
+ if (factTable instanceof CubeVirtualFactTable) {
+ return (CubeVirtualFactTable) factTable;
+ } else {
+ throw new LensException(new LensException(LensCubeErrorCode.ENTITY_TYPE_NOT_AS_EXPECTED.getLensErrorInfo(),
+ factName, "VirtualFact"));
+ }
+ }
+
+ private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException {
+ if (enableCaching) {
+ String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase();
+ if (factToVirtualFactMapping.get(cubeFactTableName) != null) {
+ synchronized (allFactTables) {
+ List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName);
+ for (String vf : virtualFactTableNames) {
+ CubeVirtualFactTable cvf = getCubeVirtualFactTable(vf);
+ cvf.setSourceCubeFactTable(cubeFactTable);
+ allFactTables.put(vf.trim().toLowerCase(), cvf);
+ }
+ }
+ }
}
}
@@ -2693,8 +2917,11 @@ public class CubeMetastoreClient {
if (updatePeriod == null) {
return storage;
}
- if (isFactTable(factOrDimTableName)) {
- return getFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod);
+ if (isVirtualFactTable(factOrDimTableName)) {
+ CubeFactTable sourceFact = (CubeFactTable) getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable();
+ return sourceFact.getTablePrefix(storage, updatePeriod);
+ } else if (isFactTable(factOrDimTableName)) {
+ return getCubeFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod);
} else {
return storage;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
new file mode 100644
index 0000000..1fc74b0
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import com.google.common.base.Optional;
+
+import com.google.common.collect.Lists;
+import lombok.Getter;
+import lombok.Setter;
+
+public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable {
+
+ @Getter
+ @Setter
+ private FactTable sourceCubeFactTable;
+ private String cubeName;
+ private static final List<FieldSchema> COLUMNS = new ArrayList<FieldSchema>();
+ @Getter
+ private Optional<Double> virtualFactWeight = Optional.absent();
+
+ static {
+ COLUMNS.add(new FieldSchema("dummy", "string", "dummy column"));
+ }
+
+ public CubeVirtualFactTable(Table hiveTable, FactTable sourceCubeFactTable) {
+ super(hiveTable);
+ this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName()));
+ this.sourceCubeFactTable = sourceCubeFactTable;
+
+ String wtStr = getProperties().get(MetastoreUtil.getCubeTableWeightKey(getName()));
+ if (wtStr != null) {
+ this.virtualFactWeight = Optional.of(Double.parseDouble(wtStr));
+ }
+ }
+
+ public CubeVirtualFactTable(String cubeName, String virtualFactName, Optional<Double> weight,
+ Map<String, String> properties, FactTable sourceFact) {
+ super(virtualFactName, COLUMNS, properties, weight.isPresent() ? weight.get() : sourceFact.weight());
+ this.cubeName = cubeName;
+ this.virtualFactWeight = weight;
+ this.sourceCubeFactTable = sourceFact;
+ addProperties();
+ }
+
+ /**
+ * Alters the weight of table
+ *
+ * @param weight Weight of the table.
+ */
+ @Override
+ public void alterWeight(double weight) {
+ this.virtualFactWeight = Optional.of(weight);
+ this.addProperties();
+ }
+
+ @Override
+ protected void addProperties() {
+ getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name());
+ getProperties().put(MetastoreUtil.getSourceFactNameKey(this.getName()), this.sourceCubeFactTable.getName());
+ if (virtualFactWeight.isPresent()) {
+ getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get()));
+ }
+ this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName);
+ }
+
+ @Override
+ public CubeTableType getTableType() {
+ return CubeTableType.FACT;
+ }
+
+ @Override
+ public Set<String> getValidColumns() {
+ return this.sourceCubeFactTable.getValidColumns();
+ }
+
+ @Override
+ public Set<String> getStorages() {
+ return this.sourceCubeFactTable.getStorages();
+ }
+
+ @Override
+ public Map<String, Set<UpdatePeriod>> getUpdatePeriods() {
+ return this.sourceCubeFactTable.getUpdatePeriods();
+ }
+
+ @Override
+ public String getCubeName() {
+ return this.cubeName;
+ }
+
+ @Override
+ public String getDataCompletenessTag() {
+ return this.sourceCubeFactTable.getDataCompletenessTag();
+ }
+
+ @Override
+ public boolean isAggregated() {
+ return this.sourceCubeFactTable.isAggregated();
+ }
+
+ @Override
+ public List<FieldSchema> getColumns() {
+ return this.sourceCubeFactTable.getColumns();
+ }
+
+ @Override
+ public double weight() {
+ return virtualFactWeight.isPresent() ? virtualFactWeight.get() : sourceCubeFactTable.weight();
+ }
+
+ public Date getAbsoluteStartTime() {
+ String absoluteStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME);
+ Date absoluteDate = null;
+ if (StringUtils.isNotBlank(absoluteStartTime)) {
+ absoluteDate = MetastoreUtil.getDateFromProperty(absoluteStartTime, false, true);
+ }
+ return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteStartTime() : absoluteDate;
+ }
+
+ public Date getRelativeStartTime() {
+ String relativeStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME);
+ Date relativeDate = null;
+ if (StringUtils.isNotBlank(relativeStartTime)) {
+ relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, true, true);
+ }
+ return relativeDate == null ? this.sourceCubeFactTable.getRelativeStartTime() : relativeDate;
+ }
+
+ public Date getStartTime() {
+ return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime()));
+ }
+
+ public Date getAbsoluteEndTime() {
+ String absoluteEndTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME);
+ Date absoluteDate = null;
+ if (StringUtils.isNotBlank(absoluteEndTime)) {
+ absoluteDate = MetastoreUtil.getDateFromProperty(absoluteEndTime, false, false);
+ }
+ return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteEndTime() : absoluteDate;
+ }
+
+ public Date getRelativeEndTime() {
+ String relativeEndTime = this.getProperties().get(MetastoreConstants.FACT_RELATIVE_END_TIME);
+ Date relativeDate = null;
+ if (StringUtils.isNotBlank(relativeEndTime)) {
+ relativeDate = MetastoreUtil.getDateFromProperty(relativeEndTime, true, false);
+ }
+ return relativeDate == null ? this.sourceCubeFactTable.getRelativeEndTime() : relativeDate;
+ }
+
+ public Date getEndTime() {
+ return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime()));
+ }
+
+ @Override
+ public boolean isVirtualFact() {
+ return true;
+ }
+
+ @Override
+ public String getSourceFactName() {
+ return this.sourceCubeFactTable.getName();
+ }
+}