You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/03/28 12:12:54 UTC
[1/2] lens git commit: LENS-975 : Add cube segmentation specification
Repository: lens
Updated Branches:
refs/heads/master a2b6125c6 -> e4f26aae8
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 3b091f5..45f41c2 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -1633,6 +1633,153 @@ public class TestMetastoreService extends LensJerseyTest {
}
}
+ private XCubeSegmentation createCubeSegmentation(String segName) {
+ return createCubeSegmentation(segName, "testCube");
+ }
+
+ private XCubeSegmentation createCubeSegmentation(String segName, String cubeName) {
+ XCubeSegmentation seg = cubeObjectFactory.createXCubeSegmentation();
+
+ //Create Xproperties
+ XProperties props1 = cubeObjectFactory.createXProperties();
+ XProperty prop1 = new XProperty();
+ prop1.setName("prop_key1");
+ prop1.setValue("prop_val1");
+ props1.getProperty().add(prop1);
+
+ XProperties props2 = cubeObjectFactory.createXProperties();
+ XProperty prop2 = new XProperty();
+ prop2.setName("prop_key2");
+ prop2.setValue("prop_val2");
+ props2.getProperty().add(prop2);
+
+ // Create XcubeSegments
+ XCubeSegments cubes = new XCubeSegments();
+ XCubeSegment c1 = cubeObjectFactory.createXCubeSegment();
+ c1.setCubeName("cube1");
+ c1.setSegmentParameters(props1);
+
+ XCubeSegment c2 = cubeObjectFactory.createXCubeSegment();
+ c2.setCubeName("cube2");
+ c2.setSegmentParameters(props2);
+
+ cubes.getCubeSegment().add(c1);
+ cubes.getCubeSegment().add(c2);
+
+ seg.setProperties(new XProperties());
+ seg.setName(segName);
+ seg.setWeight(10.0);
+ seg.setCubeName(cubeName);
+ seg.setCubeSegements(cubes);
+ Map<String, String> properties = LensUtil.getHashMap("foo", "bar");
+ seg.getProperties().getProperty().addAll(JAXBUtils.xPropertiesFromMap(properties));
+
+ return seg;
+ }
+
+ @Test(dataProvider = "mediaTypeData")
+ public void testCreateAndAlterCubeSegmentation(MediaType mediaType) throws Exception {
+ final String segname = "testCreateCubeSegmentation";
+ final String DB = dbPFX + "testCreateCubeSegmentation_DB" + mediaType.getSubtype();
+ String prevDb = getCurrentDatabase(mediaType);
+ createDatabase(DB, mediaType);
+ setCurrentDatabase(DB, mediaType);
+
+ try {
+ XCubeSegmentation seg = createCubeSegmentation(segname);
+
+ APIResult result = target()
+ .path("metastore")
+ .path("cubesegmentations").queryParam("sessionid", lensSessionId)
+ .request(mediaType)
+ .post(Entity.entity(
+ new GenericEntity<JAXBElement<XCubeSegmentation>>(
+ cubeObjectFactory.createXCubeSegmentation(seg)){}, mediaType),
+ APIResult.class);
+ assertSuccess(result);
+
+ // Get all cube segmentations, this should contain the cube segmentation created earlier
+ StringList segNames = target().path("metastore/cubesegmentations")
+ .queryParam("sessionid", lensSessionId).request(mediaType).get(StringList.class);
+ assertTrue(segNames.getElements().contains(segname.toLowerCase()));
+
+ // Get the created cubesegmentation
+ JAXBElement<XCubeSegmentation> gotCubeSegmentation = target().path("metastore/cubesegmentations")
+ .path(segname)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .get(new GenericType<JAXBElement<XCubeSegmentation>>() {});
+ XCubeSegmentation gotSeg = gotCubeSegmentation.getValue();
+ assertTrue(gotSeg.getName().equalsIgnoreCase(segname));
+ assertEquals(gotSeg.getWeight(), 10.0);
+ CubeSegmentation cs = JAXBUtils.cubeSegmentationFromXCubeSegmentation(seg);
+
+ // Check for cube segemnts
+ boolean foundCube1 = false;
+ for (CubeSegment cube : cs.getCubeSegments()) {
+ if (cube.getName().equalsIgnoreCase("cube1")
+ && cube.getProperties().get("prop_key1").equals("prop_val1")) {
+ foundCube1 = true;
+ break;
+ }
+ }
+ assertTrue(foundCube1);
+ assertEquals(cs.getProperties().get("foo"), "bar");
+
+ // update cube segmentation
+ XCubeSegmentation update = JAXBUtils.xsegmentationFromCubeSegmentation(cs);
+ XCubeSegments cubes = new XCubeSegments();
+ XCubeSegment c1 = cubeObjectFactory.createXCubeSegment();
+ c1.setCubeName("cube11");
+ XCubeSegment c2 = cubeObjectFactory.createXCubeSegment();
+ c2.setCubeName("cube22");
+ cubes.getCubeSegment().add(c1);
+ cubes.getCubeSegment().add(c2);
+
+ update.setWeight(20.0);
+ update.setCubeSegements(cubes);
+
+ result = target().path("metastore").path("cubesegmentations").path(segname)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .put(Entity.entity(new GenericEntity<JAXBElement<XCubeSegmentation>>(
+ cubeObjectFactory.createXCubeSegmentation(update)){}, mediaType),
+ APIResult.class);
+ assertSuccess(result);
+
+ // Get the updated table
+ JAXBElement<XCubeSegmentation> gotUpdatedCubeSeg = target().path("metastore/cubesegmentations").path(segname)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .get(new GenericType<JAXBElement<XCubeSegmentation>>() {});
+ XCubeSegmentation gotUpSeg = gotUpdatedCubeSeg.getValue();
+ CubeSegmentation usg = JAXBUtils.cubeSegmentationFromXCubeSegmentation(gotUpSeg);
+
+ assertEquals(usg.getCubeSegments().size(), 2);
+ for (CubeSegment segmnt : usg.getCubeSegments()) {
+ assertTrue(segmnt.getName().equals("cube11") || segmnt.getName().equals("cube22"));
+ }
+
+ // Finally, drop the cube segmentation
+ result = target().path("metastore").path("cubesegmentations").path(segname)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .delete(APIResult.class);
+
+ assertSuccess(result);
+
+ // Drop again, this time it should give a 404
+ try {
+ target().path("metastore").path("cubesegmentations").path(segname)
+ .queryParam("cascade", "true")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .delete(APIResult.class);
+ fail("Expected 404");
+ } catch (NotFoundException nfe) {
+ // PASS
+ }
+ } finally {
+ setCurrentDatabase(prevDb, mediaType);
+ dropDatabase(DB, mediaType);
+ }
+ }
+
private XFactTable createFactTable(String factName) {
return createFactTable(factName, "testCube");
}
[2/2] lens git commit: LENS-975 : Add cube segmentation specification
Posted by am...@apache.org.
LENS-975 : Add cube segmentation specification
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e4f26aae
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e4f26aae
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e4f26aae
Branch: refs/heads/master
Commit: e4f26aae8293432833a6bfca03ebcf79cd71d13d
Parents: a2b6125
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Mon Mar 28 15:42:33 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Mar 28 15:42:33 2016 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 97 ++++++++++
.../lens/cli/commands/BaseTableCrudCommand.java | 37 ++++
.../LensCubeSegmentationCommands.java | 137 +++++++++++++
.../cli/TestLensCubeSegmentationCommands.java | 156 +++++++++++++++
lens-cli/src/test/resources/seg1.xml | 45 +++++
.../java/org/apache/lens/client/LensClient.java | 25 +++
.../apache/lens/client/LensMetadataClient.java | 86 +++++++++
.../lens/cube/metadata/AbstractCubeTable.java | 25 +++
.../lens/cube/metadata/CubeFactTable.java | 20 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 145 ++++++++++++++
.../apache/lens/cube/metadata/CubeSegment.java | 35 ++++
.../lens/cube/metadata/CubeSegmentation.java | 190 +++++++++++++++++++
.../lens/cube/metadata/CubeTableType.java | 2 +-
.../lens/cube/metadata/MetastoreConstants.java | 9 +
.../lens/cube/metadata/MetastoreUtil.java | 20 ++
.../lens/cube/metadata/CubeFactTableTest.java | 1 +
.../cube/metadata/TestCubeMetastoreClient.java | 47 +++++
.../apache/lens/examples/SampleMetastore.java | 11 ++
lens-examples/src/main/resources/seg1.xml | 44 +++++
lens-examples/src/test/resources/yaml/seg1.yaml | 33 ++++
.../lens/server/api/LensConfConstants.java | 5 +
.../api/metastore/CubeMetastoreService.java | 49 +++++
.../metastore/CubeMetastoreServiceImpl.java | 84 ++++++++
.../apache/lens/server/metastore/JAXBUtils.java | 75 ++++++++
.../server/metastore/MetastoreResource.java | 157 ++++++++++++++-
.../server/metastore/TestMetastoreService.java | 147 ++++++++++++++
26 files changed, 1661 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 0f4dbea..f221a6d 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -1010,6 +1010,39 @@
</xs:sequence>
</xs:complexType>
+ <xs:complexType name="x_cube_segment">
+ <xs:annotation>
+ <xs:documentation>
+ cube segment belong to the segmentation
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element name="segment_parameters" type="x_properties" minOccurs="0" maxOccurs="1">
+ <xs:annotation>
+ <xs:documentation>
+ Segment properties.
+ The following properties can be specified at a cube segment level :
+ 1. lens.metastore.cube.column.mapping : The column mapping for columns of cube segment if they are
+ different in underlying cube. The value is speciified with comma separated map entries specified with
+ key-values separated by equalto. Example value: id=id1,name=name1
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
+ </xs:sequence>
+ <xs:attribute name="cube_name" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="x_cube_segments">
+ <xs:annotation>
+ <xs:documentation>
+ Segments in segmentation. There can be two or more such cube segments.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element name="cube_segment" minOccurs="2" maxOccurs="unbounded" type="x_cube_segment"/>
+ </xs:sequence>
+ </xs:complexType>
+
<xs:element name="x_partition_list" type="x_partition_list"/>
<xs:complexType name="x_partition_list">
<xs:annotation>
@@ -1191,6 +1224,9 @@
e.g. now.year or now.day - 6 hour etc.
7. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of
the fact will be valid here. Format same as absolute start time.
+ 8. cube.fact.relative.end.time: You can specify the end date for fact table
+ relative to current date e.g. fact table is valid for next 90days starting from today.
+ This can be specified as just a time difference e.g. "+90 days"
</xs:documentation>
</xs:annotation>
</xs:element>
@@ -1225,6 +1261,67 @@
</xs:attribute>
</xs:complexType>
+ <xs:element name="x_cube_segmentation" type="x_cube_segmentation"/>
+
+ <xs:complexType name="x_cube_segmentation">
+ <xs:annotation>
+ <xs:documentation>
+ Cube segmentation is associated to a base cube and consists of two or more cube segments. All
+ cube segments together make cube segmentation complete.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ Properties that can be set for cubesegmentation are
+ 1. cube.segmentation.absolute.start.time: start time of the cube segmentation.
+ For queries that ask for time before this, this cubesegmentation is not a candidate. Time format
+ can be specified in yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]] format.
+ 2. cube.segmentation.relative.start.time: Here you can specify cubesegmentations's relative validity
+ relative to current time. Useful if you want to specify e.g. this cubesegmentation is valid
+ for today - 90 days. Can be specified as just a time difference e.g. "-90 days".
+ Or can be specified in relative syntax.
+ e.g. now.year or now.day - 6 hour etc.
+ 3. cube.segmentation.absolute.end.time: If you're deprecating a cubesegmentation, put the final date till
+ which the data of the cubesegmentation will be valid here. Format same as absolute start time.
+ 4. cube.segmentation.relative.end.time: You can specify the end date for cubesegmentation
+ relative to current date e.g. cubesegmentationas is valid for next 90 days starting from today.
+ This can be specified as just a time difference e.g. "+90 days".
+ </xs:documentation>
+ </xs:annotation>
+ </xs:element>
+ <xs:element name="cube_segements" type="x_cube_segments" maxOccurs="1" minOccurs="0"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ Name of cube segmentation
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="cube_name" type="xs:string" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The base or parent cube to which the cube segmentation is associated.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="weight" use="required" >
+ <xs:annotation>
+ <xs:documentation>
+ The weight of the cube segmentation. LENS will use this attribute to decide the lightest table to
+ query when there are more than one eligible tables.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="xs:double">
+ <xs:minInclusive value="0"></xs:minInclusive>
+ </xs:restriction>
+ </xs:simpleType>
+ </xs:attribute>
+ </xs:complexType>
+
<xs:element name="x_native_table" type="x_native_table"/>
<xs:complexType name="x_native_table">
<xs:annotation>
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java
new file mode 100644
index 0000000..767a78c
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseTableCrudCommand.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.commands;
+
+import java.util.List;
+
+import com.google.common.base.Joiner;
+
+
+public abstract class BaseTableCrudCommand<T> extends LensCRUDCommand<T> {
+ public String showAll(String filter) {
+ List<String> all = getAll(filter);
+ if (all == null || all.isEmpty()) {
+ return "No " + getSingleObjectName() + " found" + (filter == null ? "" : " for " + filter);
+ }
+ return Joiner.on("\n").join(all);
+ }
+
+ protected abstract List<String> getAll(String filter);
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java
new file mode 100644
index 0000000..ed2ad7a
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/annotations/LensCubeSegmentationCommands.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.commands.annotations;
+
+
+import java.io.File;
+import java.util.List;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.api.metastore.XCubeSegmentation;
+import org.apache.lens.cli.commands.BaseTableCrudCommand;
+
+import org.springframework.shell.core.annotation.CliCommand;
+import org.springframework.shell.core.annotation.CliOption;
+import org.springframework.stereotype.Component;
+
+import lombok.NonNull;
+
+@Component
+@UserDocumentation(title = "Commands for CubeSegmentation Management",
+ description = "These command provide CRUD for CubeSegmentation")
+public class LensCubeSegmentationCommands extends BaseTableCrudCommand<XCubeSegmentation> {
+
+ /**
+ * Show cube segmentation
+ *
+ * @return the string
+ */
+ @CliCommand(value = "show cubesegmentations",
+ help = "display list of cubesegmentations in current database. "
+ + "If optional <cube_name> is supplied, only cubesegmentations "
+ + "belonging to cube <cube_name> will be displayed")
+ public String showCubeSegmentations(
+ @CliOption(key = {"", "cube_name"}, mandatory = false, help = "<cube_name>") String cubeName) {
+ return showAll(cubeName);
+ }
+ /**
+ * Creates the cubesegmentation
+ *
+ * @param path the cubesegmentation spec
+ * @return the string
+ */
+ @CliCommand(value = "create cubesegmentation",
+ help = "create a new cubesegmentation, taking spec from <path-to-cubesegmentation-spec file>")
+ public String createCubeSegmentation(
+ @CliOption(key = {"", "path"}, mandatory = true, help =
+ "<path-to-cubesegmentation-spec file>") @NonNull final File path) {
+ return create(path, false);
+ }
+
+ /**
+ * Describe cubesegmentation.
+ *
+ * @param name the cubesegmentation name
+ * @return the string
+ */
+ @CliCommand(value = "describe cubesegmentation", help = "describe cubesegmentation <cubesegmentation_name>")
+ public String describeCubeSegmentation(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name) {
+ return formatJson(getClient().getCubeSegmentation(name));
+ }
+
+
+ /**
+ * Update cubesegmentation.
+ *
+ * @param name the cubesegmentation to be updated
+ * @param path path to spec file
+ * @return the string
+ */
+ @CliCommand(value = "update cubesegmentation",
+ help = "update cubesegmentation <cubesegmentation_name>, taking spec from <path-to-cubesegmentation-spec file>")
+ public String updateCubeSegmentation(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<path-to-cubesegmentation-spec-file>")
+ @NonNull final File path) {
+ return update(name, path);
+ }
+
+ /**
+ * Drop cubesegmentation.
+ *
+ * @param name the cubesegmentation to be dropped
+ * @return the string
+ */
+ @CliCommand(value = "drop cubesegmentation", help = "drop cubesegmentation <cubesegmentation_name>")
+ public String dropCubeSegmentation(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cubesegmentation_name>") String name) {
+ return drop(name, false);
+ }
+
+ @Override
+ public List<String> getAll() {
+ return getClient().getAllCubeSegmentations();
+ }
+
+ @Override
+ protected APIResult doCreate(String path, boolean ignoreIfExists) {
+ return getClient().createCubeSegmentation(path);
+ }
+
+ @Override
+ protected XCubeSegmentation doRead(String name) {
+ return getClient().getCubeSegmentation(name);
+ }
+
+ @Override
+ public APIResult doUpdate(String name, String path) {
+ return getClient().updateCubeSegmentation(name, path);
+ }
+
+ @Override
+ protected APIResult doDelete(String name, boolean cascade) {
+ return getClient().dropCubeSegmentation(name);
+ }
+
+ @Override
+ protected List<String> getAll(String filter) {
+ return getClient().getAllCubeSegmentations(filter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java
new file mode 100644
index 0000000..53a52de
--- /dev/null
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeSegmentationCommands.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli;
+
+import static org.testng.Assert.*;
+
+import java.io.*;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import javax.ws.rs.NotFoundException;
+
+import org.apache.lens.cli.commands.LensCubeCommands;
+import org.apache.lens.cli.commands.annotations.LensCubeSegmentationCommands;
+import org.apache.lens.client.LensClient;
+
+import org.testng.annotations.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class TestLensCubeSegmentationCommands extends LensCliApplicationTest {
+
+ private static LensCubeSegmentationCommands command = null;
+ private static LensCubeCommands cubeCommands = null;
+
+ private void createSampleCube() throws URISyntaxException {
+ URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("sample-cube.xml");
+ String cubeList = getCubeCommand().showCubes();
+ assertFalse(cubeList.contains("sample_cube"), cubeList);
+ getCubeCommand().createCube(new File(cubeSpec.toURI()));
+ cubeList = getCubeCommand().showCubes();
+ assertTrue(cubeList.contains("sample_cube"), cubeList);
+ }
+
+ private static LensCubeSegmentationCommands getCommand() {
+ if (command == null) {
+ LensClient client = new LensClient();
+ command = new LensCubeSegmentationCommands();
+ command.setClient(client);
+ }
+ return command;
+ }
+
+ private static LensCubeCommands getCubeCommand() {
+ if (cubeCommands == null) {
+ LensClient client = new LensClient();
+ cubeCommands = new LensCubeCommands();
+ cubeCommands.setClient(client);
+ }
+ return cubeCommands;
+ }
+
+ public static void testCreateSegmentation() throws IOException {
+ LensCubeSegmentationCommands command = getCommand();
+ String segList = command.showCubeSegmentations(null);
+ assertEquals(command.showCubeSegmentations("sample_cube"), "No cubesegmentation found for sample_cube");
+ assertEquals(segList, "No cubesegmentation found");
+ URL segSpec = TestLensCubeSegmentationCommands.class.getClassLoader().getResource("seg1.xml");
+ try {
+ command.createCubeSegmentation(new File(segSpec.toURI()));
+ } catch (Exception e) {
+ fail("Unable to create cubesegmentation" + e.getMessage());
+ }
+ segList = command.showCubeSegmentations(null);
+ assertEquals(command.showCubeSegmentations("sample_cube"), segList);
+ try {
+ assertEquals(command.showCubeSegmentations("blah"), segList);
+ fail();
+ } catch (NotFoundException e) {
+ log.info("blah is not a cubesegmentation", e);
+ }
+ }
+
+ public static void testUpdateSegmentation() {
+ try {
+ LensCubeSegmentationCommands command = getCommand();
+ URL segSpec = TestLensCubeSegmentationCommands.class.getClassLoader().getResource("seg1.xml");
+ StringBuilder sb = new StringBuilder();
+ BufferedReader bufferedReader = new BufferedReader(new FileReader(segSpec.getFile()));
+ String s;
+ while ((s = bufferedReader.readLine()) != null) {
+ sb.append(s).append("\n");
+ }
+
+ bufferedReader.close();
+ String xmlContent = sb.toString();
+ xmlContent = xmlContent.replace("<property name=\"seg1.prop\" value=\"s1\"/>\n",
+ "<property name=\"seg1.prop\" value=\"s1\"/>" + "\n<property name=\"seg1.prop1\" value=\"s2\"/>\n");
+
+ File newFile = new File("target/seg2.xml");
+ Writer writer = new OutputStreamWriter(new FileOutputStream(newFile));
+ writer.write(xmlContent);
+ writer.close();
+
+ String desc = command.describeCubeSegmentation("seg1");
+ log.debug(desc);
+ String propString = "seg1.prop: s1";
+ String propString1 = "seg1.prop1: s2";
+
+ assertTrue(desc.contains(propString));
+
+ command.updateCubeSegmentation("seg1", new File("target/seg2.xml"));
+ desc = command.describeCubeSegmentation("seg1");
+ log.debug(desc);
+ assertTrue(desc.contains(propString), "The sample property value is not set");
+ assertTrue(desc.contains(propString1), "The sample property value is not set");
+
+ newFile.delete();
+
+ } catch (Throwable t) {
+ log.error("Updating of the cubesegmentation failed with ", t);
+ fail("Updating of the cubesegmentation failed with " + t.getMessage());
+ }
+
+ }
+
+ public static void testDropSegmentation() {
+ LensCubeSegmentationCommands command = getCommand();
+ String segList = command.showCubeSegmentations(null);
+ assertEquals("seg1", segList, "seg1 segmentation should be found");
+ command.dropCubeSegmentation("seg1");
+ segList = command.showCubeSegmentations(null);
+ assertEquals(segList, "No cubesegmentation found");
+ }
+
+ private void dropSampleCube() {
+ getCubeCommand().dropCube("sample_cube");
+ }
+
+ @Test
+ public void testCubeSegmentationCommands() throws IOException, URISyntaxException {
+ createSampleCube();
+ testCreateSegmentation();
+ testUpdateSegmentation();
+ testDropSegmentation();
+ dropSampleCube();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cli/src/test/resources/seg1.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/seg1.xml b/lens-cli/src/test/resources/seg1.xml
new file mode 100644
index 0000000..848ac10
--- /dev/null
+++ b/lens-cli/src/test/resources/seg1.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_cube_segmentation cube_name="sample_cube" name="seg1" weight="100.0" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="seg1.prop" value="s1"/>
+ <property name="cube.segmentation.relative.start.time" value="now -10days"/>
+ </properties>
+ <cube_segements>
+ <cube_segment cube_name="cube11">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo=bar"/>
+ </segment_parameters>
+ </cube_segment>
+ <cube_segment cube_name="cube22">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo1=bar1"/>
+ </segment_parameters>
+ </cube_segment>
+ <cube_segment cube_name="cube33">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo2=bar2"/>
+ </segment_parameters>
+ </cube_segment>
+ </cube_segements>
+</x_cube_segmentation>
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index ea0cd76..39d771c 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -612,4 +612,29 @@ public class LensClient {
public Response getLogs(String logFile) {
return this.connection.getLogs(logFile);
}
+
+ public XCubeSegmentation getCubeSegmentation(String segName) {
+ return mc.getCubeSegmentation(segName);
+ }
+
+ public List<String> getAllCubeSegmentations() {
+ return mc.getAllCubeSegmentations();
+ }
+
+ public List<String> getAllCubeSegmentations(String filter) {
+ return mc.getAllCubeSegmentations(filter);
+ }
+
+ public APIResult createCubeSegmentation(String segSpec) {
+ return mc.createCubeSegmentation(segSpec);
+ }
+
+ public APIResult updateCubeSegmentation(String segName, String segSpec) {
+ return mc.updateCubeSegmentation(segName, segSpec);
+ }
+
+ public APIResult dropCubeSegmentation(String segName) {
+ return mc.dropCubeSegmentation(segName);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index d0f2b57..f2d42ca 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -359,6 +359,20 @@ public class LensMetadataClient {
return factTables.getElements();
}
+ public List<String> getAllCubeSegmentations(String cubeName) {
+ if (cubeName == null) {
+ return getAllCubeSegmentations();
+ }
+ WebTarget target = getMetastoreWebTarget();
+ StringList cubeSegmentations;
+ cubeSegmentations = target.path("cubes").path(cubeName).path("cubesegmentations")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .get(StringList.class);
+ return cubeSegmentations.getElements();
+ }
+
+
public List<String> getAllFactTables() {
WebTarget target = getMetastoreWebTarget();
StringList factTables;
@@ -370,6 +384,17 @@ public class LensMetadataClient {
return factTables.getElements();
}
+ public List<String> getAllCubeSegmentations() {
+ WebTarget target = getMetastoreWebTarget();
+ StringList cubeSegmentations;
+ cubeSegmentations = target.path("cubesegmentations")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .get(StringList.class);
+
+ return cubeSegmentations.getElements();
+ }
+
public APIResult deleteAllFactTables(boolean cascade) {
WebTarget target = getMetastoreWebTarget();
@@ -381,6 +406,14 @@ public class LensMetadataClient {
}
+ public APIResult deleteAllCubeSegmentations() {
+ WebTarget target = getMetastoreWebTarget();
+ return target.path("cubesegmentations")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .delete(APIResult.class);
+ }
+
public XFactTable getFactTable(String factTableName) {
WebTarget target = getMetastoreWebTarget();
JAXBElement<XFactTable> table = target.path("facts").path(factTableName)
@@ -391,6 +424,16 @@ public class LensMetadataClient {
return table.getValue();
}
+ public XCubeSegmentation getCubeSegmentation(String segName) {
+ WebTarget target = getMetastoreWebTarget();
+ JAXBElement<XCubeSegmentation> seg = target.path("cubesegmentations").path(segName)
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .get(new GenericType<JAXBElement<XCubeSegmentation>>() {
+ });
+ return seg.getValue();
+ }
+
public APIResult createFactTable(XFactTable f) {
WebTarget target = getMetastoreWebTarget();
return target.path("facts")
@@ -407,6 +450,23 @@ public class LensMetadataClient {
}
}
+ public APIResult createCubeSegmentation(XCubeSegmentation seg) {
+ WebTarget target = getMetastoreWebTarget();
+ return target.path("cubesegmentations")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .post(Entity.xml(new GenericEntity<JAXBElement<XCubeSegmentation>>(objFact
+ .createXCubeSegmentation(seg)){}), APIResult.class);
+ }
+
+ public APIResult createCubeSegmentation(String segSpec) {
+ try {
+ return createCubeSegmentation(this.<XCubeSegmentation>readFromXML(segSpec));
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
+ }
+ }
+
public APIResult updateFactTable(String factName, XFactTable table) {
WebTarget target = getMetastoreWebTarget();
return target.path("facts").path(factName)
@@ -423,6 +483,24 @@ public class LensMetadataClient {
}
}
+ public APIResult updateCubeSegmentation(String segName, XCubeSegmentation seg) {
+ WebTarget target = getMetastoreWebTarget();
+ return target.path("cubesegmentations").path(segName)
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML_TYPE)
+ .put(Entity.xml(new GenericEntity<JAXBElement<XCubeSegmentation>>(objFact.
+ createXCubeSegmentation(seg)){}), APIResult.class);
+ }
+
+ public APIResult updateCubeSegmentation(String segName, String seg) {
+ try {
+ return updateCubeSegmentation(segName, this.<XCubeSegmentation>readFromXML(seg));
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
+ }
+ }
+
+
public APIResult dropFactTable(String factName, boolean cascade) {
WebTarget target = getMetastoreWebTarget();
return target.path("facts").path(factName)
@@ -436,6 +514,14 @@ public class LensMetadataClient {
return dropFactTable(factName, false);
}
+ public APIResult dropCubeSegmentation(String segName) {
+ WebTarget target = getMetastoreWebTarget();
+ return target.path("cubesegmentations").path(segName)
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .delete(APIResult.class);
+ }
+
public List<String> getAllStoragesOfFactTable(String factName) {
WebTarget target = getMetastoreWebTarget();
StringList storageList = target.path("facts").path(factName).path("storages")
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
index 01098c4..67aaff8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
@@ -20,6 +20,9 @@ package org.apache.lens.cube.metadata;
import java.util.*;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -179,6 +182,23 @@ public abstract class AbstractCubeTable implements Named {
return true;
}
+ public Date getDateFromProperty(String propKey, boolean relative, boolean start) {
+ String prop = getProperties().get(propKey);
+ try {
+ if (StringUtils.isNotBlank(prop)) {
+ if (relative) {
+ return DateUtil.resolveRelativeDate(prop, now());
+ } else {
+ return DateUtil.resolveAbsoluteDate(prop);
+ }
+ }
+ } catch (LensException e) {
+ log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop);
+ }
+ return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE;
+ }
+
+
@Override
public String toString() {
return getName();
@@ -200,4 +220,9 @@ public abstract class AbstractCubeTable implements Named {
}
return columns;
}
+
+ public Date now() {
+ return new Date();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index 643bcfe..fb958c3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -323,22 +323,6 @@ public class CubeFactTable extends AbstractCubeTable {
getProperties().put(MetastoreConstants.FACT_AGGREGATED_PROPERTY, Boolean.toString(isAggregated));
}
- public Date getDateFromProperty(String propKey, boolean relative, boolean start) {
- String prop = getProperties().get(propKey);
- try {
- if (StringUtils.isNotBlank(prop)) {
- if (relative) {
- return DateUtil.resolveRelativeDate(prop, now());
- } else {
- return DateUtil.resolveAbsoluteDate(prop);
- }
- }
- } catch (LensException e) {
- log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop);
- }
- return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE;
- }
-
public Date getAbsoluteStartTime() {
return getDateFromProperty(MetastoreConstants.FACT_ABSOLUTE_START_TIME, false, true);
}
@@ -363,7 +347,5 @@ public class CubeFactTable extends AbstractCubeTable {
return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime()));
}
- public Date now() {
- return new Date();
- }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 02bbbbd..a29f42e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -80,6 +80,9 @@ public class CubeMetastoreClient {
// map from fact name to fact table
private final Map<String, CubeFactTable> allFactTables = Maps.newConcurrentMap();
private volatile boolean allFactTablesPopulated = false;
+ //map from segmentation name to segmentation
+ private final Map<String, CubeSegmentation> allCubeSegmentations = Maps.newConcurrentMap();
+ private volatile boolean allCubeSegmentationPopulated = false;
// map from storage name to storage
private final Map<String, Storage> allStorages = Maps.newConcurrentMap();
private volatile boolean allStoragesPopulated = false;
@@ -715,6 +718,24 @@ public class CubeMetastoreClient {
}
/**
+ *
+ * @param baseCubeName The cube name ot which segmentation belong to
+ * @param segmentationName The segmentation name
+ * @param cubeSegments Participating cube segements
+ * @param weight Weight of segmentation
+ * @param properties Properties of segmentation
+ * @throws HiveException
+ */
+ public void createCubeSegmentation(String baseCubeName, String segmentationName, Set<CubeSegment> cubeSegments,
+ double weight, Map<String, String> properties) throws HiveException {
+ CubeSegmentation cubeSeg =
+ new CubeSegmentation(baseCubeName, segmentationName, cubeSegments, weight, properties);
+ createCubeSegmentation(cubeSeg);
+ // do a get to update cache
+ getCubeSegmentation(segmentationName);
+ }
+
+ /**
* Create a cube dimension table
*
* @param dimName The dimension name to which the dim-table belongs to
@@ -777,6 +798,12 @@ public class CubeMetastoreClient {
}
}
+ public void createCubeSegmentation(CubeSegmentation cubeSeg)
+ throws HiveException {
+ // create virtual cube table in metastore
+ createCubeHiveTable(cubeSeg);
+ }
+
/**
* Adds storage to fact and creates corresponding storage table
*
@@ -1402,6 +1429,17 @@ public class CubeMetastoreClient {
return CubeTableType.FACT.name().equals(tableType);
}
+ public boolean isCubeSegmentation(String segName) throws HiveException {
+ Table tbl = getTable(segName);
+ return isCubeSegmentation(tbl);
+ }
+
+ boolean isCubeSegmentation(Table tbl) {
+ String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY);
+ return CubeTableType.SEGMENTATION.name().equals(tableType);
+ }
+
+
boolean isFactTableForCube(Table tbl, String cube) {
return isFactTable(tbl) && CubeFactTable.getCubeName(tbl.getTableName(), tbl.getParameters())
.equalsIgnoreCase(cube.toLowerCase());
@@ -1526,6 +1564,14 @@ public class CubeMetastoreClient {
return isFactTable(tbl) ? new CubeFactTable(tbl) : null;
}
+ public CubeSegmentation getCubeSegmentationTable(String tableName) throws HiveException {
+ return getCubeSegmentationTable(getTable(tableName));
+ }
+
+ private CubeSegmentation getCubeSegmentationTable(Table tbl) throws HiveException {
+ return isCubeSegmentation(tbl) ? new CubeSegmentation(tbl) : null;
+ }
+
/**
* Get {@link CubeDimensionTable} object corresponding to the name
*
@@ -1690,6 +1736,25 @@ public class CubeMetastoreClient {
return fact;
}
+ public CubeSegmentation getCubeSegmentation(String segName) throws HiveException {
+ segName = segName.trim().toLowerCase();
+ CubeSegmentation seg = allCubeSegmentations.get(segName);
+ if (seg == null) {
+ synchronized (allCubeSegmentations) {
+ if (!allCubeSegmentations.containsKey(segName)) {
+ seg = getCubeSegmentationTable(segName);
+ if (enableCaching && seg != null) {
+ allCubeSegmentations.put(segName, seg);
+ }
+ } else {
+ seg = allCubeSegmentations.get(segName);
+ }
+ }
+ }
+ return seg;
+ }
+
+
private CubeInterface getCube(Table tbl) throws HiveException {
String parentCube = tbl.getParameters().get(getParentCubeNameKey(tbl.getTableName()));
if (parentCube != null) {
@@ -1833,6 +1898,34 @@ public class CubeMetastoreClient {
}
}
+ /**
+ * Get all cube segmentations in metastore
+ *
+ * @return List of cube segmentation objects
+ * @throws HiveException
+ */
+ public Collection<CubeSegmentation> getAllCubeSegmentations() throws HiveException {
+ if (!allCubeSegmentationPopulated) {
+ List<CubeSegmentation> segs = new ArrayList<>();
+ try {
+ for (String table : getAllHiveTableNames()) {
+ CubeSegmentation seg = getCubeSegmentation(table);
+ if (seg != null) {
+ segs.add(seg);
+ }
+ }
+ } catch (HiveException e) {
+ throw new HiveException("Could not get all fact tables", e);
+ }
+ allFactTablesPopulated = enableCaching;
+ return segs;
+ } else {
+ return allCubeSegmentations.values();
+ }
+ }
+
+
+
private Collection<String> getAllHiveTableNames() throws HiveException {
if (!allTablesPopulated) {
List<String> allTables = getClient().getAllTables();
@@ -1875,6 +1968,28 @@ public class CubeMetastoreClient {
return cubeFacts;
}
+ public List<CubeSegmentation> getAllCubeSegmentations(CubeInterface cube) throws HiveException {
+ String cubeName = null;
+ if (cube != null) {
+ if (cube instanceof DerivedCube) {
+ cube = ((DerivedCube) cube).getParent();
+ }
+ cubeName = cube.getName();
+ }
+ List<CubeSegmentation> cubeSegs = new ArrayList<>();
+ try {
+ for (CubeSegmentation seg : getAllCubeSegmentations()) {
+ if (cubeName == null || seg.getBaseCube().equalsIgnoreCase(cubeName)) {
+ cubeSegs.add(seg);
+ }
+ }
+ } catch (HiveException e) {
+ throw new HiveException("Could not get all segmentations of " + cube, e);
+ }
+ return cubeSegs;
+ }
+
+
/**
* Get all derived cubes of the cube, that have all fields queryable together
*
@@ -2093,6 +2208,16 @@ public class CubeMetastoreClient {
}
}
+
+ public void dropCubeSegmentation(String segName) throws HiveException {
+ if (isCubeSegmentation(segName)) {
+ dropHiveTable(segName);
+ allCubeSegmentations.remove(segName.trim().toLowerCase());
+ } else {
+ throw new HiveException(segName + " is not a CubeSegmentation");
+ }
+ }
+
/**
* Drop a storage from fact
*
@@ -2191,6 +2316,26 @@ public class CubeMetastoreClient {
}
}
+ public void alterCubeSegmentation(String segName, CubeSegmentation seg)
+ throws HiveException {
+ Table segTbl = getTable(segName);
+ if (isCubeSegmentation(segTbl)) {
+ if (!(getCubeSegmentation(segName) == seg)) {
+ dropCubeSegmentation(segName);
+ createCubeSegmentation(seg);
+ updateSegmentationCache(segName);
+ }
+ } else {
+ throw new HiveException(segName + " is not a cube segment");
+ }
+ }
+
+ private void updateSegmentationCache(String segmentName) throws HiveException {
+ if (enableCaching) {
+ allCubeSegmentations.put(segmentName.trim().toLowerCase(), getCubeSegmentationTable(refreshTable(segmentName)));
+ }
+ }
+
private void updateFactCache(String factTableName) throws HiveException {
if (enableCaching) {
allFactTables.put(factTableName.trim().toLowerCase(), getFactTable(refreshTable(factTableName)));
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java
new file mode 100644
index 0000000..8143c3b
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegment.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.Map;
+
+import lombok.Getter;
+
+public class CubeSegment implements Named {
+ @Getter
+ private String name;
+ @Getter
+ private Map<String, String> properties;
+
+ public CubeSegment(String name, Map<String, String> properties) {
+ this.name = name;
+ this.properties = properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java
new file mode 100644
index 0000000..a93fafe
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeSegmentation.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.metadata;
+
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import com.google.common.collect.Lists;
+
+public class CubeSegmentation extends AbstractCubeTable {
+
+ private String cubeName;
+ private final Set<CubeSegment> cubeSegments;
+ private static final List<FieldSchema> COLUMNS = new ArrayList<>();
+
+ static {
+ COLUMNS.add(new FieldSchema("dummy", "string", "dummy column"));
+ }
+
+ public CubeSegmentation(Table hiveTable) {
+ super(hiveTable);
+ this.cubeSegments = getCubeSegments(getName(), getProperties());
+ this.cubeName = getCubeName(getName(), getProperties());
+ }
+
+ public CubeSegmentation(String cubeName, String segmentName, Set<CubeSegment> cubeSegments) {
+ this(cubeName, segmentName, cubeSegments, 0L);
+ }
+
+ public CubeSegmentation(String cubeName, String segmentName, Set<CubeSegment> cubeSegments, double weight) {
+ this(cubeName, segmentName, cubeSegments, weight, new HashMap<String, String>());
+ }
+
+ public CubeSegmentation(String baseCube, String segmentName, Set<CubeSegment> cubeSegments,
+ double weight, Map<String, String> properties) {
+ super(segmentName, COLUMNS, properties, weight);
+ this.cubeName = baseCube;
+ this.cubeSegments = cubeSegments;
+ addProperties();
+ }
+
+ public static Set<String> getSegmentNames(Set<CubeSegment> segments) {
+ Set<String> names = new HashSet<>();
+ for (CubeSegment seg : segments) {
+ names.add(seg.getName());
+ }
+ return names;
+ }
+
+ @Override
+ protected void addProperties() {
+ super.addProperties();
+ addCubeName(getName(), getProperties(), cubeName);
+ addCubeSegmentProperties(getName(), getProperties(), cubeSegments);
+ }
+
+ private static void addCubeName(String segName, Map<String, String> props, String cubeName) {
+ props.put(MetastoreUtil.getSegmentationCubeNameKey(segName), cubeName);
+ }
+
+ private static void addCubeSegmentProperties(String name, Map<String, String> props,
+ Set<CubeSegment> cubeSegments) {
+ if (cubeSegments != null){
+ props.put(MetastoreUtil.getSegmentsListKey(name),
+ MetastoreUtil.getNamedStr(cubeSegments));
+ for (CubeSegment cubeSegment : cubeSegments) {
+ for (Map.Entry<String, String> segProp : cubeSegment.getProperties().entrySet()) {
+ if (!segProp.getKey().startsWith(MetastoreUtil.getSegmentPropertyKey(cubeSegment.getName()))) {
+ props.put(MetastoreUtil.getSegmentPropertyKey(cubeSegment.getName()).concat(segProp.getKey()),
+ segProp.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ private static Set<CubeSegment> getCubeSegments(String name, Map<String, String> props) {
+ Set<CubeSegment> cubeSegments = new HashSet<>();
+ String segmentsString = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getSegmentsListKey(name));
+ if (!StringUtils.isBlank(segmentsString)) {
+ String[] segments = segmentsString.split(",");
+ for (String seg : segments) {
+ Map<String, String> segProps = new HashMap<>();
+ String segmentPropStr = MetastoreUtil.getSegmentPropertyKey(seg);
+ for (String key : props.keySet()) {
+ if (key.startsWith(segmentPropStr)){
+ segProps.put(key, props.get(key));
+ }
+ }
+ cubeSegments.add(new CubeSegment(seg, segProps));
+ }
+ }
+ return cubeSegments;
+ }
+
+ public void addCubeSegment(CubeSegment cubeSeg) {
+ if (!cubeSegments.contains(cubeSeg)) {
+ cubeSegments.add(cubeSeg);
+ addCubeSegmentProperties(getName(), getProperties(), cubeSegments);
+ }
+ }
+
+ public void dropCubeSegment(CubeSegment cubeSeg) {
+ if (cubeSegments.contains(cubeSeg)) {
+ cubeSegments.remove(cubeSeg);
+ addCubeSegmentProperties(getName(), getProperties(), cubeSegments);
+ }
+ }
+
+ public void alterCubeSegment(Set<CubeSegment> cubeSegs) {
+ if (!cubeSegments.equals(cubeSegs)) {
+ cubeSegments.clear();
+ cubeSegments.addAll(cubeSegs);
+ addCubeSegmentProperties(getName(), getProperties(), cubeSegments);
+ }
+ }
+
+ public void alterBaseCubeName(String cubeName) {
+ this.cubeName = cubeName;
+ addCubeName(getName(), getProperties(), cubeName);
+ }
+
+ public String getBaseCube() {
+ return cubeName;
+ }
+
+ public Set<CubeSegment> getCubeSegments() {
+ return cubeSegments;
+ }
+
+ @Override
+ public CubeTableType getTableType() {
+ return CubeTableType.SEGMENTATION;
+ }
+
+ @Override
+ public Set<String> getStorages() {
+ return null;
+ }
+
+
+ public Date getAbsoluteStartTime() {
+ return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME, false, true);
+ }
+
+ public Date getRelativeStartTime() {
+ return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME, true, true);
+ }
+
+ public Date getStartTime() {
+ return Collections.max(Lists.newArrayList(getRelativeStartTime(), getAbsoluteStartTime()));
+ }
+
+ public Date getAbsoluteEndTime() {
+ return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME, false, false);
+ }
+
+ public Date getRelativeEndTime() {
+ return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME, true, false);
+ }
+
+ public Date getEndTime() {
+ return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime()));
+ }
+ static String getCubeName(String segName, Map<String, String> props) {
+ return props.get(MetastoreUtil.getSegmentationCubeNameKey(segName));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java
index ec1c837..6dd35b3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeTableType.java
@@ -20,5 +20,5 @@
package org.apache.lens.cube.metadata;
public enum CubeTableType {
- CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE
+ CUBE, DIMENSION, FACT, DIM_TABLE, STORAGE, SEGMENTATION
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index ed78d74..cd4862c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -56,6 +56,15 @@ public final class MetastoreConstants {
public static final String FACT_ABSOLUTE_END_TIME = "cube.fact.absolute.end.time";
public static final String FACT_RELATIVE_END_TIME = "cube.fact.relative.end.time";
+ // cube segmentation constants
+ public static final String SEGMENTATION_KEY_PFX = "cube.segmentation.internal.";
+ public static final String SEGMENTATION_ABSOLUTE_START_TIME = "cube.segmentation.absolute.start.time";
+ public static final String SEGMENTATION_RELATIVE_START_TIME = "cube.segmentation.relative.start.time";
+ public static final String SEGMENTATION_ABSOLUTE_END_TIME = "cube.segmentation.absolute.end.time";
+ public static final String SEGMENTATION_RELATIVE_END_TIME = "cube.segmentation.relative.end.time";
+ public static final String SEGMENTATION_CUBE_SEGMENT_SFX = ".cubesegments";
+ public static final String SEGMENT_PROP_SFX = ".props.";
+
// dim table constants
// TODO: remove this and move to "dimtable."
public static final String DIM_TBL_PFX = "dimtble.";
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index deb5368..399d5a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -359,6 +359,26 @@ public class MetastoreUtil {
}
// //////////////////////////
+ // Segmentation propertes ///
+ // /////////////////////////
+
+ public static String getSegmentationKeyPrefix(String segName) {
+ return SEGMENTATION_KEY_PFX + segName.toLowerCase();
+ }
+
+ public static String getSegmentationCubeNameKey(String name) {
+ return getSegmentationKeyPrefix(name) + CUBE_NAME_SFX;
+ }
+
+ public static String getSegmentsListKey(String name) {
+ return getSegmentationKeyPrefix(name) + SEGMENTATION_CUBE_SEGMENT_SFX;
+ }
+
+ public static String getSegmentPropertyKey(String segName) {
+ return getSegmentationKeyPrefix(segName) + SEGMENT_PROP_SFX;
+ }
+
+ // //////////////////////////
// Utils ///
// /////////////////////////
public static <E extends Named> String getNamedStr(Collection<E> set) {
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java
index 0935509..42e32cb 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java
@@ -64,6 +64,7 @@ public class CubeFactTableTest {
private CubeFactTable getMockCubeFactTable(Map<String, String> properties) {
CubeFactTable cubeFactTable = mock(CubeFactTable.class);
+
when(cubeFactTable.now()).thenReturn(now);
when(cubeFactTable.getProperties()).thenReturn(properties);
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 65387c6..18575f6 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -980,6 +980,53 @@ public class TestCubeMetastoreClient {
assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey()));
}
+ @Test(priority = 1)
+ public void testCubeSegmentation() throws Exception {
+ String segmentName = "testMetastoreCubeSegmentation";
+
+ Table cubeTbl = client.getHiveTable(CUBE_NAME);
+ assertTrue(client.isCube(cubeTbl));
+
+ Map<String, String> props = new HashMap<String, String>(){{put("foo", "bar"); }};
+ Map<String, String> prop1 = new HashMap<String, String>(){{put("foo1", "bar1"); put("foo11", "bar11"); }};
+ Map<String, String> prop2 = new HashMap<String, String>(){{put("foo2", "bar2"); }};
+ Map<String, String> prop3 = new HashMap<String, String>(){{put("foo3", "bar3"); }};
+ Map<String, String> prop5 = new HashMap<String, String>(){{put("foo5", "bar5"); }};
+
+ CubeSegment seg1 = new CubeSegment("cube1", prop1);
+ CubeSegment seg2 = new CubeSegment("cube2", prop2);
+ CubeSegment seg3 = new CubeSegment("cube3", prop3);
+ CubeSegment seg5 = new CubeSegment("cube5", prop5);
+
+ Set<CubeSegment> cubeSegs = Sets.newHashSet(seg1, seg2, seg3);
+
+ //create cube segmentation
+ client.createCubeSegmentation(CUBE_NAME, segmentName, cubeSegs, 0L, props);
+ assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 3);
+
+ //Alter cube segmentation
+ CubeSegmentation segmentation = new CubeSegmentation(Hive.get(conf).getTable(segmentName));
+ segmentation.addCubeSegment(seg5);
+ segmentation.addProperties(new HashMap<String, String>(){{put("new_key", "new_val"); }});
+ segmentation.alterBaseCubeName("segCubeAltered");
+ segmentation.alterWeight(100.0);
+ client.alterCubeSegmentation(segmentName, segmentation);
+
+ assertNotNull(client.getCubeSegmentation(segmentName));
+ assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 4);
+ assertEquals(client.getCubeSegmentation(segmentName).getBaseCube(), "segCubeAltered");
+ assertEquals(client.getCubeSegmentation(segmentName).weight(), 100.0);
+
+ //drop cubesegment to segmentation
+ segmentation.dropCubeSegment(seg5);
+ client.alterCubeSegmentation(segmentName, segmentation);
+ assertEquals(client.getCubeSegmentation(segmentName).getCubeSegments().size(), 3);
+
+ //drop segmentation
+ client.dropCubeSegmentation(segmentName);
+ assertFalse(client.tableExists(segmentName));
+ }
+
private void assertRangeValidityForStorageTable(String storageTable) throws HiveException, LensException {
Object[][] testCases = new Object[][] {
{"now - 15 days", "now - 11 days", false},
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
index 0c2301b..626655d 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
@@ -117,6 +117,7 @@ public class SampleMetastore {
createCubes();
createDimensions();
createFacts();
+ createCubeSegmentations();
createDimensionTables();
try {
DatabaseUtil.initializeDatabaseStorage();
@@ -166,6 +167,15 @@ public class SampleMetastore {
createFact("sales-aggr-continuous-fact.xml");
}
+ private void createCubeSegmentations() throws JAXBException, IOException {
+ result = metaClient.createCubeSegmentation("seg1.xml");
+ if (result.getStatus().equals(APIResult.Status.FAILED)) {
+ System.err.println("Creating cubesegmentation from : " + "seg1.xml"
+ + " failed, reason:" + result.getMessage());
+ retCode = 1;
+ }
+ }
+
public static void main(String[] args) throws Exception {
SampleMetastore metastore = null;
try {
@@ -185,6 +195,7 @@ public class SampleMetastore {
System.out.println("Dimensions:" + metastore.metaClient.getAllDimensions());
System.out.println("Fact tables:" + metastore.metaClient.getAllFactTables());
System.out.println("Dimension tables:" + metastore.metaClient.getAllDimensionTables());
+ System.out.println("CubeSegmentations:" + metastore.metaClient.getAllCubeSegmentations());
if (metastore.retCode != 0) {
System.exit(metastore.retCode);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/main/resources/seg1.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/seg1.xml b/lens-examples/src/main/resources/seg1.xml
new file mode 100644
index 0000000..c1576d7
--- /dev/null
+++ b/lens-examples/src/main/resources/seg1.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_cube_segmentation cube_name="sample_cube" name="seg1" weight="100.0" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name= "cube.segmentation.relative.start.time" value="now -10days"/>
+ </properties>
+ <cube_segements>
+ <cube_segment cube_name="cube11">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo=bar"/>
+ </segment_parameters>
+ </cube_segment>
+ <cube_segment cube_name="cube22">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo1=bar1"/>
+ </segment_parameters>
+ </cube_segment>
+ <cube_segment cube_name="cube33">
+ <segment_parameters>
+ <property name="lens.metastore.cube.column.mapping" value="foo2=bar2"/>
+ </segment_parameters>
+ </cube_segment>
+ </cube_segements>
+</x_cube_segmentation>
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-examples/src/test/resources/yaml/seg1.yaml
----------------------------------------------------------------------
diff --git a/lens-examples/src/test/resources/yaml/seg1.yaml b/lens-examples/src/test/resources/yaml/seg1.yaml
new file mode 100644
index 0000000..dd2f339
--- /dev/null
+++ b/lens-examples/src/test/resources/yaml/seg1.yaml
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+properties:
+ cube.segmentation.relative.start.time: now -10days
+cubeSegements:
+ -
+ segmentParameters:
+ lens.metastore.cube.column.mapping: foo=bar
+ cubeName: cube11
+ -
+ segmentParameters:
+ lens.metastore.cube.column.mapping: foo1=bar1
+ cubeName: cube22
+ -
+ segmentParameters:
+ lens.metastore.cube.column.mapping: foo2=bar2
+ cubeName: cube33
+name: seg1
+cubeName: sample_cube
+weight: 100.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 52a7ccc..f8a936a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -412,6 +412,11 @@ public final class LensConfConstants {
public static final String NATIVE_TABLE_COLUMN_MAPPING = METASTORE_PFX + "native.table.column.mapping";
/**
+ * The property name for setting the column mapping, if column names in cubes are different
+ */
+ public static final String CUBE_COLUMN_MAPPING = METASTORE_PFX + "cube.column.mapping";
+
+ /**
* The Constant ES_INDEX_NAME.
*/
public static final String ES_INDEX_NAME = METASTORE_PFX + "es.index.name";
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index c11fd83..431164f 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -570,4 +570,53 @@ public interface CubeMetastoreService extends LensService {
void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName,
XPartitionList partitions) throws LensException;
+
+ /**
+ *
+ * @param sessionid The session id
+ * @param cubeSeg The cube segmentation
+ * @throws LensException
+ */
+ void createCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException;
+
+ /**
+ * Create cube segmentation
+ *
+ * @param sessionid The session id
+ * @param segName Cube segmentation name
+ * @return {@link XCubeSegmentation}
+ * @throws LensException
+ */
+ XCubeSegmentation getCubeSegmentation(LensSessionHandle sessionid, String segName) throws LensException;
+
+ /**
+ * Get cube segmentation given by name
+ *
+ * @param sessionid The session id
+ * @param cubeSeg The cube segmentation
+ * @throws LensException
+ */
+
+ void updateCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException;
+
+ /**
+ * Update cube segmentation
+ *
+ * @param sessionid The session id
+ * @param cubeSegName Cube segmentation name
+ * @throws LensException
+ */
+ void dropCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException;
+
+ /**
+ * Get all cube segmentations belong to Cube
+ *
+ * @param sessionid The session id
+ * @param cubeName The cube Name
+ * @return
+ * @throws LensException
+ */
+ List<String> getAllCubeSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException;
+
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index a1acd1a..0127ef8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -507,6 +507,21 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
+ public XCubeSegmentation getCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException {
+ try {
+ acquire(sessionid);
+ CubeMetastoreClient msClient = getClient(sessionid);
+ CubeSegmentation cubeSeg = msClient.getCubeSegmentation(cubeSegName);
+ return JAXBUtils.xsegmentationFromCubeSegmentation(cubeSeg);
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionid);
+ }
+ }
+
+
+ @Override
public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException {
try {
acquire(sessionid);
@@ -526,6 +541,25 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
+ public void createCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException {
+ try {
+ acquire(sessionid);
+ getClient(sessionid).createCubeSegmentation(
+ cubeSeg.getCubeName(),
+ cubeSeg.getName(),
+ JAXBUtils.cubeSegmentsFromXCubeSegments(cubeSeg.getCubeSegements()),
+ cubeSeg.getWeight(),
+ JAXBUtils.mapFromXProperties(cubeSeg.getProperties()));
+ log.info("Created cube segmentation " + cubeSeg.getName());
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionid);
+ }
+ }
+
+
+ @Override
public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException {
try {
acquire(sessionid);
@@ -540,6 +574,19 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
+ public void updateCubeSegmentation(LensSessionHandle sessionid, XCubeSegmentation cubeSeg) throws LensException {
+ try {
+ acquire(sessionid);
+ getClient(sessionid).alterCubeSegmentation(cubeSeg.getName(), cubeSegmentationFromXCubeSegmentation(cubeSeg));
+ log.info("Updated cube segmentation " + cubeSeg.getName());
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionid);
+ }
+ }
+
+ @Override
public void dropFactTable(LensSessionHandle sessionid, String fact, boolean cascade) throws LensException {
try {
acquire(sessionid);
@@ -553,6 +600,20 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
+ public void dropCubeSegmentation(LensSessionHandle sessionid, String cubeSegName) throws LensException {
+ try {
+ acquire(sessionid);
+ getClient(sessionid).dropCubeSegmentation(cubeSegName);
+ log.info("Dropped cube segemntation " + cubeSegName);
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionid);
+ }
+
+ }
+
+ @Override
public List<String> getAllFactNames(LensSessionHandle sessionid, String cubeName) throws LensException {
try {
acquire(sessionid);
@@ -574,6 +635,29 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
}
+ @Override
+ public List<String> getAllCubeSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException {
+ try {
+ acquire(sessionid);
+ CubeMetastoreClient client = getClient(sessionid);
+ CubeInterface seg = client.getCube(cubeName);
+ if (cubeName != null && seg == null) {
+ throw new LensException("Could not get table: " + cubeName + " as a cube");
+ }
+ Collection<CubeSegmentation> segs = client.getAllCubeSegmentations(seg);
+ List<String> segNames = new ArrayList<String>(segs.size());
+ for (CubeSegmentation cs : segs) {
+ segNames.add(cs.getName());
+ }
+ return segNames;
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionid);
+ }
+
+ }
+
@Override
public List<String> getStoragesOfFact(LensSessionHandle sessionid, String fact) throws LensException {
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 6fd19a0..6571dae 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -490,6 +490,34 @@ public final class JAXBUtils {
return xpList;
}
+ public static Set<XCubeSegment> xCubeSegmentsFromCubeSegments(Set<CubeSegment> segs) {
+ Set<XCubeSegment> xsegs = new HashSet<XCubeSegment>();
+ if (segs != null && !segs.isEmpty()) {
+ for (CubeSegment seg : segs) {
+ XCubeSegment xcubeSeg = XCF.createXCubeSegment();
+ xcubeSeg.setCubeName(seg.getName());
+ xcubeSeg.setSegmentParameters(getXpropertiesFromCubeSegment(seg));
+ xsegs.add(xcubeSeg);
+ }
+ }
+ return xsegs;
+ }
+
+ public static XProperties getXpropertiesFromCubeSegment(CubeSegment cseg) {
+ XProperties xproperties = XCF.createXProperties();
+ for (String prop : cseg.getProperties().keySet()) {
+ String segPrefix = MetastoreUtil.getSegmentPropertyKey(cseg.getName());
+ if (prop.startsWith(segPrefix)){
+ XProperty xprop = XCF.createXProperty();
+ xprop.setName(prop.replace(segPrefix, ""));
+ xprop.setValue(cseg.getProperties().get(prop));
+ xproperties.getProperty().add(xprop);
+ }
+ }
+ return xproperties;
+ }
+
+
public static FieldSchema fieldSchemaFromColumn(XColumn c) {
if (c == null) {
return null;
@@ -659,6 +687,24 @@ public final class JAXBUtils {
mapFromXProperties(fact.getProperties()));
}
+ public static CubeSegmentation cubeSegmentationFromXCubeSegmentation(XCubeSegmentation seg) throws LensException {
+
+ Map<String, String> props = new HashMap<>();
+ // Skip properties with keyword internal. These properties are internal to lens
+ // and users are not supposed to see them.
+ for(String prop : mapFromXProperties(seg.getProperties()).keySet()) {
+ if (!(prop.toLowerCase().startsWith(MetastoreConstants.SEGMENTATION_KEY_PFX))) {
+ props.put(prop, mapFromXProperties(seg.getProperties()).get(prop));
+ }
+ }
+ return new CubeSegmentation(seg.getCubeName(),
+ seg.getName(),
+ cubeSegmentsFromXCubeSegments(seg.getCubeSegements()),
+ seg.getWeight(),
+ props);
+ }
+
+
public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) {
XFactTable fact = XCF.createXFactTable();
fact.setName(cFact.getName());
@@ -673,6 +719,21 @@ public final class JAXBUtils {
return fact;
}
+ public static XCubeSegmentation xsegmentationFromCubeSegmentation(CubeSegmentation cSeg) {
+ XCubeSegmentation seg = XCF.createXCubeSegmentation();
+ seg.setName(cSeg.getName());
+ seg.setProperties(new XProperties());
+ seg.setCubeSegements(new XCubeSegments());
+ seg.setWeight(cSeg.weight());
+ seg.setCubeName(cSeg.getBaseCube());
+ if (xPropertiesFromMap(cSeg.getProperties()) != null) {
+ seg.getProperties().getProperty().addAll(xPropertiesFromMap(cSeg.getProperties()));
+ }
+ seg.getCubeSegements().getCubeSegment().
+ addAll(xCubeSegmentsFromCubeSegments(cSeg.getCubeSegments()));
+ return seg;
+ }
+
public static StorageTableDesc storageTableDescFromXStorageTableDesc(
XStorageTableDesc xtableDesc) {
StorageTableDesc tblDesc = new StorageTableDesc();
@@ -770,6 +831,20 @@ public final class JAXBUtils {
return storageTableMap;
}
+ public static Set<CubeSegment> cubeSegmentsFromXCubeSegments(XCubeSegments segs) {
+ Set<CubeSegment> cubeSegs = new HashSet<>();
+ for (XCubeSegment xcube : segs.getCubeSegment()){
+ Map<String, String> segProp = new HashMap<>();
+ if (xcube.getSegmentParameters() != null) {
+ for (XProperty prop : xcube.getSegmentParameters().getProperty()) {
+ segProp.put(prop.getName(), prop.getValue());
+ }
+ }
+ cubeSegs.add(new CubeSegment(xcube.getCubeName(), segProp));
+ }
+ return cubeSegs;
+ }
+
public static Map<String, Date> timePartSpecfromXTimePartSpec(
XTimePartSpec xtimePartSpec) {
Map<String, Date> timePartSpec = new HashMap<String, Date>();
http://git-wip-us.apache.org/repos/asf/lens/blob/e4f26aae/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 684e2ed..7395c83 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -142,7 +142,23 @@ public class MetastoreResource {
getSvc().dropFactTable(sessionid, entityName, cascade);
}
}
- }, DIMENSION {
+ },
+ SEGMENTATION {
+ @Override
+ public List<String> doGetAll(LensSessionHandle sessionid) throws LensException {
+ return getSvc().getAllCubeSegmentations(sessionid, null);
+ }
+
+ @Override
+ public void doDelete(LensSessionHandle sessionid, String entityName, Boolean cascade) throws LensException {
+ if (cascade == null) {
+ getSvc().dropCubeSegmentation(sessionid, entityName);
+ } else {
+ throw new NotImplementedException();
+ }
+ }
+ }
+ , DIMENSION {
@Override
public List<String> doGetAll(LensSessionHandle sessionid) throws LensException {
return getSvc().getAllDimensionNames(sessionid);
@@ -791,6 +807,28 @@ public class MetastoreResource {
}
/**
+ * Get all cube segmentations that belong to a cube in the metastore
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param cubeName name of the base cube or derived cube
+ * @return List of {@link XCubeSegmentation} objects
+ */
+ @GET
+ @Path("/cubes/{cubeName}/cubesegmentations")
+ public StringList getAllCubeSegmentationsOfCube(
+ @QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("cubeName") String cubeName)
+ throws LensException {
+ checkSessionId(sessionid);
+ try {
+ return new StringList(getSvc().getAllCubeSegmentations(sessionid, cubeName));
+ } catch (LensException exc) {
+ checkTableNotFound(exc, cubeName);
+ throw exc;
+ }
+ }
+
+
+ /**
* Get all fact tables in the metastore in the current database
*
* @param sessionid The sessionid in which user is working
@@ -803,6 +841,22 @@ public class MetastoreResource {
return Entity.FACT.getAll(sessionid);
}
+
+ /**
+ * Get all cube segmentations in the current database
+ *
+ * @param sessionid The sessionid in which user is working
+ * @return StringList consisting of all cube segmentations
+ */
+ @GET
+ @Path("/cubesegmentations")
+ public StringList getAllCubeSegmentations(@QueryParam("sessionid") LensSessionHandle sessionid)
+ throws LensException {
+ checkSessionId(sessionid);
+ return Entity.SEGMENTATION.getAll(sessionid);
+ }
+
+
/**
* Delete all fact tables
*
@@ -820,6 +874,19 @@ public class MetastoreResource {
}
/**
+ * Delete all cube segmentations
+ *
+ * @param sessionid The sessionid in which user is working
+ * @return APIResult with state {@link Status#SUCCEEDED} in case of successful delete. APIResult with state {@link
+ * Status#FAILED} in case of delete failure. APIResult with state {@link Status#PARTIAL} in case of partial delete.
+ */
+ @DELETE
+ @Path("cubesegmentations")
+ public APIResult deleteAllCubeSegmentations(@QueryParam("sessionid") LensSessionHandle sessionid) {
+ return Entity.SEGMENTATION.deleteAll(sessionid, null);
+ }
+
+ /**
* Get the fact table specified by name
*
* @param sessionid The sessionid in which user is working
@@ -841,6 +908,28 @@ public class MetastoreResource {
}
/**
+ * Get the cube segmentation specified by name
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param cubeSegmentationName The cube segmentation name
+ * @return JAXB representation of {@link XCubeSegmentation}
+ */
+ @GET
+ @Path("/cubesegmentations/{cubeSegmentationName}")
+ public JAXBElement<XCubeSegmentation> getCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("cubeSegmentationName") String cubeSegmentationName)
+ throws LensException {
+ checkSessionId(sessionid);
+ try {
+ return X_CUBE_OBJECT_FACTORY.createXCubeSegmentation(getSvc().
+ getCubeSegmentation(sessionid, cubeSegmentationName));
+ } catch (LensException exc) {
+ checkTableNotFound(exc, cubeSegmentationName);
+ throw exc;
+ }
+ }
+
+ /**
* Create a new fact tabble
*
* @param sessionid The sessionid in which user is working
@@ -864,6 +953,30 @@ public class MetastoreResource {
}
/**
+ * Create a new cube segmentation
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param seg The {@link XCubeSegmentation} representation of the cube segmentation
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with
+ * state {@link Status#FAILED}, if create has failed
+ */
+ @POST
+ @Path("/cubesegmentations")
+ public APIResult createCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid, XCubeSegmentation seg)
+ throws LensException {
+ checkSessionId(sessionid);
+ try {
+ log.info("Create cube segmentation");
+ getSvc().createCubeSegmentation(sessionid, seg);
+ } catch (LensException exc) {
+ log.error("Exception creating cube segmentation:", exc);
+ return failure(processLensException(exc));
+ }
+ return success();
+ }
+
+
+ /**
* Update fact table definition
*
* @param sessionid The sessionid in which user is working
@@ -889,6 +1002,31 @@ public class MetastoreResource {
}
/**
+ * Update cube segmentation
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param cubeSegmentationName name of cube segmentation
+ * @param seg The {@link XCubeSegmentation} representation of the updated fact table definition
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with
+ * state {@link Status#FAILED}, if update has failed
+ */
+ @PUT
+ @Path("/cubesegmentations/{cubeSegmentationName}")
+ public APIResult updateCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("cubeSegmentationName") String cubeSegmentationName, XCubeSegmentation seg)
+ throws LensException {
+ checkSessionId(sessionid);
+ try {
+ getSvc().updateCubeSegmentation(sessionid, seg);
+ } catch (LensException exc) {
+ checkTableNotFound(exc, cubeSegmentationName);
+ log.error("Error updating segmentation {}", cubeSegmentationName, exc);
+ return failure(processLensException(exc));
+ }
+ return success();
+ }
+
+ /**
* Drop the fact table, specified by name
*
* @param sessionid The sessionid in which user is working
@@ -906,6 +1044,23 @@ public class MetastoreResource {
return Entity.FACT.delete(sessionid, factName, cascade);
}
+
+ /**
+ * Drop the cube segmentation, specified by name
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param cubeSegmentationName The cube segmentation name
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state
+ * {@link Status#FAILED}, if drop has failed
+ */
+ @DELETE
+ @Path("/cubesegmentations/{cubeSegmentationName}")
+ public APIResult dropCubeSegmentation(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("cubeSegmentationName") String cubeSegmentationName)
+ throws LensException {
+ return Entity.SEGMENTATION.delete(sessionid, cubeSegmentationName, null);
+ }
+
/**
* Get all storages of the fact table in the metastore
*