You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/03/27 09:08:26 UTC
[7/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup
using xml files instead of code
LENS-1400: Convert CubeTestSetup to setup using xml files instead of code
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/112af59c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/112af59c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/112af59c
Branch: refs/heads/lens-1381
Commit: 112af59cc4b82402a3df2c7b6687d2ee328e96b4
Parents: de464fa
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Mar 27 14:36:58 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Mar 27 14:36:58 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/api/ToXMLString.java | 9 +
.../apache/lens/api/jaxb/LensJAXBContext.java | 14 +-
.../lens/api/metastore/SchemaTraverser.java | 58 +
lens-api/src/main/resources/cube-0.1.xsd | 4 +-
.../lens/cli/commands/LensSchemaCommands.java | 111 +-
.../lens/cube/metadata/CubeFactTable.java | 13 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 195 +-
.../apache/lens/cube/metadata/JAXBUtils.java | 1114 ++++++++
.../lens/cube/parse/StorageCandidate.java | 8 +-
.../lens/cube/parse/StorageTableResolver.java | 2 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 2630 +-----------------
.../resources/schema/cubes/base/basecube.xml | 952 +++++++
.../resources/schema/cubes/base/testcube.xml | 640 +++++
.../resources/schema/cubes/derived/der1.xml | 22 +
.../resources/schema/cubes/derived/der2.xml | 43 +
.../resources/schema/cubes/derived/der3.xml | 32 +
.../schema/cubes/derived/derivedcube.xml | 24 +
.../cubes/derived/union_join_ctx_der1.xml | 25 +
.../resources/schema/dimensions/citydim.xml | 102 +
.../resources/schema/dimensions/countrydim.xml | 21 +
.../resources/schema/dimensions/cycledim1.xml | 51 +
.../resources/schema/dimensions/cycledim2.xml | 50 +
.../test/resources/schema/dimensions/daydim.xml | 41 +
.../resources/schema/dimensions/hourdim.xml | 25 +
.../test/resources/schema/dimensions/sports.xml | 25 +
.../resources/schema/dimensions/statedim.xml | 53 +
.../resources/schema/dimensions/testdim2.xml | 203 ++
.../resources/schema/dimensions/testdim3.xml | 48 +
.../resources/schema/dimensions/testdim4.xml | 25 +
.../schema/dimensions/unreachabledim.xml | 24 +
.../schema/dimensions/user_interests.xml | 30 +
.../resources/schema/dimensions/userdim.xml | 58 +
.../test/resources/schema/dimensions/zipdim.xml | 30 +
.../resources/schema/dimtables/citytable.xml | 51 +
.../resources/schema/dimtables/citytable2.xml | 29 +
.../resources/schema/dimtables/citytable3.xml | 29 +
.../resources/schema/dimtables/citytable4.xml | 28 +
.../resources/schema/dimtables/countrytable.xml | 32 +
.../dimtables/countrytable_partitioned.xml | 35 +
.../resources/schema/dimtables/cycledim1tbl.xml | 49 +
.../resources/schema/dimtables/cycledim2tbl.xml | 49 +
.../resources/schema/dimtables/daydimtbl.xml | 48 +
.../resources/schema/dimtables/hourdimtbl.xml | 48 +
.../resources/schema/dimtables/sports_tbl.xml | 48 +
.../resources/schema/dimtables/statetable.xml | 36 +
.../schema/dimtables/statetable_partitioned.xml | 37 +
.../resources/schema/dimtables/testdim2tbl.xml | 50 +
.../resources/schema/dimtables/testdim2tbl2.xml | 69 +
.../resources/schema/dimtables/testdim2tbl3.xml | 69 +
.../resources/schema/dimtables/testdim3tbl.xml | 49 +
.../resources/schema/dimtables/testdim4tbl.xml | 48 +
.../schema/dimtables/unreachabledimtable.xml | 35 +
.../schema/dimtables/user_interests_tbl.xml | 50 +
.../resources/schema/dimtables/usertable.xml | 52 +
.../resources/schema/dimtables/ziptable.xml | 35 +
.../test/resources/schema/facts/cheapfact.xml | 81 +
.../test/resources/schema/facts/summary1.xml | 81 +
.../test/resources/schema/facts/summary2.xml | 83 +
.../test/resources/schema/facts/summary3.xml | 84 +
.../test/resources/schema/facts/summary4.xml | 65 +
.../test/resources/schema/facts/testfact.xml | 192 ++
.../resources/schema/facts/testfact1_base.xml | 135 +
.../schema/facts/testfact1_raw_base.xml | 75 +
.../test/resources/schema/facts/testfact2.xml | 75 +
.../resources/schema/facts/testfact2_base.xml | 120 +
.../resources/schema/facts/testfact2_raw.xml | 75 +
.../schema/facts/testfact2_raw_base.xml | 46 +
.../resources/schema/facts/testfact3_base.xml | 117 +
.../schema/facts/testfact3_raw_base.xml | 44 +
.../schema/facts/testfact4_raw_base.xml | 45 +
.../resources/schema/facts/testfact5_base.xml | 128 +
.../schema/facts/testfact5_raw_base.xml | 38 +
.../resources/schema/facts/testfact6_base.xml | 117 +
.../schema/facts/testfact_continuous.xml | 38 +
.../schema/facts/testfact_deprecated.xml | 126 +
.../resources/schema/facts/testfactmonthly.xml | 46 +
.../schema/facts/union_join_ctx_fact1.xml | 39 +
.../schema/facts/union_join_ctx_fact2.xml | 38 +
.../schema/facts/union_join_ctx_fact3.xml | 38 +
.../schema/facts/union_join_ctx_fact5.xml | 38 +
.../schema/facts/union_join_ctx_fact6.xml | 38 +
.../resources/schema/segmentations/seg1.xml | 45 +
.../src/test/resources/schema/storages/c0.xml | 6 +
.../src/test/resources/schema/storages/c1.xml | 6 +
.../src/test/resources/schema/storages/c2.xml | 6 +
.../src/test/resources/schema/storages/c3.xml | 6 +
.../src/test/resources/schema/storages/c4.xml | 6 +
.../src/test/resources/schema/storages/c5.xml | 6 +
.../src/test/resources/schema/storages/c99.xml | 6 +
.../metastore/CubeMetastoreServiceImpl.java | 139 +-
.../apache/lens/server/metastore/JAXBUtils.java | 1116 --------
91 files changed, 7176 insertions(+), 3826 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
index e74adc9..746a82b 100644
--- a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
+++ b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
@@ -24,6 +24,10 @@ import java.util.HashMap;
import java.util.Map;
import javax.xml.bind.*;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.namespace.QName;
import org.apache.lens.api.jaxb.LensJAXBContext;
@@ -31,6 +35,11 @@ public abstract class ToXMLString {
protected static final Map<Class<?>, JAXBContext> JAXB_CONTEXTS = new HashMap<>();
public static String toString(Object o) {
+ if (!(o instanceof JAXBElement) && o.getClass().getAnnotation(XmlRootElement.class) == null
+ && o.getClass().getAnnotation(XmlType.class)!= null) {
+ o = new JAXBElement(new QName("uri:lens:cube:0.1", o.getClass().getAnnotation(XmlType.class).name()),
+ o.getClass(), null, o);
+ }
try {
StringWriter stringWriter = new StringWriter();
Class cl = null;
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
index 14fc4aa..8858b95 100644
--- a/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
+++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java
@@ -24,6 +24,7 @@ package org.apache.lens.api.jaxb;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.Reader;
import javax.xml.XMLConstants;
import javax.xml.bind.*;
@@ -114,17 +115,26 @@ public class LensJAXBContext extends JAXBContext {
return UNMARSHALLER;
}
+ public static <T> T unmarshall(File file) throws JAXBException, IOException {
+ return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue();
+ }
+ public static <T> T unmarshall(InputStream inputStream) throws JAXBException, IOException {
+ return ((JAXBElement<T>) UNMARSHALLER.unmarshal(inputStream)).getValue();
+ }
+ public static <T> T unmarshall(Reader reader) throws JAXBException, IOException {
+ return ((JAXBElement<T>) UNMARSHALLER.unmarshal(reader)).getValue();
+ }
public static <T> T unmarshallFromFile(String filename) throws JAXBException, IOException {
File file = new File(filename);
if (file.exists()) {
- return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue();
+ return unmarshall(file);
} else {
// load from classpath
InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filename);
if (stream == null) {
throw new IOException("File not found:" + filename);
}
- return ((JAXBElement<T>) UNMARSHALLER.unmarshal(stream)).getValue();
+ return unmarshall(stream);
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
new file mode 100644
index 0000000..157ad71
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
@@ -0,0 +1,58 @@
+package org.apache.lens.api.metastore;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.jaxb.LensJAXBContext;
+
+import com.google.common.collect.Maps;
+
+/*
+ * Created on 07/03/17.
+ */
+public class SchemaTraverser implements Runnable {
+ final File parent;
+ final Map<String, Class<?>> types = Maps.newLinkedHashMap();
+ private final SchemaEntityProcessor action;
+ {
+ types.put("storages", XStorage.class);
+ types.put("cubes/base", XBaseCube.class);
+ types.put("cubes/derived", XDerivedCube.class);
+ types.put("dimensions", XDimension.class);
+ types.put("facts", XFactTable.class);
+ types.put("dimtables", XDimensionTable.class);
+ types.put("dimensiontables", XDimensionTable.class);
+ types.put("dimensiontables", XDimensionTable.class);
+ types.put("segmentations", XSegmentation.class);
+ }
+ private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml");
+
+ public interface SchemaEntityProcessor extends BiConsumer<File, Class<?>> {
+ }
+
+ public SchemaTraverser(File parent, SchemaEntityProcessor action) {
+ this.parent = parent;
+ this.action = action;
+ }
+
+ @Override
+ public void run() {
+ for (Map.Entry<String, Class<?>> entry : types.entrySet()) {
+ File f = new File(parent, entry.getKey());
+ if (f.exists()) {
+ assert f.isDirectory();
+ File[] files = f.listFiles(XML_FILTER);
+ if (files != null) {
+ for (File entityFile : files) {
+ action.accept(entityFile.getAbsoluteFile(), entry.getValue());
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 060eb43..1d8a624 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -390,7 +390,7 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="1"/>
+ <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
@@ -707,7 +707,7 @@
<xs:complexType name="x_columns">
<xs:sequence>
- <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="1"/>
+ <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index feabf9c..befe4e6 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -18,11 +18,21 @@
*/
package org.apache.lens.cli.commands;
-import java.io.*;
+import java.io.File;
+import java.io.FilenameFilter;
import java.util.List;
+import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.lens.api.metastore.SchemaTraverser;
+import org.apache.lens.api.metastore.XBaseCube;
+import org.apache.lens.api.metastore.XDerivedCube;
+import org.apache.lens.api.metastore.XDimension;
+import org.apache.lens.api.metastore.XDimensionTable;
+import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.metastore.XSegmentation;
+import org.apache.lens.api.metastore.XStorage;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
import org.springframework.beans.factory.annotation.Autowired;
@@ -35,6 +45,7 @@ import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
@Component
@UserDocumentation(title = "Creating schema with one command",
@@ -84,15 +95,52 @@ public class LensSchemaCommands implements CommandMarker {
logger.setLevel(Level.FINE);
}
- private static final FilenameFilter XML_FILTER = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(".xml");
- }
- };
+ private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml");
+ private static final Map<Class<?>, String> CREATE_COMMAND_MAP = Maps.newHashMap();
+ private static final Map<Class<?>, String> UPDATE_COMMAND_MAP = Maps.newHashMap();
+
@Autowired
private JLineShellComponent shell;
+ static {
+ CREATE_COMMAND_MAP.put(XStorage.class, "create storage --path %s");
+ UPDATE_COMMAND_MAP.put(XStorage.class, "update storage --name %s --path %s");
+ CREATE_COMMAND_MAP.put(XDimension.class, "create dimension --path %s");
+ UPDATE_COMMAND_MAP.put(XDimension.class, "update dimension --name %s --path %s");
+ CREATE_COMMAND_MAP.put(XBaseCube.class, "create cube --path %s");
+ UPDATE_COMMAND_MAP.put(XBaseCube.class, "update cube --name %s --path %s");
+ CREATE_COMMAND_MAP.put(XDerivedCube.class, "create cube --path %s");
+ UPDATE_COMMAND_MAP.put(XDerivedCube.class, "update cube --name %s --path %s");
+ CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s");
+ UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s");
+ CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s");
+ UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s");
+ CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s");
+ UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s");
+ CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s");
+ UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s");
+ }
+
+ private final SchemaTraverser.SchemaEntityProcessor processor = (entityFile, type) -> {
+ String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4);
+ String entityPath = entityFile.getAbsolutePath();
+ String createCommand = String.format(CREATE_COMMAND_MAP.get(type), entityPath);
+ String entityType = createCommand.substring(8, createCommand.indexOf(" ", 9));
+ logger.fine(createCommand);
+ if (shell.executeScriptLine(createCommand)) {
+ logger.info("Created " + entityType + " " + entityName);
+ } else {
+ logger.warning("Create failed, trying update");
+ String updateCommand = String.format(UPDATE_COMMAND_MAP.get(type), entityName, entityPath);
+ logger.fine(updateCommand);
+ if (shell.executeScriptLine(updateCommand)) {
+ logger.info("Updated " + entityType + " " + entityName);
+ } else {
+ logger.severe("Couldn't create or update " + entityType + " " + entityName);
+ }
+ }
+ };
+
@CliCommand(value = {"schema", "create schema"},
help = "Parses the specified resource file and executes commands for "
+ "creation/updation of schema\nExpected structure is " + STRUCTURE)
@@ -108,55 +156,10 @@ public class LensSchemaCommands implements CommandMarker {
// ignore result. it can fail if database already exists
shell.executeCommand("create database " + database);
if (shell.executeScriptLine("use " + database)) {
- createOrUpdate(new File(schemaDirectory, "storages"), "storage",
- "create storage --path %s", "update storage --name %s --path %s");
- createOrUpdate(new File(schemaDirectory, "dimensions"), "dimension",
- "create dimension --path %s", "update dimension --name %s --path %s");
- createOrUpdate(new File(new File(schemaDirectory, "cubes"), "base"), "base cube",
- "create cube --path %s", "update cube --name %s --path %s");
- createOrUpdate(new File(new File(schemaDirectory, "cubes"), "derived"), "derived cube",
- "create cube --path %s", "update cube --name %s --path %s");
- createOrUpdate(new File(schemaDirectory, "dimensiontables"), "dimension table",
- "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s");
- createOrUpdate(new File(schemaDirectory, "dimtables"), "dimension table",
- "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s");
- createOrUpdate(new File(schemaDirectory, "facts"), "fact",
- "create fact --path %s", "update fact --fact_name %s --path %s");
- createOrUpdate(new File(schemaDirectory, "segmentations"), "fact",
- "create segmentation --path %s", "update segmentation --name %s --path %s");
+ SchemaTraverser schemaTraverser = new SchemaTraverser(schemaDirectory, processor);
+ schemaTraverser.run();
} else {
throw new IllegalStateException("Switching to database " + database + " failed");
}
}
-
- public List<File> createOrUpdate(File parent, String entityType, String createSyntax, String updateSyntax) {
- List<File> failedFiles = Lists.newArrayList();
- // Create/update entities
- if (parent.exists()) {
- Assert.isTrue(parent.isDirectory(), parent.toString() + " must be a directory");
- for (File entityFile : parent.listFiles(XML_FILTER)) {
- String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4);
- String entityPath = entityFile.getAbsolutePath();
- String createCommand = String.format(createSyntax, entityPath);
- logger.fine(createCommand);
- if (shell.executeScriptLine(createCommand)) {
- logger.info("Created " + entityType + " " + entityName);
- } else {
- logger.warning("Create failed, trying update");
- String updateCommand = String.format(updateSyntax, entityName, entityPath);
- logger.fine(updateCommand);
- if (shell.executeScriptLine(updateCommand)) {
- logger.info("Updated " + entityType + " " + entityName);
- } else {
- logger.severe("Couldn't create or update " + entityType + " " + entityName);
- failedFiles.add(entityFile);
- }
- }
- }
- }
- if (!failedFiles.isEmpty()) {
- logger.severe("Failed for " + entityType + ": " + failedFiles);
- }
- return failedFiles;
- }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index 896a7a1..e00122d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -111,21 +111,16 @@ public class CubeFactTable extends AbstractCubeTable {
private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) {
Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>();
- for (Map.Entry entry : storageUpdatePeriods.entrySet()) {
- String storage = (String) entry.getKey();
- for (UpdatePeriod period : (Set<UpdatePeriod>) entry.getValue()) {
+ for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) {
+ String storage = entry.getKey();
+ for (UpdatePeriod period : entry.getValue()) {
String storagePrefixKey = MetastoreUtil
.getUpdatePeriodStoragePrefixKey(factName.trim(), storage, period.getName());
String storageTableNamePrefix = props.get(storagePrefixKey);
if (storageTableNamePrefix == null) {
storageTableNamePrefix = storage;
}
- Map<UpdatePeriod, String> mapOfUpdatePeriods = ret.get(storage);
- if (mapOfUpdatePeriods == null) {
- mapOfUpdatePeriods = new HashMap<>();
- ret.put(storage, mapOfUpdatePeriods);
- }
- mapOfUpdatePeriods.put(period, storageTableNamePrefix);
+ ret.computeIfAbsent(storage, k -> new HashMap<>()).put(period, storageTableNamePrefix);
}
}
return ret;
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 78fb6d3..c8a2498 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -20,12 +20,25 @@
package org.apache.lens.cube.metadata;
import static org.apache.lens.cube.metadata.DateUtil.resolveDate;
+import static org.apache.lens.cube.metadata.JAXBUtils.getStorageTableDescFromHiveTable;
+import static org.apache.lens.cube.metadata.JAXBUtils.segmentationFromXSegmentation;
import static org.apache.lens.cube.metadata.MetastoreUtil.*;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.lens.api.metastore.XCube;
+import org.apache.lens.api.metastore.XDerivedCube;
+import org.apache.lens.api.metastore.XDimension;
+import org.apache.lens.api.metastore.XDimensionTable;
+import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.metastore.XSegmentation;
+import org.apache.lens.api.metastore.XStorage;
+import org.apache.lens.api.metastore.XStorageTableElement;
+import org.apache.lens.api.metastore.XUpdatePeriod;
+import org.apache.lens.api.metastore.XUpdatePeriodTableDescriptor;
+import org.apache.lens.api.metastore.XUpdatePeriods;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.Storage.LatestInfo;
import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
@@ -50,6 +63,10 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.thrift.TException;
+import org.jvnet.jaxb2_commons.lang.Equals;
+import org.jvnet.jaxb2_commons.lang.HashCode;
+import org.jvnet.jaxb2_commons.lang.ToString;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -290,6 +307,61 @@ public class CubeMetastoreClient {
}
}
+
+ public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException {
+ if (entity instanceof XStorage) {
+ createStorage((XStorage) entity);
+ } else if (entity instanceof XCube) {
+ createCube((XCube)entity);
+ } else if (entity instanceof XDimension) {
+ createDimension((XDimension) entity);
+ } else if (entity instanceof XFactTable) {
+ createCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XDimensionTable) {
+ createCubeDimensionTable((XDimensionTable) entity);
+ } else if (entity instanceof XSegmentation) {
+ createSegmentation((XSegmentation) entity);
+ } else {
+ throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass());
+ }
+ }
+
+ public <T extends Equals & HashCode & ToString> void updateEntity(String name, T entity)
+ throws LensException, HiveException {
+ if (entity instanceof XStorage) {
+ alterStorage((XStorage) entity);
+ } else if (entity instanceof XCube) {
+ alterCube((XCube)entity);
+ } else if (entity instanceof XDimension) {
+ alterDimension((XDimension) entity);
+ } else if (entity instanceof XFactTable) {
+ alterCubeFactTable((XFactTable) entity);
+ } else if (entity instanceof XDimensionTable) {
+ alterCubeDimensionTable((XDimensionTable) entity);
+ } else if (entity instanceof XSegmentation) {
+ alterSegmentation((XSegmentation) entity);
+ } else {
+ throw new LensException("Unable to alter entity " + entity + " as it's unrecognizable: " + entity.getClass());
+ }
+ }
+
+
+ public static Map<String, String> addFactColStartTimePropertyToFactProperties(XFactTable fact) {
+ Map<String, String> props = new HashMap<String, String>();
+ props.putAll(JAXBUtils.mapFromXProperties(fact.getProperties()));
+ props.putAll(JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns()));
+ return props;
+ }
+ public void createCubeFactTable(XFactTable fact) throws LensException {
+ createCubeFactTable(fact.getCubeName(),
+ fact.getName(),
+ JAXBUtils.fieldSchemaListFromColumns(fact.getColumns()),
+ JAXBUtils.getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()),
+ fact.getWeight(),
+ addFactColStartTimePropertyToFactProperties(fact),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
+ JAXBUtils.storageTablePrefixMapOfStorage(fact.getStorageTables()));
+ }
public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
@@ -302,6 +374,7 @@ public class CubeMetastoreClient {
}
+
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
* storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
@@ -619,12 +692,22 @@ public class CubeMetastoreClient {
}
}
+ public void createStorage(XStorage storage) throws LensException {
+ createStorage(JAXBUtils.storageFromXStorage(storage));
+ }
+
public void createStorage(Storage storage) throws LensException {
createCubeHiveTable(storage);
// do a get to update cache
getStorage(storage.getName());
}
+ public void createCube(XCube cube) throws LensException {
+ Cube parent = cube instanceof XDerivedCube ? (Cube) getCube(
+ ((XDerivedCube) cube).getParent()) : null;
+ createCube(JAXBUtils.hiveCubeFromXCube(cube, parent));
+ }
+
/**
* Create cube in metastore defined by {@link Cube} or {@link DerivedCube} object
*
@@ -714,6 +797,9 @@ public class CubeMetastoreClient {
createDimension(dim);
}
+ public void createDimension(XDimension dim) throws LensException {
+ createDimension(JAXBUtils.dimensionFromXDimension(dim));
+ }
/**
* Create dimension in metastore defined by {@link Dimension} object
*
@@ -783,6 +869,18 @@ public class CubeMetastoreClient {
getSegmentation(segmentationName);
}
+ public void createCubeDimensionTable(XDimensionTable xDimTable) throws LensException {
+ List<FieldSchema> columns = JAXBUtils.fieldSchemaListFromColumns(xDimTable.getColumns());
+ Map<String, UpdatePeriod> updatePeriodMap =
+ JAXBUtils.dumpPeriodsFromStorageTables(xDimTable.getStorageTables());
+
+ Map<String, String> properties = JAXBUtils.mapFromXProperties(xDimTable.getProperties());
+ Map<String, StorageTableDesc> storageDesc = JAXBUtils.tableDescPrefixMapFromXStorageTables(
+ xDimTable.getStorageTables());
+ log.info("# Columns: " + columns);
+ createCubeDimensionTable(xDimTable.getDimensionName(), xDimTable.getTableName(), columns, xDimTable.getWeight(),
+ updatePeriodMap, properties, storageDesc);
+ }
/**
* Create a cube dimension table
*
@@ -846,6 +944,14 @@ public class CubeMetastoreClient {
}
}
+ public void createSegmentation(XSegmentation cubeSeg) throws LensException {
+ createSegmentation(
+ cubeSeg.getCubeName(),
+ cubeSeg.getName(),
+ JAXBUtils.segmentsFromXSegments(cubeSeg.getSegements()),
+ cubeSeg.getWeight(),
+ JAXBUtils.mapFromXProperties(cubeSeg.getProperties()));
+ }
public void createSegmentation(Segmentation cubeSeg)
throws LensException {
// create virtual cube table in metastore
@@ -1618,6 +1724,47 @@ public class CubeMetastoreClient {
return CubeTableType.DIMENSION.name().equals(tableType);
}
+ public XFactTable getXFactTable(String tableName) throws LensException {
+ return getXFactTable(getFactTable(tableName));
+ }
+ public XFactTable getXFactTable(CubeFactTable cft) throws LensException {
+
+ XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft);
+ Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap();
+ for (String storageName : cft.getStorages()) {
+ Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName);
+ // This map tells if there are different tables for different update period.
+ Map<UpdatePeriod, String> updatePeriodToTableMap = storageMap.get(storageName);
+ Set<String> tableNames = new HashSet<>();
+ for (UpdatePeriod updatePeriod : updatePeriods) {
+ tableNames.add(updatePeriodToTableMap.get(updatePeriod));
+ }
+ if (tableNames.size() <= 1) {
+ XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
+ getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName)));
+ tblElement.setStorageName(storageName);
+ for (UpdatePeriod p : updatePeriods) {
+ tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
+ }
+ factTable.getStorageTables().getStorageTable().add(tblElement);
+ } else {
+ // Multiple storage tables.
+ XStorageTableElement tblElement = new XStorageTableElement();
+ tblElement.setStorageName(storageName);
+ XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
+ tblElement.setUpdatePeriods(xUpdatePeriods);
+ for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
+ XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
+ updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
+ this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue()))));
+ updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod)entry.getKey()).name()));
+ xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
+ }
+ factTable.getStorageTables().getStorageTable().add(tblElement);
+ }
+ }
+ return factTable;
+ }
/**
* Get {@link CubeFactTable} object corresponding to the name
*
@@ -1634,6 +1781,25 @@ public class CubeMetastoreClient {
return new Segmentation(getTableWithTypeFailFast(tableName, CubeTableType.SEGMENTATION));
}
+ public XDimensionTable getXDimensionTable(String dimTable) throws LensException {
+ return getXDimensionTable(getDimensionTable(dimTable));
+ }
+ public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException {
+ XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable);
+ if (!dimTable.getStorages().isEmpty()) {
+ for (String storageName : dimTable.getStorages()) {
+ XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
+ this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName)));
+ tblElement.setStorageName(storageName);
+ UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName);
+ if (p != null) {
+ tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
+ }
+ dt.getStorageTables().getStorageTable().add(tblElement);
+ }
+ }
+ return dt;
+ }
/**
* Get {@link CubeDimensionTable} object corresponding to the name
*
@@ -2144,6 +2310,11 @@ public class CubeMetastoreClient {
}
}
+ public void alterCube(XCube cube) throws HiveException, LensException {
+ Cube parent = cube instanceof XDerivedCube ? (Cube) getCube(
+ ((XDerivedCube) cube).getParent()) : null;
+ alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent));
+ }
/**
* Alter cube specified by the name to new definition
*
@@ -2162,10 +2333,13 @@ public class CubeMetastoreClient {
/**
* Alter dimension specified by the dimension name to new definition
*
- * @param dimName The cube name to be altered
* @param newDim The new dimension definition
* @throws HiveException
*/
+ public void alterDimension(XDimension newDim) throws HiveException, LensException {
+ alterDimension(newDim.getName(), JAXBUtils.dimensionFromXDimension(newDim));
+ }
+
public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException {
Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION);
alterCubeTable(dimName, tbl, newDim);
@@ -2177,10 +2351,12 @@ public class CubeMetastoreClient {
/**
* Alter storage specified by the name to new definition
*
- * @param storageName The storage name to be altered
* @param storage The new storage definition
* @throws LensException
*/
+ public void alterStorage(XStorage storage) throws LensException, HiveException {
+ alterStorage(storage.getName(), JAXBUtils.storageFromXStorage(storage));
+ }
public void alterStorage(String storageName, Storage storage) throws LensException, HiveException {
Table storageTbl = getTableWithTypeFailFast(storageName, CubeTableType.STORAGE);
alterCubeTable(storageName, storageTbl, storage);
@@ -2333,7 +2509,11 @@ public class CubeMetastoreClient {
dropHiveTable(dimTblName);
allDimTables.remove(dimTblName.trim().toLowerCase());
}
-
+ public void alterCubeFactTable(XFactTable fact) throws LensException, HiveException {
+ alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(fact.getStorageTables()),
+ JAXBUtils.columnStartAndEndTimeFromXColumns(fact.getColumns()));
+ }
/**
* Alter a cubefact with new definition and alter underlying storage tables as well.
*
@@ -2361,6 +2541,9 @@ public class CubeMetastoreClient {
updateFactCache(factTableName);
}
+ public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException {
+ alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg));
+ }
public void alterSegmentation(String segName, Segmentation seg)
throws HiveException, LensException {
getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION);
@@ -2388,7 +2571,11 @@ public class CubeMetastoreClient {
allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName)));
}
}
-
+ public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException {
+ alterCubeDimensionTable(dimensionTable.getTableName(),
+ JAXBUtils.cubeDimTableFromDimTable(dimensionTable),
+ JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables()));
+ }
/**
* Alter dimension table with new dimension definition and underlying storage tables as well
*
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
new file mode 100644
index 0000000..e1e3d16
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
@@ -0,0 +1,1114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.lang.reflect.Constructor;
+import java.text.ParseException;
+import java.util.*;
+
+import javax.ws.rs.WebApplicationException;
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+
+import org.apache.lens.api.metastore.*;
+import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.InputFormat;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Maps;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Utilities for converting to and from JAXB types to hive.ql.metadata.cube types
+ */
+@Slf4j
+public final class JAXBUtils {
+ private JAXBUtils() {
+
+ }
+
+ private static final ObjectFactory XCF = new ObjectFactory();
+
+ /**
+ * Create a hive ql cube object from corresponding JAXB object
+ *
+ * @param cube JAXB Cube
+ * @return {@link Cube}
+ * @throws LensException
+ */
+ public static CubeInterface hiveCubeFromXCube(XCube cube, Cube parent) throws LensException {
+ if (cube instanceof XDerivedCube) {
+ XDerivedCube dcube = (XDerivedCube) cube;
+ Set<String> dims = new LinkedHashSet<String>();
+ dims.addAll(dcube.getDimAttrNames().getAttrName());
+
+ Set<String> measures = new LinkedHashSet<String>();
+ measures.addAll(dcube.getMeasureNames().getMeasureName());
+
+ Map<String, String> properties = mapFromXProperties(cube.getProperties());
+ return new DerivedCube(cube.getName(), measures, dims, properties, 0L, parent);
+ } else {
+ XBaseCube bcube = (XBaseCube) cube;
+ Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>();
+ if (bcube.getDimAttributes() != null && !bcube.getDimAttributes().getDimAttribute().isEmpty()) {
+ for (XDimAttribute xd : bcube.getDimAttributes().getDimAttribute()) {
+ dims.add(hiveDimAttrFromXDimAttr(xd));
+ }
+ }
+
+ Set<CubeMeasure> measures = new LinkedHashSet<CubeMeasure>();
+ for (XMeasure xm : bcube.getMeasures().getMeasure()) {
+ measures.add(hiveMeasureFromXMeasure(xm));
+ }
+
+ Set<ExprColumn> expressions = new LinkedHashSet<ExprColumn>();
+ if (bcube.getExpressions() != null && !bcube.getExpressions().getExpression().isEmpty()) {
+ for (XExprColumn xe : bcube.getExpressions().getExpression()) {
+ expressions.add(hiveExprColumnFromXExprColumn(xe));
+ }
+ }
+
+ Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>();
+ if (bcube.getJoinChains() != null && !bcube.getJoinChains().getJoinChain().isEmpty()) {
+ for (XJoinChain xj : bcube.getJoinChains().getJoinChain()) {
+ joinchains.add(joinChainFromXJoinChain(xj));
+ }
+ }
+
+ Map<String, String> properties = mapFromXProperties(cube.getProperties());
+ return new Cube(cube.getName(), measures, dims, expressions, joinchains, properties, 0L);
+ }
+ }
+
+ /**
+ * Get XCube from hive.ql.metadata.Cube
+ *
+ * @param c
+ * @return {@link XCube}
+ */
+ public static XCube xCubeFromHiveCube(CubeInterface c) {
+ XCube xc;
+ if (c.isDerivedCube()) {
+ XDerivedCube xdc = XCF.createXDerivedCube();
+ xdc.setMeasureNames(new XMeasureNames());
+ xdc.setDimAttrNames(new XDimAttrNames());
+ xc = xdc;
+ xdc.getMeasureNames().getMeasureName().addAll(c.getMeasureNames());
+ xdc.getDimAttrNames().getAttrName().addAll(c.getDimAttributeNames());
+ xdc.setParent(((DerivedCube) c).getParent().getName());
+ } else {
+ XBaseCube xbc = XCF.createXBaseCube();
+ xbc.setMeasures(new XMeasures());
+ xbc.setDimAttributes(new XDimAttributes());
+ xbc.setExpressions(new XExpressions());
+ xbc.setJoinChains(new XJoinChains());
+ xc = xbc;
+ for (CubeMeasure cm : c.getMeasures()) {
+ xbc.getMeasures().getMeasure().add(xMeasureFromHiveMeasure(cm));
+ }
+
+ for (ExprColumn ec : c.getExpressions()) {
+ xbc.getExpressions().getExpression().add(xExprColumnFromHiveExprColumn(ec));
+ }
+ for (CubeDimAttribute cd : c.getDimAttributes()) {
+ xbc.getDimAttributes().getDimAttribute().add(xDimAttrFromHiveDimAttr(cd, (Cube) c));
+ }
+ for (JoinChain jc : c.getJoinChains()) {
+ xbc.getJoinChains().getJoinChain().add(getXJoinChainFromJoinChain(jc));
+ }
+ }
+ xc.setName(c.getName());
+ xc.setProperties(new XProperties());
+ xc.getProperties().getProperty().addAll(xPropertiesFromMap(((AbstractCubeTable) c).getProperties()));
+ return xc;
+ }
+
+ /**
+ * Create a hive ql CubeDimension from JAXB counterpart
+ *
+ * @param xd
+ * @return {@link org.apache.lens.cube.metadata.CubeDimAttribute}
+ */
+ public static CubeDimAttribute hiveDimAttrFromXDimAttr(XDimAttribute xd) throws LensException {
+ Date startDate = getDateFromXML(xd.getStartTime());
+ Date endDate = getDateFromXML(xd.getEndTime());
+
+ CubeDimAttribute hiveDim;
+
+ if (xd.getHierarchy() != null) {
+ List<CubeDimAttribute> hierarchy = new ArrayList<>();
+ for (XDimAttribute hd : xd.getHierarchy().getDimAttribute()) {
+ hierarchy.add(hiveDimAttrFromXDimAttr(hd));
+ }
+ hiveDim = new HierarchicalDimAttribute(xd.getName(), xd.getDescription(), hierarchy);
+ } else if (xd.getChainRefColumn() != null
+ && !xd.getChainRefColumn().isEmpty()) {
+ hiveDim = new ReferencedDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
+ xd.getDescription()),
+ xd.getDisplayString(),
+ getChainRefColumns(xd.getChainRefColumn()),
+ startDate,
+ endDate,
+ null,
+ xd.getNumDistinctValues(),
+ xd.getValues(),
+ mapFromXProperties(xd.getTags())
+ );
+ } else {
+ hiveDim = new BaseDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(),
+ xd.getDescription()),
+ xd.getDisplayString(),
+ startDate,
+ endDate,
+ null,
+ xd.getNumDistinctValues(),
+ xd.getValues(),
+ mapFromXProperties(xd.getTags())
+ );
+ }
+ return hiveDim;
+ }
+
+ private static List<ChainRefCol> getChainRefColumns(List<XChainColumn> chainCols) {
+ List<ChainRefCol> chainRefCols = new ArrayList<>();
+ for (XChainColumn chainCol : chainCols) {
+ chainRefCols.add(new ChainRefCol(chainCol.getChainName(), chainCol.getRefCol()));
+ }
+ return chainRefCols;
+ }
+
+ /**
+ * Get XMLGregorianCalendar from Date.
+ *
+ * Useful for converting from java code to XML spec.
+ *
+ * @param d Date value
+ * @return XML value
+ */
+ public static XMLGregorianCalendar getXMLGregorianCalendar(Date d) {
+ if (d == null) {
+ return null;
+ }
+
+ GregorianCalendar c1 = new GregorianCalendar();
+ c1.setTime(d);
+ try {
+ return DatatypeFactory.newInstance().newXMLGregorianCalendar(c1);
+ } catch (DatatypeConfigurationException e) {
+ log.warn("Error converting date " + d, e);
+ return null;
+ }
+ }
+
+ /**
+ * Get Date from XMLGregorianCalendar
+ *
+ * Useful for converting from XML spec to java code.
+ *
+ * @param cal XML value
+ * @return Date value
+ */
+ public static Date getDateFromXML(XMLGregorianCalendar cal) {
+ if (cal == null) {
+ return null;
+ }
+ return cal.toGregorianCalendar().getTime();
+ }
+
+ /**
+ * Create XMeasure from hive ql cube measure
+ */
+ public static XMeasure xMeasureFromHiveMeasure(CubeMeasure cm) {
+ if (cm == null) {
+ return null;
+ }
+
+ XMeasure xm = XCF.createXMeasure();
+ xm.setName(cm.getName());
+ xm.setDescription(cm.getDescription());
+ xm.setDisplayString(cm.getDisplayString());
+ xm.setDefaultAggr(cm.getAggregate());
+ xm.setFormatString(cm.getFormatString());
+ xm.setType(XMeasureType.valueOf(cm.getType().toUpperCase()));
+ xm.setUnit(cm.getUnit());
+ xm.setStartTime(getXMLGregorianCalendar(cm.getStartTime()));
+ xm.setEndTime(getXMLGregorianCalendar(cm.getEndTime()));
+ xm.setMin(cm.getMin());
+ xm.setMax(cm.getMax());
+ xm.setTags(getXProperties(xPropertiesFromMap(cm.getTags())));
+ return xm;
+ }
+
+ public static XProperties getXProperties(List<XProperty> prop) {
+ XProperties properties = XCF.createXProperties();
+ properties.getProperty().addAll(prop);
+ return properties;
+ }
+
+ /**
+ * Create XExprColumn from hive ExprColum
+ */
+ public static XExprColumn xExprColumnFromHiveExprColumn(ExprColumn ec) {
+ if (ec == null) {
+ return null;
+ }
+
+ XExprColumn xe = XCF.createXExprColumn();
+ xe.setName(ec.getName());
+ xe.setType(ec.getType());
+ xe.setDescription(ec.getDescription());
+ xe.setDisplayString(ec.getDisplayString());
+ xe.getExprSpec().addAll(xExprSpecFromExprColumn(ec.getExpressionSpecs()));
+ xe.setTags(getXProperties(xPropertiesFromMap(ec.getTags())));
+ return xe;
+ }
+
+ private static Collection<XExprSpec> xExprSpecFromExprColumn(Collection<ExprSpec> esSet) {
+ List<XExprSpec> xes = new ArrayList<XExprSpec>();
+ for (ExprSpec es : esSet) {
+ XExprSpec e = new XExprSpec();
+ e.setExpr(es.getExpr());
+ e.setStartTime(getXMLGregorianCalendar(es.getStartTime()));
+ e.setEndTime(getXMLGregorianCalendar(es.getEndTime()));
+ xes.add(e);
+ }
+ return xes;
+ }
+
+ private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) throws LensException {
+ List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size());
+ for (XExprSpec xes : xesList) {
+ esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime())));
+ }
+ return esArray.toArray(new ExprSpec[0]);
+ }
+
+ /**
+ * Create XDimAttribute from CubeDimAttribute
+ */
+ public static XDimAttribute xDimAttrFromHiveDimAttr(CubeDimAttribute cd, AbstractBaseTable baseTable) {
+ XDimAttribute xd = XCF.createXDimAttribute();
+ xd.setName(cd.getName());
+ xd.setDescription(cd.getDescription());
+ xd.setDisplayString(cd.getDisplayString());
+ xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime()));
+ xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime()));
+ xd.setTags(getXProperties(xPropertiesFromMap(cd.getTags())));
+ if (cd instanceof ReferencedDimAttribute) {
+ ReferencedDimAttribute rd = (ReferencedDimAttribute) cd;
+ if (!rd.getChainRefColumns().isEmpty()) {
+ for (ChainRefCol crCol : rd.getChainRefColumns()) {
+ XChainColumn xcc = new XChainColumn();
+ xcc.setChainName(crCol.getChainName());
+ xcc.setRefCol(crCol.getRefColumn());
+ if (baseTable.getChainByName(crCol.getChainName()) == null) {
+ log.error("Missing chain definition for " + crCol.getChainName());
+ } else {
+ xcc.setDestTable(baseTable.getChainByName(crCol.getChainName()).getDestTable());
+ }
+ xd.getChainRefColumn().add(xcc);
+ }
+ }
+ xd.setType(rd.getType());
+ Optional<Long> numOfDistinctValues = rd.getNumOfDistinctValues();
+ if (numOfDistinctValues.isPresent()) {
+ xd.setNumDistinctValues(numOfDistinctValues.get());
+ }
+ if (rd.getValues().isPresent()) {
+ xd.getValues().addAll(rd.getValues().get());
+ }
+ } else if (cd instanceof BaseDimAttribute) {
+ BaseDimAttribute bd = (BaseDimAttribute) cd;
+ xd.setType(bd.getType());
+ Optional<Long> numOfDistinctValues = bd.getNumOfDistinctValues();
+ if (numOfDistinctValues.isPresent()) {
+ xd.setNumDistinctValues(numOfDistinctValues.get());
+ }
+ if (bd.getValues().isPresent()) {
+ xd.getValues().addAll(bd.getValues().get());
+ }
+ } else if (cd instanceof HierarchicalDimAttribute) {
+ HierarchicalDimAttribute hd = (HierarchicalDimAttribute) cd;
+ XDimAttributes hierarchy = new XDimAttributes();
+ for (CubeDimAttribute hdDim : hd.getHierarchy()) {
+ hierarchy.getDimAttribute().add(xDimAttrFromHiveDimAttr(hdDim, baseTable));
+ }
+ xd.setHierarchy(hierarchy);
+ }
+ return xd;
+ }
+
+ /**
+ * Create XJoinChain from cube join chain
+ */
+ public static XJoinChain getXJoinChainFromJoinChain(JoinChain jc) {
+ XJoinChain xjc = XCF.createXJoinChain();
+ xjc.setName(jc.getName());
+ xjc.setDescription(jc.getDescription());
+ xjc.setDisplayString(jc.getDisplayString());
+ xjc.setDestTable(jc.getDestTable());
+ xjc.setPaths(new XJoinPaths());
+
+ for (JoinChain.Path path : jc.getPaths()) {
+ xjc.getPaths().getPath().add(xJoinPathFromJoinPath(path));
+ }
+ return xjc;
+ }
+
+ public static XJoinPath xJoinPathFromJoinPath(JoinChain.Path path) {
+ XJoinPath xjp = XCF.createXJoinPath();
+ xjp.setEdges(new XJoinEdges());
+ for (JoinChain.Edge edge : path.getLinks()) {
+ XJoinEdge xje = XCF.createXJoinEdge();
+ xje.setFrom(xTabReferenceFromTabReference(edge.getFrom()));
+ xje.setTo(xTabReferenceFromTabReference(edge.getTo()));
+ xjp.getEdges().getEdge().add(xje);
+ }
+ return xjp;
+ }
+
+ public static List<XTableReference> xTabReferencesFromHiveTabReferences(List<TableReference> hiveRefs) {
+ List<XTableReference> xrefList = new ArrayList<XTableReference>();
+
+ for (TableReference hRef : hiveRefs) {
+ xrefList.add(xTabReferenceFromTabReference(hRef));
+ }
+ return xrefList;
+ }
+
+ private static XTableReference xTabReferenceFromTabReference(TableReference ref) {
+ XTableReference xref = XCF.createXTableReference();
+ xref.setTable(ref.getDestTable());
+ xref.setColumn(ref.getDestColumn());
+ xref.setMapsToMany(ref.isMapsToMany());
+ return xref;
+ }
+
+ /**
+ * Create hive ql CubeMeasure from JAXB counterpart
+ *
+ * @param xm
+ * @return {@link CubeMeasure}
+ */
+ public static CubeMeasure hiveMeasureFromXMeasure(XMeasure xm) {
+ Date startDate = xm.getStartTime() == null ? null : xm.getStartTime().toGregorianCalendar().getTime();
+ Date endDate = xm.getEndTime() == null ? null : xm.getEndTime().toGregorianCalendar().getTime();
+ CubeMeasure cm = new ColumnMeasure(new FieldSchema(xm.getName(), xm.getType().name().toLowerCase(),
+ xm.getDescription()),
+ xm.getDisplayString(),
+ xm.getFormatString(),
+ xm.getDefaultAggr(),
+ xm.getUnit(),
+ startDate,
+ endDate,
+ null,
+ xm.getMin(),
+ xm.getMax(),
+ mapFromXProperties(xm.getTags())
+ );
+ return cm;
+ }
+
+ /**
+ * Create cube's JoinChain from JAXB counterpart
+ *
+ * @param xj
+ * @return {@link JoinChain}
+ */
+ public static JoinChain joinChainFromXJoinChain(XJoinChain xj) {
+ JoinChain jc = new JoinChain(xj.getName(), xj.getDisplayString(), xj.getDescription());
+ for (int i = 0; i < xj.getPaths().getPath().size(); i++) {
+ XJoinPath xchain = xj.getPaths().getPath().get(i);
+ List<TableReference> chain = new ArrayList<TableReference>(xchain.getEdges().getEdge().size() * 2);
+
+ for (XJoinEdge xRef : xchain.getEdges().getEdge()) {
+ chain.add(new TableReference(xRef.getFrom().getTable(), xRef.getFrom().getColumn(),
+ xRef.getFrom().isMapsToMany()));
+ chain.add(new TableReference(xRef.getTo().getTable(), xRef.getTo().getColumn(), xRef.getTo().isMapsToMany()));
+ }
+ jc.addPath(chain);
+ }
+ return jc;
+ }
+
+ public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws LensException {
+ ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(),
+ xe.getDescription()),
+ xe.getDisplayString(),
+ mapFromXProperties(xe.getTags()),
+ exprSpecFromXExprColumn(xe.getExprSpec()));
+ return ec;
+ }
+
+ /**
+ * Convert JAXB properties to Map<String, String>
+ *
+ * @param xProperties
+ * @return {@link Map}
+ */
+ public static Map<String, String> mapFromXProperties(XProperties xProperties) {
+ Map<String, String> properties = new HashMap<String, String>();
+ if (xProperties != null && !xProperties.getProperty().isEmpty()) {
+ for (XProperty xp : xProperties.getProperty()) {
+ properties.put(xp.getName(), xp.getValue());
+ }
+ }
+ return properties;
+ }
+
+ /**
+ * Convert string map to XProperties
+ */
+ public static List<XProperty> xPropertiesFromMap(Map<String, String> map) {
+ List<XProperty> xpList = new ArrayList<XProperty>();
+ if (map != null && !map.isEmpty()) {
+ for (Map.Entry<String, String> e : map.entrySet()) {
+ XProperty property = XCF.createXProperty();
+ property.setName(e.getKey());
+ property.setValue(e.getValue());
+ xpList.add(property);
+ }
+ }
+ return xpList;
+ }
+
+ public static Set<XSegment> xSegmentsFromSegments(Set<Segment> segs) {
+ Set<XSegment> xsegs = new HashSet<XSegment>();
+ if (segs != null && !segs.isEmpty()) {
+ for (Segment seg : segs) {
+ XSegment xcubeSeg = XCF.createXSegment();
+ xcubeSeg.setCubeName(seg.getName());
+ xcubeSeg.setSegmentParameters(getXpropertiesFromSegment(seg));
+ xsegs.add(xcubeSeg);
+ }
+ }
+ return xsegs;
+ }
+
+ public static XProperties getXpropertiesFromSegment(Segment cseg) {
+ XProperties xproperties = XCF.createXProperties();
+ for (String prop : cseg.getProperties().keySet()) {
+ String segPrefix = MetastoreUtil.getSegmentPropertyKey(cseg.getName());
+ if (prop.startsWith(segPrefix)){
+ XProperty xprop = XCF.createXProperty();
+ xprop.setName(prop.replace(segPrefix, ""));
+ xprop.setValue(cseg.getProperties().get(prop));
+ xproperties.getProperty().add(xprop);
+ }
+ }
+ return xproperties;
+ }
+
+
+ public static FieldSchema fieldSchemaFromColumn(XColumn c) {
+ if (c == null) {
+ return null;
+ }
+
+ return new FieldSchema(c.getName(), c.getType().toLowerCase(), c.getComment());
+ }
+
+ public static XColumn columnFromFieldSchema(FieldSchema fs) {
+ if (fs == null) {
+ return null;
+ }
+ XColumn c = XCF.createXColumn();
+ c.setName(fs.getName());
+ c.setType(fs.getType());
+ c.setComment(fs.getComment());
+ return c;
+ }
+
+ public static ArrayList<FieldSchema> fieldSchemaListFromColumns(XColumns columns) {
+ if (columns != null && !columns.getColumn().isEmpty()) {
+ ArrayList<FieldSchema> fsList = new ArrayList<FieldSchema>(columns.getColumn().size());
+ for (XColumn c : columns.getColumn()) {
+ fsList.add(fieldSchemaFromColumn(c));
+ }
+ return fsList;
+ }
+ return null;
+ }
+
+ public static Map<String, String> columnStartAndEndTimeFromXColumns(XColumns columns) {
+ if (columns != null && !columns.getColumn().isEmpty()) {
+ Map<String, String> colStartTimeMap = new HashMap<String, String>();
+ for (XColumn c : columns.getColumn()) {
+ if (!(c.getStartTime() == null)) {
+ colStartTimeMap.put(MetastoreConstants.FACT_COL_START_TIME_PFX.concat(c.getName()), c.getStartTime());
+ }
+ if (!(c.getEndTime() == null)) {
+ colStartTimeMap.put(MetastoreConstants.FACT_COL_END_TIME_PFX.concat(c.getName()), c.getEndTime());
+ }
+ }
+ return colStartTimeMap;
+ }
+ return null;
+ }
+
+ public static List<XColumn> columnsFromFieldSchemaList(List<FieldSchema> fslist) {
+ List<XColumn> cols = new ArrayList<XColumn>();
+ if (fslist == null || fslist.isEmpty()) {
+ return cols;
+ }
+
+ for (FieldSchema fs : fslist) {
+ cols.add(columnFromFieldSchema(fs));
+ }
+ return cols;
+ }
+
+ public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) {
+ if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
+ Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>();
+
+ for (XStorageTableElement ste : storageTables.getStorageTable()) {
+ Set<UpdatePeriod> updatePeriods = new TreeSet<>();
+ // Check if the update period array is empty.
+ List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod();
+ if (updatePeriodList.isEmpty()) {
+ List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods()
+ .getUpdatePeriodTableDescriptor();
+ for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) {
+ updatePeriodList.add(tableDescriptor.getUpdatePeriod());
+ }
+ }
+ for (XUpdatePeriod upd : updatePeriodList) {
+ updatePeriods.add(UpdatePeriod.valueOf(upd.name()));
+ }
+ factUpdatePeriods.put(ste.getStorageName(), updatePeriods);
+ }
+ return factUpdatePeriods;
+ }
+ return null;
+ }
+
+ public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) {
+ if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
+ Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>();
+
+ for (XStorageTableElement ste : storageTables.getStorageTable()) {
+ UpdatePeriod dumpPeriod = null;
+ if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) {
+ dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name());
+ }
+ dumpPeriods.put(ste.getStorageName(), dumpPeriod);
+ }
+ return dumpPeriods;
+ }
+ return null;
+ }
+
+ public static Storage storageFromXStorage(XStorage xs) {
+ if (xs == null) {
+ return null;
+ }
+
+ Storage storage;
+ try {
+ Class<?> clazz = Class.forName(xs.getClassname());
+ Constructor<?> constructor = clazz.getConstructor(String.class);
+ storage = (Storage) constructor.newInstance(xs.getName());
+ storage.addProperties(mapFromXProperties(xs.getProperties()));
+ return storage;
+ } catch (Exception e) {
+ log.error("Could not create storage class" + xs.getClassname() + "with name:" + xs.getName(), e);
+ throw new WebApplicationException(e);
+ }
+ }
+
+ public static XStorage xstorageFromStorage(Storage storage) {
+ if (storage == null) {
+ return null;
+ }
+
+ XStorage xstorage = null;
+ xstorage = XCF.createXStorage();
+ xstorage.setProperties(new XProperties());
+ xstorage.setName(storage.getName());
+ xstorage.setClassname(storage.getClass().getCanonicalName());
+ xstorage.getProperties().getProperty().addAll(xPropertiesFromMap(storage.getProperties()));
+ return xstorage;
+ }
+
+ public static XDimensionTable dimTableFromCubeDimTable(CubeDimensionTable cubeDimTable) {
+ if (cubeDimTable == null) {
+ return null;
+ }
+
+ XDimensionTable dimTab = XCF.createXDimensionTable();
+ dimTab.setDimensionName(cubeDimTable.getDimName());
+ dimTab.setTableName(cubeDimTable.getName());
+ dimTab.setWeight(cubeDimTable.weight());
+ dimTab.setColumns(new XColumns());
+ dimTab.setProperties(new XProperties());
+ dimTab.setStorageTables(new XStorageTables());
+
+ for (FieldSchema column : cubeDimTable.getColumns()) {
+ dimTab.getColumns().getColumn().add(columnFromFieldSchema(column));
+ }
+ dimTab.getProperties().getProperty().addAll(xPropertiesFromMap(cubeDimTable.getProperties()));
+
+ return dimTab;
+ }
+
+ public static List<? extends XTableReference> dimRefListFromTabRefList(
+ List<TableReference> tabRefs) {
+ if (tabRefs != null && !tabRefs.isEmpty()) {
+ List<XTableReference> xTabRefs = new ArrayList<XTableReference>(tabRefs.size());
+ for (TableReference ref : tabRefs) {
+ XTableReference xRef = XCF.createXTableReference();
+ xRef.setColumn(ref.getDestColumn());
+ xRef.setTable(ref.getDestTable());
+ xRef.setMapsToMany(ref.isMapsToMany());
+ xTabRefs.add(xRef);
+ }
+ return xTabRefs;
+ }
+
+ return null;
+ }
+
+ public static CubeDimensionTable cubeDimTableFromDimTable(XDimensionTable dimensionTable) throws LensException {
+
+ return new CubeDimensionTable(dimensionTable.getDimensionName(),
+ dimensionTable.getTableName(),
+ fieldSchemaListFromColumns(dimensionTable.getColumns()),
+ dimensionTable.getWeight(),
+ dumpPeriodsFromStorageTables(dimensionTable.getStorageTables()),
+ mapFromXProperties(dimensionTable.getProperties()));
+ }
+
+ public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensException {
+ List<FieldSchema> columns = fieldSchemaListFromColumns(fact.getColumns());
+
+ Map<String, Set<UpdatePeriod>> storageUpdatePeriods = getFactUpdatePeriodsFromStorageTables(
+ fact.getStorageTables());
+ Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap = storageTablePrefixMapOfStorage(
+ fact.getStorageTables());
+ return new CubeFactTable(fact.getCubeName(), fact.getName(), columns, storageUpdatePeriods, fact.getWeight(),
+ mapFromXProperties(fact.getProperties()), storageTablePrefixMap);
+ }
+
+ public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException {
+
+ Map<String, String> props = new HashMap<>();
+ // Skip properties with keyword internal. These properties are internal to lens
+ // and users are not supposed to see them.
+ for(String prop : mapFromXProperties(seg.getProperties()).keySet()) {
+ if (!(prop.toLowerCase().startsWith(MetastoreConstants.SEGMENTATION_KEY_PFX))) {
+ props.put(prop, mapFromXProperties(seg.getProperties()).get(prop));
+ }
+ }
+ return new Segmentation(seg.getCubeName(),
+ seg.getName(),
+ segmentsFromXSegments(seg.getSegements()),
+ seg.getWeight(),
+ props);
+ }
+
+
+ public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) {
+ XFactTable fact = XCF.createXFactTable();
+ fact.setName(cFact.getName());
+ fact.setColumns(new XColumns());
+ fact.setProperties(new XProperties());
+ fact.setStorageTables(new XStorageTables());
+ fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties()));
+ fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns()));
+ fact.setWeight(cFact.weight());
+ fact.setCubeName(cFact.getCubeName());
+ return fact;
+ }
+
+ public static XSegmentation xsegmentationFromSegmentation(Segmentation cSeg) {
+ XSegmentation seg = XCF.createXSegmentation();
+ seg.setName(cSeg.getName());
+ seg.setProperties(new XProperties());
+ seg.setSegements(new XSegments());
+ seg.setWeight(cSeg.weight());
+ seg.setCubeName(cSeg.getBaseCube());
+ if (xPropertiesFromMap(cSeg.getProperties()) != null) {
+ seg.getProperties().getProperty().addAll(xPropertiesFromMap(cSeg.getProperties()));
+ }
+ seg.getSegements().getSegment().
+ addAll(xSegmentsFromSegments(cSeg.getSegments()));
+ return seg;
+ }
+
+ public static StorageTableDesc storageTableDescFromXStorageTableDesc(
+ XStorageTableDesc xtableDesc) {
+ StorageTableDesc tblDesc = new StorageTableDesc();
+ tblDesc.setTblProps(mapFromXProperties(xtableDesc.getTableParameters()));
+ tblDesc.setSerdeProps(mapFromXProperties(xtableDesc.getSerdeParameters()));
+ tblDesc.setPartCols(fieldSchemaListFromColumns(xtableDesc.getPartCols()));
+ tblDesc.setTimePartCols(xtableDesc.getTimePartCols());
+ tblDesc.setExternal(xtableDesc.isExternal());
+ tblDesc.setLocation(xtableDesc.getTableLocation());
+ tblDesc.setInputFormat(xtableDesc.getInputFormat());
+ tblDesc.setOutputFormat(xtableDesc.getOutputFormat());
+ tblDesc.setFieldDelim(xtableDesc.getFieldDelimiter());
+ tblDesc.setFieldEscape(xtableDesc.getEscapeChar());
+ tblDesc.setCollItemDelim(xtableDesc.getCollectionDelimiter());
+ tblDesc.setLineDelim(xtableDesc.getLineDelimiter());
+ tblDesc.setMapKeyDelim(xtableDesc.getMapKeyDelimiter());
+ tblDesc.setSerName(xtableDesc.getSerdeClassName());
+ tblDesc.setStorageHandler(xtableDesc.getStorageHandlerName());
+ return tblDesc;
+ }
+
+ public static StorageTableDesc storageTableDescFromXStorageTableElement(
+ XStorageTableElement storageTableElement) {
+ return storageTableDescFromXStorageTableDesc(storageTableElement.getTableDesc());
+ }
+
+ public static XStorageTableElement getXStorageTableFromHiveTable(Table tbl) {
+ XStorageTableElement tblElement = new XStorageTableElement();
+ tblElement.setUpdatePeriods(new XUpdatePeriods());
+ tblElement.setTableDesc(getStorageTableDescFromHiveTable(tbl));
+ return tblElement;
+ }
+
+ public static XStorageTableDesc getStorageTableDescFromHiveTable(Table tbl) {
+ XStorageTableDesc tblDesc = new XStorageTableDesc();
+ tblDesc.setPartCols(new XColumns());
+ tblDesc.setTableParameters(new XProperties());
+ tblDesc.setSerdeParameters(new XProperties());
+ tblDesc.getPartCols().getColumn().addAll(columnsFromFieldSchemaList(tbl.getPartCols()));
+ String timePartCols = tbl.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS);
+ if (timePartCols != null) {
+ tblDesc.getTimePartCols().addAll(Arrays.asList(org.apache.commons.lang.StringUtils.split(timePartCols, ",")));
+ }
+ tblDesc.setNumBuckets(tbl.getNumBuckets());
+ tblDesc.getBucketCols().addAll(tbl.getBucketCols());
+ List<String> sortCols = new ArrayList<String>();
+ List<Integer> sortOrders = new ArrayList<Integer>();
+ for (Order order : tbl.getSortCols()) {
+ sortCols.add(order.getCol());
+ sortOrders.add(order.getOrder());
+ }
+ tblDesc.getSortCols().addAll(sortCols);
+ tblDesc.getSortColOrder().addAll(sortOrders);
+
+ XSkewedInfo xskewinfo = new XSkewedInfo();
+ xskewinfo.getColNames().addAll(tbl.getSkewedColNames());
+ for (List<String> value : tbl.getSkewedColValues()) {
+ XSkewColList colVallist = new XSkewColList();
+ colVallist.getElements().addAll(value);
+ xskewinfo.getColValues().add(colVallist);
+ XSkewedValueLocation valueLocation = new XSkewedValueLocation();
+ if (tbl.getSkewedColValueLocationMaps().get(value) != null) {
+ valueLocation.setValue(colVallist);
+ valueLocation.setLocation(tbl.getSkewedColValueLocationMaps().get(value));
+ xskewinfo.getValueLocationMap().add(valueLocation);
+ }
+ }
+
+ tblDesc.getTableParameters().getProperty().addAll(xPropertiesFromMap(tbl.getParameters()));
+ tblDesc.getSerdeParameters().getProperty().addAll(xPropertiesFromMap(
+ tbl.getTTable().getSd().getSerdeInfo().getParameters()));
+ tblDesc.setExternal(tbl.getTableType().equals(TableType.EXTERNAL_TABLE));
+ tblDesc.setCompressed(tbl.getTTable().getSd().isCompressed());
+ tblDesc.setTableLocation(tbl.getDataLocation().toString());
+ tblDesc.setInputFormat(tbl.getInputFormatClass().getCanonicalName());
+ tblDesc.setOutputFormat(tbl.getOutputFormatClass().getCanonicalName());
+ tblDesc.setFieldDelimiter(tbl.getSerdeParam(serdeConstants.FIELD_DELIM));
+ tblDesc.setLineDelimiter(tbl.getSerdeParam(serdeConstants.LINE_DELIM));
+ tblDesc.setCollectionDelimiter(tbl.getSerdeParam(serdeConstants.COLLECTION_DELIM));
+ tblDesc.setMapKeyDelimiter(tbl.getSerdeParam(serdeConstants.MAPKEY_DELIM));
+ tblDesc.setEscapeChar(tbl.getSerdeParam(serdeConstants.ESCAPE_CHAR));
+ tblDesc.setSerdeClassName(tbl.getSerializationLib());
+ tblDesc.setStorageHandlerName(tbl.getStorageHandler() != null
+ ? tbl.getStorageHandler().getClass().getCanonicalName() : "");
+ return tblDesc;
+ }
+
+ public static Map<String, StorageTableDesc> tableDescPrefixMapFromXStorageTables(XStorageTables storageTables) {
+ Map<String, StorageTableDesc> storageTablePrefixToDescMap = new HashMap<>();
+ if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
+ for (XStorageTableElement sTbl : storageTables.getStorageTable()) {
+ if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl
+ .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) {
+ for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods()
+ .getUpdatePeriodTableDescriptor()) {
+ // Get table name with update period as the prefix.
+ storageTablePrefixToDescMap.put(updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName(),
+ storageTableDescFromXStorageTableDesc(updatePeriodTable.getTableDesc()));
+ }
+ } else {
+ storageTablePrefixToDescMap.put(sTbl.getStorageName(), storageTableDescFromXStorageTableElement(sTbl));
+ }
+ }
+ }
+ return storageTablePrefixToDescMap;
+ }
+
+ public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) {
+ Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>();
+ if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
+ for (XStorageTableElement sTbl : storageTables.getStorageTable()) {
+ Map<UpdatePeriod, String> storageNameMap = new HashMap<>();
+ if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl
+ .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) {
+ for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods()
+ .getUpdatePeriodTableDescriptor()) {
+ // Get table name with update period as the prefix.
+ storageNameMap.put(UpdatePeriod.valueOf(updatePeriodTable.getUpdatePeriod().value()),
+ updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName());
+ }
+ } else {
+ for (XUpdatePeriod updatePeriod : sTbl.getUpdatePeriods().getUpdatePeriod()) {
+ storageNameMap.put(UpdatePeriod.valueOf(updatePeriod.value()), sTbl.getStorageName());
+ }
+ }
+ storageTableMap.put(sTbl.getStorageName(), storageNameMap);
+ }
+ }
+ return storageTableMap;
+ }
+
+ public static Set<Segment> segmentsFromXSegments(XSegments segs) {
+ Set<Segment> cubeSegs = new HashSet<>();
+ for (XSegment xcube : segs.getSegment()){
+ Map<String, String> segProp = new HashMap<>();
+ if (xcube.getSegmentParameters() != null) {
+ for (XProperty prop : xcube.getSegmentParameters().getProperty()) {
+ segProp.put(prop.getName(), prop.getValue());
+ }
+ }
+ cubeSegs.add(new Segment(xcube.getCubeName(), segProp));
+ }
+ return cubeSegs;
+ }
+
+ public static Map<String, Date> timePartSpecfromXTimePartSpec(
+ XTimePartSpec xtimePartSpec) {
+ Map<String, Date> timePartSpec = new HashMap<String, Date>();
+ if (xtimePartSpec != null && !xtimePartSpec.getPartSpecElement().isEmpty()) {
+ for (XTimePartSpecElement xtimePart : xtimePartSpec.getPartSpecElement()) {
+ timePartSpec.put(xtimePart.getKey(), getDateFromXML(xtimePart.getValue()));
+ }
+ }
+ return timePartSpec;
+ }
+
+ public static Map<String, String> nonTimePartSpecfromXNonTimePartSpec(
+ XPartSpec xnonTimePartSpec) {
+ Map<String, String> nonTimePartSpec = new HashMap<String, String>();
+ if (xnonTimePartSpec != null && !xnonTimePartSpec.getPartSpecElement().isEmpty()) {
+ for (XPartSpecElement xPart : xnonTimePartSpec.getPartSpecElement()) {
+ nonTimePartSpec.put(xPart.getKey(), xPart.getValue());
+ }
+ }
+ return nonTimePartSpec;
+ }
+
+ public static XPartitionList xpartitionListFromPartitionList(String cubeTableName, List<Partition> partitions,
+ List<String> timePartCols) throws HiveException {
+ XPartitionList xPartitionList = new XPartitionList();
+ xPartitionList.getPartition();
+ if (partitions != null) {
+ for (Partition partition : partitions) {
+ xPartitionList.getPartition().add(xpartitionFromPartition(cubeTableName, partition, timePartCols));
+ }
+ }
+ return xPartitionList;
+ }
+
+ public static XPartition xpartitionFromPartition(String cubeTableName, Partition p, List<String> timePartCols)
+ throws HiveException {
+ XPartition xp = new XPartition();
+ xp.setFactOrDimensionTableName(cubeTableName);
+ xp.setPartitionParameters(new XProperties());
+ xp.setSerdeParameters(new XProperties());
+ xp.setName(p.getCompleteName());
+ xp.setLocation(p.getLocation());
+ xp.setInputFormat(p.getInputFormatClass().getCanonicalName());
+ xp.setOutputFormat(p.getOutputFormatClass().getCanonicalName());
+ xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters()));
+ String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
+ xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam));
+ LinkedHashMap<String, String> partSpec = p.getSpec();
+ xp.setFullPartitionSpec(new XPartSpec());
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ XPartSpecElement e = new XPartSpecElement();
+ e.setKey(entry.getKey());
+ e.setValue(entry.getValue());
+ xp.getFullPartitionSpec().getPartSpecElement().add(e);
+ }
+ try {
+ xp.setTimePartitionSpec(new XTimePartSpec());
+ xp.setNonTimePartitionSpec(new XPartSpec());
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (timePartCols.contains(entry.getKey())) {
+ XTimePartSpecElement timePartSpecElement = new XTimePartSpecElement();
+ timePartSpecElement.setKey(entry.getKey());
+ timePartSpecElement
+ .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse(
+ entry.getValue())));
+ xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
+ } else {
+ XPartSpecElement partSpecElement = new XPartSpecElement();
+ partSpecElement.setKey(entry.getKey());
+ partSpecElement.setValue(entry.getValue());
+ xp.getNonTimePartitionSpec().getPartSpecElement().add(partSpecElement);
+ }
+ }
+ } catch (ParseException exc) {
+ log.debug("can't form time part spec from " + partSpec, exc);
+ xp.setTimePartitionSpec(null);
+ xp.setNonTimePartitionSpec(null);
+ }
+ xp.setSerdeClassname(p.getTPartition().getSd().getSerdeInfo().getSerializationLib());
+ xp.getSerdeParameters().getProperty().addAll(xPropertiesFromMap(
+ p.getTPartition().getSd().getSerdeInfo().getParameters()));
+ return xp;
+ }
+
+ public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException {
+ partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters()));
+ partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters()));
+ partition.setLocation(xp.getLocation());
+ if (xp.getInputFormat() != null) {
+ partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class));
+ }
+ if (xp.getOutputFormat() != null) {
+ Class<? extends HiveOutputFormat> outputFormatClass =
+ Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class);
+ partition.setOutputFormatClass(outputFormatClass);
+ }
+ partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name());
+ partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname());
+ }
+
+ public static StoragePartitionDesc storagePartSpecFromXPartition(
+ XPartition xpart) {
+ StoragePartitionDesc partDesc = new StoragePartitionDesc(
+ xpart.getFactOrDimensionTableName(),
+ timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()),
+ nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()),
+ UpdatePeriod.valueOf(xpart.getUpdatePeriod().name()));
+ partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters()));
+ partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters()));
+ partDesc.setLocation(xpart.getLocation());
+ partDesc.setInputFormat(xpart.getInputFormat());
+ partDesc.setOutputFormat(xpart.getOutputFormat());
+ partDesc.setSerializationLib(xpart.getSerdeClassname());
+ return partDesc;
+ }
+
+ public static List<StoragePartitionDesc> storagePartSpecListFromXPartitionList(
+ final XPartitionList xpartList) {
+ ArrayList<StoragePartitionDesc> ret = new ArrayList<StoragePartitionDesc>();
+ for (XPartition xpart : xpartList.getPartition()) {
+ ret.add(storagePartSpecFromXPartition(xpart));
+ }
+ return ret;
+ }
+
+ public static Dimension dimensionFromXDimension(XDimension dimension) throws LensException {
+ Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>();
+ for (XDimAttribute xd : dimension.getAttributes().getDimAttribute()) {
+ dims.add(hiveDimAttrFromXDimAttr(xd));
+ }
+
+ Set<ExprColumn> expressions = new LinkedHashSet<ExprColumn>();
+ if (dimension.getExpressions() != null && !dimension.getExpressions().getExpression().isEmpty()) {
+ for (XExprColumn xe : dimension.getExpressions().getExpression()) {
+ expressions.add(hiveExprColumnFromXExprColumn(xe));
+ }
+ }
+
+ Set<JoinChain> joinchains = new LinkedHashSet<JoinChain>();
+ if (dimension.getJoinChains() != null && !dimension.getJoinChains().getJoinChain().isEmpty()) {
+ for (XJoinChain xj : dimension.getJoinChains().getJoinChain()) {
+ joinchains.add(joinChainFromXJoinChain(xj));
+ }
+ }
+
+ Map<String, String> properties = mapFromXProperties(dimension.getProperties());
+ return new Dimension(dimension.getName(), dims, expressions, joinchains, properties, 0L);
+ }
+
+ public static XDimension xdimensionFromDimension(Dimension dimension) {
+ XDimension xd = XCF.createXDimension();
+ xd.setName(dimension.getName());
+ xd.setAttributes(new XDimAttributes());
+ xd.setExpressions(new XExpressions());
+ xd.setJoinChains(new XJoinChains());
+ xd.setProperties(new XProperties());
+
+ xd.getProperties().getProperty().addAll(xPropertiesFromMap(((AbstractCubeTable) dimension).getProperties()));
+ for (CubeDimAttribute cd : dimension.getAttributes()) {
+ xd.getAttributes().getDimAttribute().add(xDimAttrFromHiveDimAttr(cd, dimension));
+ }
+
+ for (ExprColumn ec : dimension.getExpressions()) {
+ xd.getExpressions().getExpression().add(xExprColumnFromHiveExprColumn(ec));
+ }
+
+ for (JoinChain jc : dimension.getJoinChains()) {
+ xd.getJoinChains().getJoinChain().add(getXJoinChainFromJoinChain(jc));
+ }
+
+ return xd;
+ }
+
+ public static XNativeTable nativeTableFromMetaTable(Table table) {
+ XNativeTable xtable = XCF.createXNativeTable();
+ xtable.setColumns(new XColumns());
+ xtable.setName(table.getTableName());
+ xtable.setDbname(table.getDbName());
+ xtable.setOwner(table.getOwner());
+ xtable.setCreatetime(table.getTTable().getCreateTime());
+ xtable.setLastAccessTime(table.getTTable().getLastAccessTime());
+ xtable.getColumns().getColumn().addAll(columnsFromFieldSchemaList(table.getCols()));
+ xtable.setStorageDescriptor(getStorageTableDescFromHiveTable(table));
+ xtable.setTableType(table.getTableType().name());
+ return xtable;
+ }
+
+ public static Map<String, String> getFullPartSpecAsMap(XPartition partition) {
+ Map<String, String> spec = Maps.newHashMap();
+ if (partition.getTimePartitionSpec() != null) {
+ for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) {
+ spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format()
+ .format(getDateFromXML(timePartSpecElement.getValue())));
+ }
+ }
+ if (partition.getNonTimePartitionSpec() != null) {
+ for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) {
+ spec.put(partSpecElement.getKey(), partSpecElement.getValue());
+ }
+ }
+ return spec;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 25acb01..e6e9f8f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -635,12 +635,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
log.info("Completeness for the measure_tag {} is {}, threshold: {}, for the hour {}", tag,
completenessResult.getValue(), completenessThreshold, formatter.format(completenessResult.getKey()));
String measureorExprFromTag = tagToMeasureOrExprMap.get(tag);
- Map<String, Float> incompletePartition = dataCompletenessMap.get(measureorExprFromTag);
- if (incompletePartition == null) {
- incompletePartition = new HashMap<>();
- dataCompletenessMap.put(measureorExprFromTag, incompletePartition);
- }
- incompletePartition.put(formatter.format(completenessResult.getKey()), completenessResult.getValue());
+ dataCompletenessMap.computeIfAbsent(measureorExprFromTag, k -> new HashMap<>())
+ .put(formatter.format(completenessResult.getKey()), completenessResult.getValue());
isDataComplete = false;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index bc008ae..22e2e09 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -59,7 +59,7 @@ class StorageTableResolver implements ContextRewriter {
validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL);
if (maxIntervalStr != null) {
- this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr);
+ this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase());
} else {
this.maxInterval = null;
}