You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:50:15 UTC
[41/50] [abbrv] incubator-lens git commit: LENS-257: Altering fact
table should alter its storage table descriptions also
LENS-257: Altering fact table should alter its storage table descriptions also
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/20949e03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/20949e03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/20949e03
Branch: refs/heads/current-release-line
Commit: 20949e0312907de3dc8a7e3b725b322e44b5ab42
Parents: 8f716a7
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Apr 10 19:56:42 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Apr 10 19:56:42 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 59 +++++----
.../org/apache/lens/cube/metadata/Storage.java | 3 +-
.../cube/metadata/TestCubeMetastoreClient.java | 130 +++++++++++++++----
.../metastore/CubeMetastoreServiceImpl.java | 8 +-
.../server/metastore/TestMetastoreService.java | 122 ++++++++++++++++-
5 files changed, 265 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/20949e03/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 11ef7ec..fa56213 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -395,12 +395,18 @@ public class CubeMetastoreClient {
Hive.closeCurrent();
}
- private void createStorageHiveTable(Table parent, String storage, StorageTableDesc crtTblDesc) throws HiveException {
+ private void createOrAlterStorageHiveTable(Table parent, String storage, StorageTableDesc crtTblDesc)
+ throws HiveException {
try {
Table tbl = getStorage(storage).getStorageTable(getClient(), parent, crtTblDesc);
- getClient().createTable(tbl);
- // do get to update cache
- getTable(tbl.getTableName());
+ if (tableExists(tbl.getTableName())) {
+ // alter table
+ alterHiveTable(tbl.getTableName(), tbl);
+ } else {
+ getClient().createTable(tbl);
+ // do get to update cache
+ getTable(tbl.getTableName());
+ }
} catch (Exception e) {
throw new HiveException("Exception creating table", e);
}
@@ -624,7 +630,7 @@ public class CubeMetastoreClient {
if (storageTableDescs != null) {
// create tables for each storage
for (Map.Entry<String, StorageTableDesc> entry : storageTableDescs.entrySet()) {
- createStorageHiveTable(cTable, entry.getKey(), entry.getValue());
+ createOrAlterStorageHiveTable(cTable, entry.getKey(), entry.getValue());
}
}
}
@@ -641,7 +647,7 @@ public class CubeMetastoreClient {
public void addStorage(CubeFactTable fact, String storage, Set<UpdatePeriod> updatePeriods,
StorageTableDesc storageTableDesc) throws HiveException {
fact.addStorage(storage, updatePeriods);
- createStorageHiveTable(getTable(fact.getName()), storage, storageTableDesc);
+ createOrAlterStorageHiveTable(getTable(fact.getName()), storage, storageTableDesc);
alterCubeTable(fact.getName(), getTable(fact.getName()), fact);
updateFactCache(fact.getName());
}
@@ -658,7 +664,7 @@ public class CubeMetastoreClient {
public void addStorage(CubeDimensionTable dim, String storage, UpdatePeriod dumpPeriod,
StorageTableDesc storageTableDesc) throws HiveException {
dim.alterSnapshotDumpPeriod(storage, dumpPeriod);
- createStorageHiveTable(getTable(dim.getName()), storage, storageTableDesc);
+ createOrAlterStorageHiveTable(getTable(dim.getName()), storage, storageTableDesc);
alterCubeTable(dim.getName(), getTable(dim.getName()), dim);
updateDimCache(dim.getName());
}
@@ -971,9 +977,9 @@ public class CubeMetastoreClient {
return partSpec;
}
- public boolean tableExists(String cubeName) throws HiveException {
+ public boolean tableExists(String tblName) throws HiveException {
try {
- return (getClient().getTable(cubeName.toLowerCase(), false) != null);
+ return (getClient().getTable(tblName.toLowerCase(), false) != null);
} catch (HiveException e) {
throw new HiveException("Could not check whether table exists", e);
}
@@ -1754,7 +1760,6 @@ public class CubeMetastoreClient {
}
if (enableCaching) {
// refresh the table in cache
- //TODO: don't need to fetch again. can put table -> hiveTable
refreshTable(table);
}
}
@@ -1967,22 +1972,24 @@ public class CubeMetastoreClient {
}
/**
- * Alter a cubefact with new definition
+ * Alter a cubefact with new definition and alter underlying storage tables as well.
*
* @param factTableName
* @param cubeFactTable
+ * @param storageTableDescs
+ *
* @throws HiveException
* @throws InvalidOperationException
*/
- public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable) throws HiveException {
+ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable,
+ Map<String, StorageTableDesc> storageTableDescs) throws HiveException {
Table factTbl = getTable(factTableName);
if (isFactTable(factTbl)) {
- boolean colsChanged = alterCubeTable(factTableName, factTbl, cubeFactTable);
- if (colsChanged) {
- // Change schema of all the storage tables
- for (String storage : cubeFactTable.getStorages()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factTableName, storage);
- alterHiveTable(storageTableName, getTable(storageTableName), cubeFactTable.getColumns());
+ alterCubeTable(factTableName, factTbl, cubeFactTable);
+ if (storageTableDescs != null) {
+ // create/alter tables for each storage
+ for (Map.Entry<String, StorageTableDesc> entry : storageTableDescs.entrySet()) {
+ createOrAlterStorageHiveTable(getTable(factTableName), entry.getKey(), entry.getValue());
}
}
updateFactCache(factTableName);
@@ -2004,22 +2011,22 @@ public class CubeMetastoreClient {
}
/**
- * Alter dimension table with new dimension definition
+ * Alter dimension table with new dimension definition and underlying storage tables as well
*
* @param dimTableName
* @param cubeDimensionTable
* @throws HiveException
* @throws InvalidOperationException
*/
- public void alterCubeDimensionTable(String dimTableName, CubeDimensionTable cubeDimensionTable) throws HiveException {
+ public void alterCubeDimensionTable(String dimTableName, CubeDimensionTable cubeDimensionTable,
+ Map<String, StorageTableDesc> storageTableDescs) throws HiveException {
Table dimTbl = getTable(dimTableName);
if (isDimensionTable(dimTbl)) {
- boolean colsChanged = alterCubeTable(dimTableName, dimTbl, cubeDimensionTable);
- if (colsChanged) {
- // Change schema of all the storage tables
- for (String storage : cubeDimensionTable.getStorages()) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTableName, storage);
- alterHiveTable(storageTableName, getTable(storageTableName), cubeDimensionTable.getColumns());
+ alterCubeTable(dimTableName, dimTbl, cubeDimensionTable);
+ if (storageTableDescs != null) {
+ // create/alter tables for each storage
+ for (Map.Entry<String, StorageTableDesc> entry : storageTableDescs.entrySet()) {
+ createOrAlterStorageHiveTable(getTable(dimTableName), entry.getKey(), entry.getValue());
}
}
updateDimCache(dimTableName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/20949e03/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index da70535..50ec4ce 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -24,6 +24,7 @@ import java.util.*;
import java.util.Map.Entry;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -142,7 +143,7 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
tbl.setNumBuckets(crtTbl.getNumBuckets());
}
- if (crtTbl.getStorageHandler() != null) {
+ if (!StringUtils.isBlank(crtTbl.getStorageHandler())) {
tbl.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE,
crtTbl.getStorageHandler());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/20949e03/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index baa659c..7337ee9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -78,6 +78,7 @@ public class TestCubeMetastoreClient {
private static String c1 = "C1";
private static String c2 = "C2";
private static String c3 = "C3";
+ private static String c4 = "C4";
private static Dimension zipDim, cityDim, stateDim, countryDim;
private static Set<CubeDimAttribute> zipAttrs = new HashSet<CubeDimAttribute>();
private static Set<CubeDimAttribute> cityAttrs = new HashSet<CubeDimAttribute>();
@@ -327,9 +328,14 @@ public class TestCubeMetastoreClient {
client.createStorage(hdfsStorage3);
Assert.assertEquals(3, client.getAllStorages().size());
+ Storage hdfsStorage4 = new HDFSStorage(c4);
+ client.createStorage(hdfsStorage4);
+ Assert.assertEquals(4, client.getAllStorages().size());
+
Assert.assertEquals(hdfsStorage, client.getStorage(c1));
Assert.assertEquals(hdfsStorage2, client.getStorage(c2));
Assert.assertEquals(hdfsStorage3, client.getStorage(c3));
+ Assert.assertEquals(hdfsStorage4, client.getStorage(c4));
}
@Test(priority = 1)
@@ -782,7 +788,7 @@ public class TestCubeMetastoreClient {
// Partition with different schema
FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part");
cubeFact.alterColumn(newcol);
- client.alterCubeFactTable(cubeFact.getName(), cubeFact);
+ client.alterCubeFactTable(cubeFact.getName(), cubeFact, storageTables);
String storageTableName = MetastoreUtil.getFactStorageTableName(factName, c1);
Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
@@ -879,7 +885,7 @@ public class TestCubeMetastoreClient {
alterupdates.add(UpdatePeriod.MONTHLY);
factTable.alterStorage(c2, alterupdates);
- client.alterCubeFactTable(factName, factTable);
+ client.alterCubeFactTable(factName, factTable, storageTables);
Table factHiveTable = Hive.get(conf).getTable(factName);
CubeFactTable altered = new CubeFactTable(factHiveTable);
@@ -901,9 +907,47 @@ public class TestCubeMetastoreClient {
}
Assert.assertTrue(contains);
- client.addStorage(altered, c3, updates, s1);
- Assert.assertTrue(altered.getStorages().contains("C3"));
- Assert.assertTrue(altered.getUpdatePeriods().get("C3").equals(updates));
+ // alter storage table desc
+ String c1TableName = MetastoreUtil.getFactStorageTableName(factName, c1);
+ Table c1Table = client.getTable(c1TableName);
+ Assert.assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
+ TextInputFormat.class.getCanonicalName());
+ s1 = new StorageTableDesc();
+ s1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
+ s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ s1.setPartCols(partCols);
+ s1.setTimePartCols(timePartCols);
+ s1.setFieldDelim(":");
+ storageTables.put(c1, s1);
+ storageTables.put(c4, s1);
+ factTable.addStorage(c4, updates);
+ client.alterCubeFactTable(factName, factTable, storageTables);
+ CubeFactTable altered2 = client.getCubeFact(factName);
+ Assert.assertTrue(client.tableExists(c1TableName));
+ Table alteredC1Table = client.getTable(c1TableName);
+ Assert.assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
+ SequenceFileInputFormat.class.getCanonicalName());
+ Assert.assertEquals(alteredC1Table.getSerdeParam(serdeConstants.FIELD_DELIM), ":");
+
+ boolean storageTableColsAltered = false;
+ for (FieldSchema column : alteredC1Table.getAllCols()) {
+ if (column.getName().equals("testfactcoladd") && column.getType().equals("int")) {
+ storageTableColsAltered = true;
+ break;
+ }
+ }
+ Assert.assertTrue(storageTableColsAltered);
+
+ Assert.assertTrue(altered2.getStorages().contains("C4"));
+ Assert.assertTrue(altered2.getUpdatePeriods().get("C4").equals(updates));
+ String c4TableName = MetastoreUtil.getFactStorageTableName(factName, c4);
+ Assert.assertTrue(client.tableExists(c4TableName));
+
+ // add storage
+ client.addStorage(altered2, c3, updates, s1);
+ CubeFactTable altered3 = client.getCubeFact(factName);
+ Assert.assertTrue(altered3.getStorages().contains("C3"));
+ Assert.assertTrue(altered3.getUpdatePeriods().get("C3").equals(updates));
String storageTableName = MetastoreUtil.getFactStorageTableName(factName, c3);
Assert.assertTrue(client.tableExists(storageTableName));
client.dropStorageFromFact(factName, c2);
@@ -1888,7 +1932,7 @@ public class TestCubeMetastoreClient {
// Partition with different schema
cubeDim.alterColumn(newcol);
- client.alterCubeDimensionTable(cubeDim.getName(), cubeDim);
+ client.alterCubeDimensionTable(cubeDim.getName(), cubeDim, storageTables);
Map<String, Date> timeParts2 = new HashMap<String, Date>();
timeParts2.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1);
@@ -2100,7 +2144,7 @@ public class TestCubeMetastoreClient {
@Test(priority = 2)
public void testAlterDim() throws Exception {
- String dimName = "test_alter_dim";
+ String dimTblName = "test_alter_dim";
List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
dimColumns.add(new FieldSchema("zipcode", "int", "code"));
@@ -2120,24 +2164,24 @@ public class TestCubeMetastoreClient {
Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
storageTables.put(c1, s1);
- client.createCubeDimensionTable(zipDim.getName(), dimName, dimColumns, 100L, dumpPeriods, null, storageTables);
+ client.createCubeDimensionTable(zipDim.getName(), dimTblName, dimColumns, 100L, dumpPeriods, null, storageTables);
- CubeDimensionTable dimTable = client.getDimensionTable(dimName);
+ CubeDimensionTable dimTable = client.getDimensionTable(dimTblName);
dimTable.alterColumn(new FieldSchema("testAddDim", "string", "test add column"));
List<CubeDimensionTable> tbls = client.getAllDimensionTables(zipDim);
boolean found = false;
for (CubeDimensionTable dim : tbls) {
- if (dim.getName().equalsIgnoreCase(dimName)) {
+ if (dim.getName().equalsIgnoreCase(dimTblName)) {
found = true;
break;
}
}
Assert.assertTrue(found);
- client.alterCubeDimensionTable(dimName, dimTable);
+ client.alterCubeDimensionTable(dimTblName, dimTable, storageTables);
- Table alteredHiveTable = Hive.get(conf).getTable(dimName);
+ Table alteredHiveTable = Hive.get(conf).getTable(dimTblName);
CubeDimensionTable altered = new CubeDimensionTable(alteredHiveTable);
List<FieldSchema> columns = altered.getColumns();
boolean contains = false;
@@ -2151,9 +2195,9 @@ public class TestCubeMetastoreClient {
// Test alter column
dimTable.alterColumn(new FieldSchema("testAddDim", "int", "change type"));
- client.alterCubeDimensionTable(dimName, dimTable);
+ client.alterCubeDimensionTable(dimTblName, dimTable, storageTables);
- altered = new CubeDimensionTable(Hive.get(conf).getTable(dimName));
+ altered = new CubeDimensionTable(Hive.get(conf).getTable(dimTblName));
boolean typeChanged = false;
for (FieldSchema column : altered.getColumns()) {
if (column.getName().equals("testadddim") && column.getType().equals("int")) {
@@ -2162,21 +2206,59 @@ public class TestCubeMetastoreClient {
}
}
Assert.assertTrue(typeChanged);
+
+ // alter storage table desc
+ String c1TableName = MetastoreUtil.getDimStorageTableName(dimTblName, c1);
+ Table c1Table = client.getTable(c1TableName);
+ Assert.assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
+ TextInputFormat.class.getCanonicalName());
+ s1 = new StorageTableDesc();
+ s1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
+ s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ s1.setPartCols(partCols);
+ s1.setTimePartCols(timePartCols);
+ storageTables.put(c1, s1);
+ storageTables.put(c4, s1);
+ dimTable.alterSnapshotDumpPeriod(c4, null);
+ client.alterCubeDimensionTable(dimTblName, dimTable, storageTables);
+ CubeDimensionTable altered2 = client.getDimensionTable(dimTblName);
+ Assert.assertTrue(client.tableExists(c1TableName));
+ Table alteredC1Table = client.getTable(c1TableName);
+ Assert.assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
+ SequenceFileInputFormat.class.getCanonicalName());
+ boolean storageTblColAltered = false;
+ for (FieldSchema column : alteredC1Table.getAllCols()) {
+ if (column.getName().equals("testadddim") && column.getType().equals("int")) {
+ storageTblColAltered = true;
+ break;
+ }
+ }
+ Assert.assertTrue(storageTblColAltered);
+ String c4TableName = MetastoreUtil.getDimStorageTableName(dimTblName, c4);
+ Assert.assertTrue(client.tableExists(c4TableName));
+ Table c4Table = client.getTable(c4TableName);
+ Assert.assertEquals(c4Table.getInputFormatClass().getCanonicalName(),
+ SequenceFileInputFormat.class.getCanonicalName());
+ Assert.assertTrue(altered2.getStorages().contains("C4"));
+ Assert.assertFalse(altered2.hasStorageSnapshots("C4"));
+
StorageTableDesc s2 = new StorageTableDesc();
s2.setInputFormat(TextInputFormat.class.getCanonicalName());
s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
client.addStorage(dimTable, c2, null, s2);
client.addStorage(dimTable, c3, UpdatePeriod.DAILY, s1);
- Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimName, c2)));
- Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimName, c3)));
- Assert.assertFalse(dimTable.hasStorageSnapshots("C2"));
- Assert.assertTrue(dimTable.hasStorageSnapshots("C3"));
- client.dropStorageFromDim(dimName, "C1");
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimName, c1)));
- client.dropDimensionTable(dimName, true);
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimName, c2)));
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimName, c3)));
- Assert.assertFalse(client.tableExists(dimName));
+ Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c2)));
+ Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c3)));
+ CubeDimensionTable altered3 = client.getDimensionTable(dimTblName);
+ Assert.assertFalse(altered3.hasStorageSnapshots("C2"));
+ Assert.assertTrue(altered3.hasStorageSnapshots("C3"));
+ client.dropStorageFromDim(dimTblName, "C1");
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c1)));
+ client.dropDimensionTable(dimTblName, true);
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c2)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c3)));
+ Assert.assertFalse(client.tableExists(dimTblName));
+ // alter storage tables
}
@Test(priority = 2)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/20949e03/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 185972e..0e6d057 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -347,8 +347,8 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).alterCubeDimensionTable(dimensionTable.getTableName(),
- JAXBUtils.cubeDimTableFromDimTable(dimensionTable));
- // TODO alter storage tables
+ JAXBUtils.cubeDimTableFromDimTable(dimensionTable),
+ JAXBUtils.storageTableMapFromXStorageTables(dimensionTable.getStorageTables()));
LOG.info("Updated dimension table " + dimensionTable.getTableName());
} catch (HiveException exc) {
throw new LensException(exc);
@@ -526,8 +526,8 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException {
try {
acquire(sessionid);
- getClient(sessionid).alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact));
- // TODO alter storage tables
+ getClient(sessionid).alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact),
+ JAXBUtils.storageTableMapFromXStorageTables(fact.getStorageTables()));
LOG.info("Updated fact table " + fact.getName());
} catch (HiveException e) {
throw new LensException(e);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/20949e03/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 933714c..7795c73 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -51,6 +51,7 @@ import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -887,7 +888,6 @@ public class TestMetastoreService extends LensJerseyTest {
XStorageTableDesc xs1 = cubeObjectFactory.createXStorageTableDesc();
xs1.setCollectionDelimiter(",");
xs1.setEscapeChar("\\");
- xs1.setFieldDelimiter("");
xs1.setFieldDelimiter("\t");
xs1.setLineDelimiter("\n");
xs1.setMapKeyDelimiter("\r");
@@ -1274,6 +1274,46 @@ public class TestMetastoreService extends LensJerseyTest {
Map<String, String> updProps = JAXBUtils.mapFromXProperties(dt3.getProperties());
assertEquals(updProps.get("foodim"), "bardim1");
+ // Update storage tables
+ dt3.getStorageTables().getStorageTable().get(0).getTableDesc().setFieldDelimiter(":");
+ dt3.getStorageTables().getStorageTable().get(0).getTableDesc().setInputFormat(
+ SequenceFileInputFormat.class.getCanonicalName());
+ // add one more storage table
+ createStorage("testAlterDimStorage");
+ XStorageTableElement newStorage = createStorageTblElement("testAlterDimStorage", dt3.getTableName(),
+ (String[])null);
+ newStorage.getTableDesc().setFieldDelimiter(":");
+ dt3.getStorageTables().getStorageTable().add(newStorage);
+ // Update the table
+ result = target().path("metastore/dimtables")
+ .path(table)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .put(Entity.xml(cubeObjectFactory.createXDimensionTable(dt3)), APIResult.class);
+ assertEquals(result.getStatus(), Status.SUCCEEDED);
+
+ // Get the updated table
+ JAXBElement<XDimensionTable> dtElement4 = target().path("metastore/dimtables").path(table)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .get(new GenericType<JAXBElement<XDimensionTable>>() {});
+ XDimensionTable dt4 = dtElement4.getValue();
+ assertEquals(dt4.getStorageTables().getStorageTable().size(), 2);
+
+ WebTarget nativeTarget = target().path("metastore").path("nativetables");
+ // get all native tables
+ StringList nativetables = nativeTarget.queryParam("sessionid", lensSessionId).request(mediaType).get(
+ StringList.class);
+ assertTrue(nativetables.getElements().contains("test_" + table));
+ assertTrue(nativetables.getElements().contains("testalterdimstorage_" + table));
+
+ // get native table and validate altered property
+ XNativeTable newdNativeTable = nativeTarget.path("testalterdimstorage_" + table)
+ .queryParam("sessionid", lensSessionId)
+ .request(mediaType).get(XNativeTable.class);
+ assertEquals(newdNativeTable.getStorageDescriptor().getFieldDelimiter(), ":");
+ XNativeTable alteredNativeTable = nativeTarget.path("test_" + table).queryParam("sessionid", lensSessionId)
+ .request(mediaType).get(XNativeTable.class);
+ assertEquals(alteredNativeTable.getStorageDescriptor().getInputFormat(),
+ SequenceFileInputFormat.class.getCanonicalName());
// Drop table
result =
target().path("metastore/dimtables").path(table)
@@ -1526,13 +1566,77 @@ public class TestMetastoreService extends LensJerseyTest {
assertTrue(cf.getUpdatePeriods().get("S1").contains(UpdatePeriod.HOURLY));
assertTrue(cf.getUpdatePeriods().get("S2").contains(UpdatePeriod.DAILY));
+ // Finally, drop the fact table
+ result = target().path("metastore").path("facts").path(table)
+ .queryParam("cascade", "true")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .delete(APIResult.class);
+
+ assertEquals(result.getStatus(), Status.SUCCEEDED);
+
+ // Drop again, this time it should give a 404
+ try {
+ result = target().path("metastore").path("facts").path(table)
+ .queryParam("cascade", "true")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .delete(APIResult.class);
+ fail("Expected 404");
+ } catch (NotFoundException nfe) {
+ // PASS
+ }
+ } finally {
+ setCurrentDatabase(prevDb);
+ dropDatabase(DB);
+ }
+ }
+
+ @Test
+ public void testUpdateFactTable() throws Exception {
+ final String table = "testUpdateFactTable";
+ final String DB = dbPFX + "testUpdateFactTable_DB";
+ String prevDb = getCurrentDatabase();
+ createDatabase(DB);
+ setCurrentDatabase(DB);
+ createStorage("S1");
+ createStorage("S2");
+ createStorage("S3");
+ try {
+
+ XFactTable f = createFactTable(table);
+ f.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY"));
+ f.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "DAILY"));
+ final FormDataMultiPart mp = new FormDataMultiPart();
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(),
+ lensSessionId, medType));
+ mp.bodyPart(new FormDataBodyPart(
+ FormDataContentDisposition.name("fact").fileName("fact").build(),
+ cubeObjectFactory.createXFactTable(f), medType));
+ APIResult result = target()
+ .path("metastore")
+ .path("facts")
+ .request(mediaType)
+ .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+
+ // Get the created table
+ JAXBElement<XFactTable> gotFactElement = target().path("metastore/facts").path(table)
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .get(new GenericType<JAXBElement<XFactTable>>() {});
+ XFactTable gotFact = gotFactElement.getValue();
+ assertTrue(gotFact.getName().equalsIgnoreCase(table));
+ assertEquals(gotFact.getWeight(), 10.0);
+ CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact);
+
// Do some changes to test update
cf.alterWeight(20.0);
cf.alterColumn(new FieldSchema("c2", "int", "changed to int"));
XFactTable update = JAXBUtils.factTableFromCubeFactTable(cf);
- update.getStorageTables().getStorageTable().add(createStorageTblElement("S1", table, "HOURLY"));
+ XStorageTableElement s1Tbl = createStorageTblElement("S1", table, "HOURLY");
+ s1Tbl.getTableDesc().setFieldDelimiter("#");
+ update.getStorageTables().getStorageTable().add(s1Tbl);
update.getStorageTables().getStorageTable().add(createStorageTblElement("S2", table, "MONTHLY"));
+ update.getStorageTables().getStorageTable().add(createStorageTblElement("S3", table, "DAILY"));
// Update
result = target().path("metastore").path("facts").path(table)
@@ -1549,6 +1653,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(ucf.weight(), 20.0);
assertTrue(ucf.getUpdatePeriods().get("S2").contains(UpdatePeriod.MONTHLY));
+ assertTrue(ucf.getUpdatePeriods().get("S3").contains(UpdatePeriod.DAILY));
boolean foundC2 = false;
for (FieldSchema fs : cf.getColumns()) {
@@ -1559,6 +1664,19 @@ public class TestMetastoreService extends LensJerseyTest {
}
assertTrue(foundC2);
+ WebTarget nativeTarget = target().path("metastore").path("nativetables");
+ // get all native tables
+ StringList nativetables = nativeTarget.queryParam("sessionid", lensSessionId).request(mediaType).get(
+ StringList.class);
+ assertTrue(nativetables.getElements().contains("s1_" + table.toLowerCase()));
+ assertTrue(nativetables.getElements().contains("s2_" + table.toLowerCase()));
+ assertTrue(nativetables.getElements().contains("s3_" + table.toLowerCase()));
+
+ // get native table and validate altered property
+ XNativeTable alteredNativeTable = nativeTarget.path("s1_" + table).queryParam("sessionid", lensSessionId)
+ .request(mediaType).get(XNativeTable.class);
+ assertEquals(alteredNativeTable.getStorageDescriptor().getFieldDelimiter(), "#");
+
// Finally, drop the fact table
result = target().path("metastore").path("facts").path(table)
.queryParam("cascade", "true")