You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by sh...@apache.org on 2015/07/29 06:57:10 UTC
[23/50] [abbrv] incubator-lens git commit: LENS-652 : Add Update
partition api and cli command
LENS-652 : Add Update partition api and cli command
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/7de74eb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/7de74eb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/7de74eb3
Branch: refs/heads/LENS-581
Commit: 7de74eb3abc01d9e5c1afe55f97b309328d81d9e
Parents: d8e071c
Author: Rajat Khadelwal <pr...@apache.org>
Authored: Tue Jul 21 14:07:29 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jul 21 14:07:29 2015 +0530
----------------------------------------------------------------------
lens-api/pom.xml | 2 +
.../lens/cli/commands/BaseLensCommand.java | 2 +-
.../commands/LensDimensionTableCommands.java | 35 +-
.../lens/cli/commands/LensFactCommands.java | 42 ++
.../cli/commands/PhysicalTableCrudCommand.java | 15 +
.../apache/lens/cli/LensCliApplicationTest.java | 2 +
.../cli/TestLensDimensionTableCommands.java | 107 ++--
.../apache/lens/cli/TestLensFactCommands.java | 42 +-
.../src/test/resources/dim1-local-parts.xml | 5 +-
.../src/test/resources/fact1-local-parts.xml | 4 +-
.../java/org/apache/lens/client/LensClient.java | 31 ++
.../apache/lens/client/LensMetadataClient.java | 95 ++++
.../lens/cube/metadata/CubeMetastoreClient.java | 107 +++-
.../apache/lens/cube/metadata/HDFSStorage.java | 16 +
.../lens/cube/metadata/MetastoreUtil.java | 6 +-
.../lens/cube/metadata/PartitionMetahook.java | 29 +-
.../org/apache/lens/cube/metadata/Storage.java | 47 +-
.../lens/cube/metadata/TimePartition.java | 20 +-
.../apache/lens/cube/metadata/UpdatePeriod.java | 19 +
.../lens/cube/parse/StorageTableResolver.java | 7 +-
.../cube/metadata/TestCubeMetastoreClient.java | 538 ++++++++++---------
.../apache/lens/cube/parse/CubeTestSetup.java | 10 +-
.../src/main/resources/hivedriver-default.xml | 27 +-
.../api/metastore/CubeMetastoreService.java | 21 +-
.../metastore/CubeMetastoreServiceImpl.java | 69 ++-
.../apache/lens/server/metastore/JAXBUtils.java | 65 ++-
.../server/metastore/MetastoreResource.java | 116 +++-
.../server/metastore/TestMetastoreService.java | 109 ++--
.../org/apache/lens/storage/db/DBStorage.java | 16 +
.../apache/lens/storage/db/TestDBStorage.java | 2 +-
src/site/apt/admin/hivedriver-config.apt | 12 +-
src/site/apt/user/cli.apt | 14 +-
32 files changed, 1166 insertions(+), 466 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index 507dd01..e1b763d 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -89,6 +89,8 @@
<arg>-extension</arg>
<arg>-Xinheritance</arg>
<arg>-XtoString</arg>
+ <arg>-Xequals</arg>
+ <arg>-XhashCode</arg>
</args>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index 74b4e91..bdfae24 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -143,7 +143,7 @@ public class BaseLensCommand implements ExecutionProcessor {
.replaceAll("]", "\n").replaceAll(",", "").replaceAll("\"", "").replaceAll("\n\n", "\n");
}
- public String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) {
+ public static String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) {
path = path.replaceAll("/$", "");
if (path.startsWith("~")) {
path = path.replaceFirst("~", System.getProperty("user.home"));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
index 6a93393..5fd80a1 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.util.List;
import org.apache.lens.api.APIResult;
@@ -31,6 +32,8 @@ import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
+import lombok.NonNull;
+
/**
* The Class LensDimensionTableCommands.
*/
@@ -189,7 +192,7 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliCommand(value = "dimtable list partitions",
help = "get all partitions associated with dimtable <dimtable_name>, "
+ "storage <storage_name> filtered by <partition-filter>")
- public String getAllPartitionsOfDim(
+ public String getAllPartitionsOfDimtable(
@CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
@CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
@CliOption(key = {"", "filter"}, mandatory = false, help = "<partition-filter>") String filter) {
@@ -231,6 +234,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") String path) {
return addPartition(tableName, storageName, path);
}
+ @CliCommand(value = "dimtable update single-partition",
+ help = "update single partition to dimtable <dimtable_name>'s"
+ + " storage <storage_name>, reading spec from <partition-spec-path>"
+ + " The partition has to exist to be eligible for updation.")
+ public String updatePartitionOfDimtable(
+ @CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") @NonNull final File path) {
+ return updatePartition(tableName, storageName, path);
+ }
/**
* Adds the partitions to dim table.
@@ -250,6 +263,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
return addPartitions(tableName, storageName, path);
}
+ @CliCommand(value = "dimtable update partitions",
+ help = "update multiple partition to dimtable <dimtable_name>'s"
+ + " storage <storage_name>, reading partition list spec from <partition-list-spec-path>"
+ +" The partitions have to exist to be eligible for updation.")
+ public String updatePartitionsOfDimtable(
+ @CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
+ return updatePartitions(tableName, storageName, path);
+ }
@Override
protected XStorageTableElement readStorage(String tableName, String storage) {
@@ -298,6 +321,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
}
@Override
+ protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionOfDim(tableName, storageName, validPath);
+ }
+
+ @Override
+ protected APIResult doUpdatePartitions(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionsOfDim(tableName, storageName, validPath);
+ }
+
+ @Override
public List<String> getAll() {
return getClient().getAllDimensionTables();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index bdb9c38..5493dba 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.util.List;
import org.apache.lens.api.APIResult;
@@ -30,6 +31,8 @@ import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
+import lombok.NonNull;
+
/**
* The Class LensFactCommands.
*/
@@ -226,6 +229,24 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") String path) {
return addPartition(tableName, storageName, path);
}
+ /**
+ * updates the partition in the fact.
+ *
+ * @param tableName fact name
+ * @param storageName storage name
+ * @param path partition spec path
+ * @return the string
+ */
+ @CliCommand(value = "fact update single-partition",
+ help = "update single partition to fact <fact_name>'s"
+ + " storage <storage_name>, reading spec from <partition-spec-path>"
+ + " The partition has to exist to be eligible for updation.")
+ public String updatePartitionOfFact(
+ @CliOption(key = {"", "fact_name"}, mandatory = true, help = "<fact_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") @NonNull final File path) {
+ return updatePartition(tableName, storageName, path);
+ }
@CliCommand(value = "fact add partitions",
help = "add multiple partition to fact <fact_name>'s"
@@ -237,6 +258,17 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
return addPartitions(tableName, storageName, path);
}
+ @CliCommand(value = "fact update partitions",
+ help = "update multiple partition of fact <fact_name>'s"
+ + " storage <storage_name>, reading partition list spec from <partition-list-spec-path>"
+ +" The partitions have to exist to be eligible for updation.")
+ public String updatePartitionsOfFact(
+ @CliOption(key = {"", "fact_name"}, mandatory = true, help = "<fact_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
+ return updatePartitions(tableName, storageName, path);
+ }
+
@CliCommand(value = "fact timelines",
help = "get timelines for fact. Can optionally specify storage, update period and time dimension to filter by."
+ " Instead of time dimension, partition column can be directly passed as <time_dimension>")
@@ -294,6 +326,16 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
}
@Override
+ protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionOfFact(tableName, storageName, validPath);
+ }
+
+ @Override
+ protected APIResult doUpdatePartitions(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionsOfFact(tableName, storageName, validPath);
+ }
+
+ @Override
public List<String> getAll() {
return getClient().getAllFactTables();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
index a479c14..c505680 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.io.IOException;
import java.util.List;
@@ -84,10 +85,20 @@ public abstract class PhysicalTableCrudCommand<T> extends LensCRUDCommand<T> {
return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
+ public String updatePartition(String tableName, String storageName, File path) {
+ // TODO: path.getPath() to be changed to path after LENS-634
+ return doUpdatePartition(tableName, storageName, getValidPath(path.getPath(), false, true)).toString()
+ .toLowerCase();
+ }
+
public String addPartitions(String tableName, String storageName, String path) {
return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
+ public String updatePartitions(String tableName, String storageName, String path) {
+ return doUpdatePartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
+ }
+
public String dropPartitions(String tableName, String storageName, String filter) {
return doDropPartitions(tableName, storageName, filter).toString().toLowerCase();
}
@@ -111,4 +122,8 @@ public abstract class PhysicalTableCrudCommand<T> extends LensCRUDCommand<T> {
protected abstract APIResult doAddPartitions(String tableName, String storageName, String path);
protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter);
+
+ protected abstract APIResult doUpdatePartition(String tableName, String storageName, String validPath);
+
+ protected abstract APIResult doUpdatePartitions(String tableName, String storageName, String validPath);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java b/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
index 2de0cad..8647ac3 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
@@ -31,6 +31,8 @@ import org.testng.annotations.BeforeTest;
* The Class LensCliApplicationTest.
*/
public class LensCliApplicationTest extends LensAllApplicationJerseyTest {
+ public static final String SUCCESS_MESSAGE = "<?xml version=\"1.0\" encoding=\"utf-8\" "
+ + "standalone=\"yes\"?><result><status>succeeded</status><message></message></result>";
@Override
protected int getTestPort() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
index a87d0b7..56e962d 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.cli;
+import static org.testng.Assert.*;
+
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
@@ -28,7 +30,6 @@ import org.apache.lens.cli.commands.LensDimensionCommands;
import org.apache.lens.cli.commands.LensDimensionTableCommands;
import org.apache.lens.client.LensClient;
-import org.testng.Assert;
import org.testng.annotations.Test;
import lombok.extern.slf4j.Slf4j;
@@ -104,8 +105,8 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
throws IOException {
LensDimensionTableCommands command = getCommand();
String dimList = command.showDimensionTables(null);
- Assert.assertEquals(dimList, "No dimensiontable found");
- Assert.assertEquals(command.showDimensionTables("test_dim"), "No dimensiontable found for test_dim");
+ assertEquals(dimList, "No dimensiontable found");
+ assertEquals(command.showDimensionTables("test_dim"), "No dimensiontable found for test_dim");
// add local storage before adding fact table
TestLensStorageCommands.addLocalStorage(storageName);
@@ -115,24 +116,24 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.createDimensionTable(new File(dimSpec.toURI()).getAbsolutePath());
} catch (Exception e) {
log.error("Unable to create dimtable", e);
- Assert.fail("Unable to create dimtable" + e.getMessage());
+ fail("Unable to create dimtable" + e.getMessage());
}
dimList = command.showDimensionTables(null);
- Assert.assertEquals(command.showDimensionTables("test_dim"), dimList);
+ assertEquals(command.showDimensionTables("test_dim"), dimList);
try {
- Assert.assertEquals(command.showDimensionTables("blah"), dimList);
- Assert.fail();
+ assertEquals(command.showDimensionTables("blah"), dimList);
+ fail();
} catch (NotFoundException e) {
log.info("blah is not a table", e);
}
try {
- Assert.assertEquals(command.showDimensionTables("dim_table2"), dimList);
- Assert.fail();
+ assertEquals(command.showDimensionTables("dim_table2"), dimList);
+ fail();
} catch (NotFoundException e) {
log.info("dim_table2 is a table, but not a dimension", e);
}
- Assert.assertTrue(dimList.contains(tableName), "dim_table table should be found");
+ assertTrue(dimList.contains(tableName), "dim_table table should be found");
}
/**
@@ -168,13 +169,13 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
String propString = "name : dim2.prop value : d2";
String propString1 = "name : dim2.prop value : d1";
String propString2 = "name : dim2.prop1 value : d2";
- Assert.assertTrue(desc.contains(propString));
+ assertTrue(desc.contains(propString));
command.updateDimensionTable("dim_table2", "/tmp/local-dim1.xml");
desc = command.describeDimensionTable("dim_table2");
log.debug(desc);
- Assert.assertTrue(desc.contains(propString1));
- Assert.assertTrue(desc.contains(propString2));
+ assertTrue(desc.contains(propString1));
+ assertTrue(desc.contains(propString2));
} finally {
newFile.delete();
@@ -189,16 +190,16 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
private static void testDimStorageActions() throws URISyntaxException {
LensDimensionTableCommands command = getCommand();
String result = command.getDimStorages("dim_table2");
- Assert.assertEquals(DIM_LOCAL, result);
+ assertEquals(DIM_LOCAL, result);
command.dropAllDimStorages("dim_table2");
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(result, "No storage found for dim_table2");
+ assertEquals(result, "No storage found for dim_table2");
addLocalStorageToDim();
result = command.getDimStorages("dim_table2");
- Assert.assertNotEquals(result, "No storage found for dim_table2");
+ assertNotEquals(result, "No storage found for dim_table2");
command.dropStorageFromDim("dim_table2", DIM_LOCAL);
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(result, "No storage found for dim_table2");
+ assertEquals(result, "No storage found for dim_table2");
addLocalStorageToDim();
}
@@ -213,46 +214,70 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
URL resource = TestLensDimensionTableCommands.class.getClassLoader().getResource("dim-local-storage-element.xml");
command.addNewDimStorage("dim_table2", new File(resource.toURI()).getAbsolutePath());
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(DIM_LOCAL, result);
+ assertEquals(DIM_LOCAL, result);
result = command.getStorageFromDim("dim_table2", DIM_LOCAL);
String partString = "DAILY";
- Assert.assertTrue(result.contains(partString));
+ assertTrue(result.contains(partString));
}
/**
* Test dim partition actions.
*/
- private static void testDimPartitionActions() {
+ private static void testDimPartitionActions() throws URISyntaxException {
LensDimensionTableCommands command = getCommand();
- String result;
- result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- Assert.assertTrue(result.trim().isEmpty());
+ assertTrue(command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null).trim().isEmpty());
+ //TODO: remove getAbsolutePath()
+ String singlePartPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("dim1-local-part.xml").toURI()).getAbsolutePath();
+ String multiplePartsPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("dim1-local-parts.xml").toURI()).getAbsolutePath();
+
+ assertTrue(command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null).trim().isEmpty());
+
+ assertEquals(command.addPartitionToDimtable("dim_table2", DIM_LOCAL, singlePartPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionOfDimtable("dim_table2", DIM_LOCAL, new File(singlePartPath)), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+ assertEquals(command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionsOfDimtable("dim_table2", DIM_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+
+ // Wrong files:
try {
- command.addPartitionToDimtable("dim_table2", DIM_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("dim1-local-part.xml").toURI()).getAbsolutePath());
+ command.addPartitionToDimtable("dim_table2", DIM_LOCAL, multiplePartsPath);
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ // pass
}
- verifyAndDeletePartition();
try {
- command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("dim1-local-parts.xml").toURI()).getAbsolutePath());
+ command.updatePartitionOfDimtable("dim_table2", DIM_LOCAL, new File(multiplePartsPath));
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ // pass
+ }
+
+ try {
+ command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, singlePartPath);
+ fail("Should fail");
+ } catch (Throwable t) {
+ // pass
+ }
+
+ try {
+ command.updatePartitionsOfDimtable("dim_table2", DIM_LOCAL, singlePartPath);
+ fail("Should fail");
+ } catch (Throwable t) {
+ // pass
}
- verifyAndDeletePartition();
}
- private static void verifyAndDeletePartition() {
- String result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
+ private static void verifyAndDeletePartitions() {
+ String result = command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null);
String partString = "DAILY";
- Assert.assertTrue(result.contains(partString));
+ assertTrue(result.contains(partString));
command.dropAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- Assert.assertTrue(result.trim().isEmpty());
+ result = command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null);
+ assertTrue(result.trim().isEmpty());
}
/**
@@ -269,7 +294,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.addPartitionToDimtable(tableName, storageName, new File(resource.toURI()).getAbsolutePath());
} catch (Throwable t) {
log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
}
}
@@ -279,10 +304,10 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
public static void dropDim1Table() {
LensDimensionTableCommands command = getCommand();
String dimList = command.showDimensionTables(null);
- Assert.assertEquals("dim_table2", dimList, "dim table should be found");
+ assertEquals("dim_table2", dimList, "dim table should be found");
command.dropDimensionTable("dim_table2", false);
dimList = command.showDimensionTables(null);
- Assert.assertEquals(dimList, "No dimensiontable found", "Dim tables should not be found");
+ assertEquals(dimList, "No dimensiontable found", "Dim tables should not be found");
TestLensStorageCommands.dropStorage(DIM_LOCAL);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 13bfbd2..fa342c8 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -222,40 +222,44 @@ public class TestLensFactCommands extends LensCliApplicationTest {
/**
* Test fact partition actions.
*/
- private void testFactPartitionActions() {
+ private void testFactPartitionActions() throws URISyntaxException {
LensFactCommands command = getCommand();
- String result;
verifyEmptyTimelines();
- result = command.getAllPartitionsOfFact("fact1", FACT_LOCAL, null);
- assertTrue(result.trim().isEmpty());
+ assertTrue(command.getAllPartitionsOfFact("fact1", FACT_LOCAL, null).trim().isEmpty());
+ String singlePartPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath();
+ String multiplePartsPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath();
+ assertEquals(command.addPartitionToFact("fact1", FACT_LOCAL, singlePartPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionOfFact("fact1", FACT_LOCAL, new File(singlePartPath)), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+ assertEquals(command.addPartitionsToFact("fact1", FACT_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionsOfFact("fact1", FACT_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+
+ // Wrong files:
try {
- command.addPartitionToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath());
+ command.addPartitionToFact("fact1", FACT_LOCAL, multiplePartsPath);
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
- fail("Unable to locate the storage part file for adding new storage to fact table fact1");
+ // pass
}
- verifyAndDeletePartitions();
try {
- command.addPartitionsToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
+ command.updatePartitionOfFact("fact1", FACT_LOCAL, new File(multiplePartsPath));
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
- fail("Unable to locate the storage part file for adding new storage to fact table fact1");
+ // pass
}
- verifyAndDeletePartitions();
- // Wrong files:
try {
- command.addPartitionToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
+ command.addPartitionsToFact("fact1", FACT_LOCAL, singlePartPath);
fail("Should fail");
} catch (Throwable t) {
// pass
}
+
try {
- command.addPartitionsToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath());
+ command.updatePartitionsOfFact("fact1", FACT_LOCAL, singlePartPath);
fail("Should fail");
} catch (Throwable t) {
// pass
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/resources/dim1-local-parts.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/dim1-local-parts.xml b/lens-cli/src/test/resources/dim1-local-parts.xml
index 87fbc6d..8edc5c7 100644
--- a/lens-cli/src/test/resources/dim1-local-parts.xml
+++ b/lens-cli/src/test/resources/dim1-local-parts.xml
@@ -21,10 +21,7 @@
-->
<x_partition_list xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
-
- <partition fact_or_dimension_table_name="dim_table2" location="/tmp/dim_table2" update_period="DAILY"
- xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <partition fact_or_dimension_table_name="dim_table2" location="/tmp/dim_table2" update_period="DAILY">
<time_partition_spec>
<part_spec_element key="dt" value="2014-03-27T12:00:00"/>
</time_partition_spec>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/resources/fact1-local-parts.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/fact1-local-parts.xml b/lens-cli/src/test/resources/fact1-local-parts.xml
index 6201320..9ca7d1f 100644
--- a/lens-cli/src/test/resources/fact1-local-parts.xml
+++ b/lens-cli/src/test/resources/fact1-local-parts.xml
@@ -21,9 +21,7 @@
-->
<x_partition_list xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
- <partition fact_or_dimension_table_name="fact1" location="/tmp/fact1" update_period="HOURLY"
- xmlns="uri:lens:cube:0.1"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <partition fact_or_dimension_table_name="fact1" location="/tmp/fact1" update_period="HOURLY">
<time_partition_spec>
<part_spec_element key="dt" value="2014-03-27T12:00:00"/>
</time_partition_spec>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index b15b024..d7df1ba 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -530,6 +530,37 @@ public class LensClient {
public APIResult addPartitionsToDim(String table, String storage, String partsSpec) {
return mc.addPartitionsToDimensionTable(table, storage, partsSpec);
}
+ public APIResult updatePartitionOfFact(String table, String storage, String partSpec) {
+ return mc.updatePartitionOfFactTable(table, storage, partSpec);
+ }
+
+ public APIResult updatePartitionsOfFact(String table, String storage, String partsSpec) {
+ return mc.updatePartitionsOfFactTable(table, storage, partsSpec);
+ }
+
+ public APIResult updatePartitionOfFact(String table, String storage, XPartition xp) {
+ return mc.updatePartitionOfFactTable(table, storage, xp);
+ }
+
+ public APIResult updatePartitionsOfFact(String table, String storage, XPartitionList xpList) {
+ return mc.updatePartitionsOfFactTable(table, storage, xpList);
+ }
+
+ public APIResult updatePartitionOfDim(String table, String storage, String partSpec) {
+ return mc.updatePartitionOfDimensionTable(table, storage, partSpec);
+ }
+
+ public APIResult updatePartitionOfDim(String table, String storage, XPartition xp) {
+ return mc.updatePartitionOfDimensionTable(table, storage, xp);
+ }
+
+ public APIResult updatePartitionsOfDim(String table, String storage, XPartitionList xpList) {
+ return mc.updatePartitionsOfDimensionTable(table, storage, xpList);
+ }
+
+ public APIResult updatePartitionsOfDim(String table, String storage, String partsSpec) {
+ return mc.updatePartitionsOfDimensionTable(table, storage, partsSpec);
+ }
public QueryPrepareHandle prepare(String sql, String queryName) {
return statement.prepareQuery(sql, queryName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index b4e0552..dba13ca 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -967,6 +967,101 @@ public class LensMetadataClient {
return new APIResult(Status.FAILED, "File not found");
}
}
+ public APIResult updatePartitionOfDimensionTable(String dimTblName, String storage,
+ XPartition partition) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("dimtables").path(dimTblName)
+ .path("storages").path(storage).path("partition")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartition(partition)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionOfDimensionTable(String dimTblName, String storage,
+ String partitionSpec) {
+ try {
+ return updatePartitionOfDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionsOfDimensionTable(String dimTblName, String storage,
+ XPartitionList partitions) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("dimtables").path(dimTblName)
+ .path("storages").path(storage).path("partitions")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartitionList(partitions)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionsOfDimensionTable(String dimTblName, String storage,
+ String partitionsSpec) {
+ try {
+ return updatePartitionsOfDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionOfFactTable(String fact, String storage,
+ XPartition partition) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("facts").path(fact)
+ .path("storages").path(storage).path("partition")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartition(partition)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionOfFactTable(String fact, String storage,
+ String partitionSpec) {
+ try {
+ return updatePartitionOfFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionsOfFactTable(String fact, String storage,
+ XPartitionList partitions) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("facts").path(fact)
+ .path("storages").path(storage).path("partitions")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartitionList(partitions)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionsOfFactTable(String fact, String storage,
+ String partitionsSpec) {
+ try {
+ return updatePartitionsOfFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
public Date getLatestDateOfCube(String cubeName, String timePartition) {
return getMetastoreWebTarget().path("cubes").path(cubeName).path("latestdate")
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index dc2d9f6..c8abca0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -34,10 +34,8 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
+import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.thrift.TException;
@@ -172,6 +170,67 @@ public class CubeMetastoreClient {
return ret;
}
+ public void updatePartition(String fact, String storageName, Partition partition)
+ throws HiveException, InvalidOperationException {
+ updatePartitions(fact, storageName, Collections.singletonList(partition));
+ }
+
+ public void updatePartitions(String factOrDimtableName, String storageName, List<Partition> partitions)
+ throws HiveException, InvalidOperationException {
+ List<Partition> partitionsToAlter = Lists.newArrayList();
+ partitionsToAlter.addAll(partitions);
+ partitionsToAlter.addAll(getAllLatestPartsEquivalentTo(factOrDimtableName, storageName, partitions));
+ getStorage(storageName).updatePartitions(getClient(), factOrDimtableName, partitionsToAlter);
+ }
+
+ private List<Partition> getAllLatestPartsEquivalentTo(String factOrDimtableName, String storageName,
+ List<Partition> partitions) throws HiveException {
+ if (isFactTable(factOrDimtableName)) {
+ return Lists.newArrayList();
+ }
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimtableName, storageName);
+ Table storageTable = getTable(storageTableName);
+ List<String> timePartCols = getTimePartColNamesOfTable(storageTable);
+ List<Partition> latestParts = Lists.newArrayList();
+ for (Partition partition : partitions) {
+ LinkedHashMap<String, String> partSpec = partition.getSpec();
+ LinkedHashMap<String, String> timePartSpec = Maps.newLinkedHashMap();
+ LinkedHashMap<String, String> nonTimePartSpec = Maps.newLinkedHashMap();
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (timePartCols.contains(entry.getKey())) {
+ timePartSpec.put(entry.getKey(), entry.getValue());
+ } else {
+ nonTimePartSpec.put(entry.getKey(), entry.getValue());
+ }
+ }
+ for (String timePartCol : timePartCols) {
+ Partition latestPart = getLatestPart(storageTableName, timePartCol, nonTimePartSpec);
+ if (latestPart != null) {
+ LinkedHashMap<String, String> latestPartSpec = latestPart.getSpec();
+ latestPartSpec.put(timePartCol, partSpec.get(timePartCol));
+ if (partSpec.equals(latestPartSpec)) {
+ latestPart.getParameters().putAll(partition.getParameters());
+ latestPart.getParameters().put(MetastoreUtil.getLatestPartTimestampKey(timePartCol),
+ partSpec.get(timePartCol));
+ latestPart.getTPartition().getSd().getSerdeInfo().getParameters().putAll(
+ partition.getTPartition().getSd().getSerdeInfo().getParameters());
+ latestPart.setLocation(partition.getLocation());
+ latestPart.setInputFormatClass(partition.getInputFormatClass());
+ latestPart.setOutputFormatClass(partition.getOutputFormatClass());
+ // the following is a fix because hive has a bug: https://issues.apache.org/jira/browse/HIVE-11278.
+ latestPart.getTPartition().getSd().setOutputFormat(
+ HiveFileFormatUtils.getOutputFormatSubstitute(partition.getOutputFormatClass(), false).getName());
+ latestPart.getTPartition().getSd().getSerdeInfo()
+ .setSerializationLib(partition.getTPartition().getSd().getSerdeInfo().getSerializationLib());
+ latestParts.add(latestPart);
+ }
+ }
+ }
+ }
+ return latestParts;
+ }
+
+
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
* storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
@@ -257,13 +316,13 @@ public class CubeMetastoreClient {
if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
storage) != null) {
for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTable)) {
+ for (String partCol : getTimePartColNamesOfTable(storageTable)) {
ensureEntry(storageTableName, updatePeriod, partCol);
}
}
}
// Then add all existing partitions for batch addition in respective timelines.
- List<String> timeParts = getTimePartsOfTable(storageTable);
+ List<String> timeParts = getTimePartColNamesOfTable(storageTable);
List<FieldSchema> partCols = storageTable.getPartCols();
for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
UpdatePeriod period = deduceUpdatePeriod(partition);
@@ -290,7 +349,7 @@ public class CubeMetastoreClient {
String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
log.info("loading from table properties: " + storageTableName);
for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTableName)) {
+ for (String partCol : getTimePartColNamesOfTable(storageTableName)) {
ensureEntry(storageTableName, updatePeriod, partCol).init(getTable(storageTableName));
}
}
@@ -723,7 +782,7 @@ public class CubeMetastoreClient {
* @throws HiveException
*/
public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException, LensException {
- addPartitions(Arrays.asList(partSpec), storageName);
+ addPartitions(Collections.singletonList(partSpec), storageName);
}
/** batch addition */
@@ -731,9 +790,9 @@ public class CubeMetastoreClient {
throws HiveException, LensException {
for (Map.Entry<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> group : groupPartitionDescs(
storagePartitionDescs).entrySet()) {
- String fact = group.getKey();
+ String factOrDimtable = group.getKey();
for (Map.Entry<UpdatePeriod, List<StoragePartitionDesc>> entry : group.getValue().entrySet()) {
- addPartitions(fact, storageName, entry.getKey(), entry.getValue());
+ addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue());
}
}
}
@@ -1052,7 +1111,7 @@ public class CubeMetastoreClient {
public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName);
return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec);
}
@@ -1086,6 +1145,15 @@ public class CubeMetastoreClient {
return getClient().getPartitions(getHiveTable(storageTableName));
}
+ public Partition getPartitionByFilter(String storageTableName, String filter) throws HiveException {
+ List<Partition> parts = getPartitionsByFilter(storageTableName, filter);
+ if (parts.size() != 1) {
+ throw new HiveException(
+ "filter " + filter + " did not result in unique partition. Got " + parts.size() + "partitions");
+ }
+ return parts.iterator().next();
+ }
+
public List<Partition> getPartitionsByFilter(String storageTableName, String filter) throws HiveException {
try {
return getClient().getPartitionsByFilter(getTable(storageTableName), filter);
@@ -1116,7 +1184,7 @@ public class CubeMetastoreClient {
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps)
throws HiveException {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTblName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName);
return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName),
partitionTimestamps);
}
@@ -1164,12 +1232,12 @@ public class CubeMetastoreClient {
return getTable(tableName);
}
- private List<String> getTimePartsOfTable(String storageTableName) throws HiveException {
- return getTimePartsOfTable(getTable(storageTableName));
+ public List<String> getTimePartColNamesOfTable(String storageTableName) throws HiveException {
+ return getTimePartColNamesOfTable(getTable(storageTableName));
}
/** extract from table properties */
- public List<String> getTimePartsOfTable(Table table) {
+ public List<String> getTimePartColNamesOfTable(Table table) {
List<String> ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS),
","));
return ret == null ? new ArrayList<String>() : ret;
@@ -1380,7 +1448,8 @@ public class CubeMetastoreClient {
if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) {
for (String storageName : dimTable.getStorages()) {
if (dimTable.hasStorageSnapshots(storageName)) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTable.getName(), storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(),
+ storageName);
if (dimLatestPartitionExists(storageTableName,
getDimension(dimTable.getDimName()).getTimedDimension())) {
latestLookupCache.add(storageTableName.trim().toLowerCase());
@@ -1976,7 +2045,7 @@ public class CubeMetastoreClient {
public void dropStorageFromFact(String factName, String storage) throws HiveException {
CubeFactTable cft = getFactTable(factName);
cft.dropStorage(storage);
- dropHiveTable(MetastoreUtil.getFactStorageTableName(factName, storage));
+ dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
alterCubeTable(factName, getTable(factName), cft);
updateFactCache(factName);
}
@@ -1984,7 +2053,7 @@ public class CubeMetastoreClient {
// updateFact will be false when fact is fully dropped
private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws HiveException {
CubeFactTable cft = getFactTable(factName);
- dropHiveTable(MetastoreUtil.getFactStorageTableName(factName, storage));
+ dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
if (updateFact) {
cft.dropStorage(storage);
alterCubeTable(factName, getTable(factName), cft);
@@ -2006,7 +2075,7 @@ public class CubeMetastoreClient {
// updateDimTbl will be false when dropping dimTbl
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws HiveException {
CubeDimensionTable cdt = getDimensionTable(dimTblName);
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTblName, storage);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storage);
dropHiveTable(storageTableName);
latestLookupCache.remove(storageTableName.trim().toLowerCase());
if (updateDimTbl) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
index 8107f4c..c4c3e9b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
public class HDFSStorage extends Storage {
@@ -52,6 +53,21 @@ public class HDFSStorage extends Storage {
}
@Override
+ public void preUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void commitUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void rollbackUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
public void preDropPartition(String storageTableName, List<String> partVals) throws HiveException {
// No op
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 8e2c9ed..8fbfe63 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -34,14 +34,10 @@ public class MetastoreUtil {
}
- public static final String getFactStorageTableName(String factName, String storageName) {
+ public static final String getFactOrDimtableStorageTableName(String factName, String storageName) {
return getStorageTableName(factName, Storage.getPrefix(storageName));
}
- public static final String getDimStorageTableName(String dimName, String storageName) {
- return getStorageTableName(dimName, Storage.getPrefix(storageName));
- }
-
public static final String getStorageTableName(String cubeTableName, String storagePrefix) {
return storagePrefix + cubeTableName;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
index 1174d25..317c673 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
@@ -21,11 +21,12 @@ package org.apache.lens.cube.metadata;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
public interface PartitionMetahook {
/**
- * Called before calling add partition
+ * Called before calling add partition(s)
*
* @param storagePartitionDesc
* @throws HiveException
@@ -33,7 +34,7 @@ public interface PartitionMetahook {
void preAddPartitions(List<StoragePartitionDesc> storagePartitionDesc) throws HiveException;
/**
- * Called after successfully adding the partition
+ * Called after successfully adding the partition(s)
*
* @param storagePartitionDesc
* @throws HiveException
@@ -49,6 +50,30 @@ public interface PartitionMetahook {
void rollbackAddPartitions(List<StoragePartitionDesc> storagePartitionDesc) throws HiveException;
/**
+ * Called before calling update partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void preUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
+ * Called after successfully updating partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void commitUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
+ * Called after failed updating partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void rollbackUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
* Called before calling drop partition
*
* @param storageTableName
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index c754042..873aa49 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -28,6 +28,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.metadata.*;
@@ -348,16 +349,53 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
}
}
client.createPartitions(addParts);
- commitAddPartitions(storagePartitionDescs);
success = true;
} finally {
- if (!success) {
+ if (success) {
+ commitAddPartitions(storagePartitionDescs);
+ } else {
rollbackAddPartitions(storagePartitionDescs);
}
}
}
/**
+ * Update existing partition
+ * @param client hive client instance
+ * @param fact fact name
+ * @param partition partition to be updated
+ * @throws InvalidOperationException
+ * @throws HiveException
+ */
+ public void updatePartition(Hive client, String fact, Partition partition)
+ throws InvalidOperationException, HiveException {
+ client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition);
+ }
+
+ /**
+ * Update existing partitions
+ * @param client hive client instance
+ * @param fact fact name
+ * @param partitions partitions to be updated
+ * @throws InvalidOperationException
+ * @throws HiveException
+ */
+ public void updatePartitions(Hive client, String fact, List<Partition> partitions)
+ throws InvalidOperationException, HiveException {
+ boolean success = false;
+ try {
+ client.alterPartitions(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partitions);
+ success = true;
+ } finally {
+ if (success) {
+ commitUpdatePartition(partitions);
+ } else {
+ rollbackUpdatePartition(partitions);
+ }
+ }
+ }
+
+ /**
* Drop the partition in the underlying hive table and update latest partition link
*
* @param client The metastore client
@@ -413,10 +451,11 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
}
}
}
- commitDropPartition(storageTableName, partVals);
success = true;
} finally {
- if (!success) {
+ if (success) {
+ commitDropPartition(storageTableName, partVals);
+ } else {
rollbackDropPartition(storageTableName, partVals);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index 7c966d8..11e6ef6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -24,8 +24,6 @@ import java.util.Date;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.lang3.time.DateUtils;
-
import lombok.Data;
import lombok.NonNull;
@@ -41,7 +39,7 @@ public class TimePartition implements Comparable<TimePartition>, Named {
this.updatePeriod = updatePeriod;
Calendar cal = Calendar.getInstance();
cal.setTime(date);
- this.date = truncate(date, updatePeriod);
+ this.date = updatePeriod.truncate(date);
this.dateString = updatePeriod.format().format(this.date);
}
@@ -109,22 +107,6 @@ public class TimePartition implements Comparable<TimePartition>, Named {
return this.date.after(when.date);
}
- private Date truncate(Date date, UpdatePeriod updatePeriod) {
- if (updatePeriod.equals(UpdatePeriod.WEEKLY)) {
- Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
- Calendar cal = Calendar.getInstance();
- cal.setTime(truncDate);
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
- return cal.getTime();
- } else if (updatePeriod.equals(UpdatePeriod.QUARTERLY)) {
- Date dt = DateUtils.truncate(date, updatePeriod.calendarField());
- dt.setMonth(dt.getMonth() - dt.getMonth() % 3);
- return dt;
- } else {
- return DateUtils.truncate(date, updatePeriod.calendarField());
- }
- }
-
protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) {
return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index 966dbdf..253323b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -23,11 +23,14 @@ import static java.util.Calendar.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.util.Calendar;
import java.util.Comparator;
import java.util.Date;
import org.apache.lens.cube.parse.DateUtil;
+import org.apache.commons.lang3.time.DateUtils;
+
public enum UpdatePeriod implements Named {
SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"),
MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"),
@@ -214,6 +217,22 @@ public enum UpdatePeriod implements Named {
return normalizationFactor;
}
+ public Date truncate(Date date) {
+ if (this.equals(UpdatePeriod.WEEKLY)) {
+ Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(truncDate);
+ cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+ return cal.getTime();
+ } else if (this.equals(UpdatePeriod.QUARTERLY)) {
+ Date dt = DateUtils.truncate(date, this.calendarField());
+ dt.setMonth(dt.getMonth() - dt.getMonth() % 3);
+ return dt;
+ } else {
+ return DateUtils.truncate(date, this.calendarField());
+ }
+ }
+
public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> {
@Override
public int compare(UpdatePeriod o1, UpdatePeriod o2) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index ce6f434..373c80a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST;
@@ -182,7 +183,7 @@ class StorageTableResolver implements ContextRewriter {
Map<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>();
for (String storage : dimtable.getStorages()) {
if (isStorageSupported(storage)) {
- String tableName = MetastoreUtil.getDimStorageTableName(dimtable.getName(), storage).toLowerCase();
+ String tableName = getFactOrDimtableStorageTableName(dimtable.getName(), storage).toLowerCase();
if (validDimTables != null && !validDimTables.contains(tableName)) {
LOG.info("Not considering dim storage table:" + tableName + " as it is not a valid dim storage");
skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.INVALID));
@@ -307,7 +308,7 @@ class StorageTableResolver implements ContextRewriter {
}
String getStorageTableName(CubeFactTable fact, String storage, List<String> validFactStorageTables) {
- String tableName = MetastoreUtil.getFactStorageTableName(fact.getName(), storage).toLowerCase();
+ String tableName = getFactOrDimtableStorageTableName(fact.getName(), storage).toLowerCase();
if (validFactStorageTables != null && !validFactStorageTables.contains(tableName)) {
LOG.info("Skipping storage table " + tableName + " as it is not valid");
return null;
@@ -370,7 +371,7 @@ class StorageTableResolver implements ContextRewriter {
String partCol = range.getPartitionColumn();
boolean partColNotSupported = rangeParts.isEmpty();
for(String storage: cfact.fact.getStorages()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(cfact.fact.getName(), storage).toLowerCase();
+ String storageTableName = getFactOrDimtableStorageTableName(cfact.fact.getName(), storage).toLowerCase();
partColNotSupported &= skipStorageCauses.containsKey(storageTableName)
&& skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST)
&& skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol);