You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by av...@apache.org on 2012/12/21 23:58:15 UTC
svn commit: r1425204 - in /incubator/hcatalog/trunk: ./
webhcat/java-client/src/main/java/org/apache/hcatalog/api/
webhcat/java-client/src/test/java/org/apache/hcatalog/api/
Author: avandana
Date: Fri Dec 21 23:58:15 2012
New Revision: 1425204
URL: http://svn.apache.org/viewvc?rev=1425204&view=rev
Log:
HCAT-567 HCatClient must allow retrieval of multiple partitions using a partial partition spec.
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
incubator/hcatalog/trunk/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1425204&r1=1425203&r2=1425204&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Dec 21 23:58:15 2012
@@ -155,6 +155,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-567 HCatClient must allow retrieval of multiple partitions using a partial partition spec.(mithun via avandana)
+
HCAT-556 HCatalog-trunk doesn't build against Hive-0.10.0 (mithun via avandana)
HCAT-566 HCatTable doesn't report partition columns correctly. (mithun via traviscrawford)
Modified: incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java?rev=1425204&r1=1425203&r2=1425204&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java (original)
+++ incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java Fri Dec 21 23:58:15 2012
@@ -204,11 +204,29 @@ public abstract class HCatClient {
throws HCatException;
/**
+ * Gets all the partitions that match the specified (and possibly partial) partition specification.
+ * A partial partition-specification is one where not all partition-keys have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: getPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would return 1 partition.
+ * 2. Partial partition spec: getPartitions('myDb', 'myTable', {dt='20120101'}) would return all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us', 'uk' and 'in').
+ * @param dbName The name of the database.
+ * @param tblName The name of the table.
+ * @param partitionSpec The partition specification. (Need not include all partition keys.)
+ * @return A list of partitions.
+ * @throws HCatException
+ */
+ public abstract List<HCatPartition> getPartitions(String dbName, String tblName, Map<String, String> partitionSpec)
+ throws HCatException;
+
+ /**
* Gets the partition.
*
* @param dbName The database name.
* @param tableName The table name.
- * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}.
+ * @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}. All partition-key-values
+ * must be specified.
* @return An instance of HCatPartitionInfo.
* @throws HCatException
*/
@@ -235,16 +253,21 @@ public abstract class HCatClient {
throws HCatException;
/**
- * Drops partition.
- *
+ * Drops partition(s) that match the specified (and possibly partial) partition specification.
+ * A partial partition-specification is one where not all partition-keys have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions ('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: dropPartitions('myDb', 'myTable', {dt='20120101', region='us'}) would drop 1 partition.
+ * 2. Partial partition spec: dropPartitions('myDb', 'myTable', {dt='20120101'}) would drop all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us', 'uk' and 'in').
* @param dbName The database name.
* @param tableName The table name.
* @param partitionSpec The partition specification, {[col_name,value],[col_name2,value2]}.
* @param ifExists Hive returns an error if the partition specified does not exist, unless ifExists is set to true.
* @throws HCatException,ConnectionFailureException
*/
- public abstract void dropPartition(String dbName, String tableName,
- Map<String, String> partitionSpec, boolean ifExists)
+ public abstract void dropPartitions(String dbName, String tableName,
+ Map<String, String> partitionSpec, boolean ifExists)
throws HCatException;
/**
Modified: incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java?rev=1425204&r1=1425203&r2=1425204&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java (original)
+++ incubator/hcatalog/trunk/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java Fri Dec 21 23:58:15 2012
@@ -326,12 +326,45 @@ public class HCatClientHMSImpl extends H
}
@Override
+ public List<HCatPartition> getPartitions(String dbName, String tblName, Map<String, String> partitionSpec) throws HCatException {
+ return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec));
+ }
+
+ private static String getFilterString(Map<String, String> partitionSpec) {
+ final String AND = " AND ";
+
+ StringBuilder filter = new StringBuilder();
+ for (Map.Entry<String, String> entry : partitionSpec.entrySet()) {
+ filter.append(entry.getKey()).append("=").append("\"").append(entry.getValue()).append("\"").append(AND);
+ }
+
+ int length = filter.toString().length();
+ if (length > 0)
+ filter.delete(length - AND.length(), length);
+
+ return filter.toString();
+ }
+
+ @Override
public HCatPartition getPartition(String dbName, String tableName,
Map<String, String> partitionSpec) throws HCatException {
HCatPartition partition = null;
try {
+ List<HCatFieldSchema> partitionColumns = getTable(checkDB(dbName), tableName).getPartCols();
+ if (partitionColumns.size() != partitionSpec.size()) {
+ throw new HCatException("Partition-spec doesn't have the right number of partition keys.");
+ }
+
ArrayList<String> ptnValues = new ArrayList<String>();
- ptnValues.addAll(partitionSpec.values());
+ for (HCatFieldSchema partitionColumn : partitionColumns) {
+ String partKey = partitionColumn.getName();
+ if (partitionSpec.containsKey(partKey)) {
+ ptnValues.add(partitionSpec.get(partKey)); // Partition-keys added in order.
+ }
+ else {
+ throw new HCatException("Invalid partition-key specified: " + partKey);
+ }
+ }
Partition hivePartition = hmsClient.getPartition(checkDB(dbName),
tableName, ptnValues);
if (hivePartition != null) {
@@ -382,19 +415,22 @@ public class HCatClientHMSImpl extends H
}
@Override
- public void dropPartition(String dbName, String tableName,
- Map<String, String> partitionSpec, boolean ifExists)
+ public void dropPartitions(String dbName, String tableName,
+ Map<String, String> partitionSpec, boolean ifExists)
throws HCatException {
try {
- List<String> ptnValues = new ArrayList<String>();
- ptnValues.addAll(partitionSpec.values());
- hmsClient.dropPartition(checkDB(dbName), tableName, ptnValues,
- ifExists);
- } catch (NoSuchObjectException e) {
- if (!ifExists) {
- throw new ObjectNotFoundException(
- "NoSuchObjectException while dropping partition.", e);
+ dbName = checkDB(dbName);
+ List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
+ getFilterString(partitionSpec), (short)-1);
+
+ for (Partition partition : partitions) {
+ dropPartition(partition, ifExists);
}
+
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition. " +
+ "Either db(" + dbName + ") or table(" + tableName + ") missing.", e);
} catch (MetaException e) {
throw new HCatException("MetaException while dropping partition.",
e);
@@ -404,6 +440,18 @@ public class HCatClientHMSImpl extends H
}
}
+ private void dropPartition(Partition partition, boolean ifExists)
+ throws HCatException, MetaException, TException {
+ try {
+ hmsClient.dropPartition(partition.getDbName(), partition.getTableName(), partition.getValues());
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition: " + partition.getValues(), e);
+ }
+ }
+ }
+
@Override
public List<HCatPartition> listPartitionsByFilter(String dbName,
String tblName, String filter) throws HCatException {
Modified: incubator/hcatalog/trunk/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1425204&r1=1425203&r2=1425204&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (original)
+++ incubator/hcatalog/trunk/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java Fri Dec 21 23:58:15 2012
@@ -215,7 +215,7 @@ public class TestHCatClient {
HCatPartition ptn = client.getPartition(dbName, tableName, firstPtn);
assertTrue(ptn != null);
- client.dropPartition(dbName, tableName, firstPtn, true);
+ client.dropPartitions(dbName, tableName, firstPtn, true);
ptnList = client.listPartitionsByFilter(dbName,
tableName, null);
assertTrue(ptnList.size() == 2);
@@ -533,4 +533,103 @@ public class TestHCatClient {
assertTrue("Unexpected exception! " + unexpected.getMessage(), false);
}
}
+
+ @Test
+ public void testGetPartitionsWithPartialSpec() throws Exception {
+ try {
+ HCatClient client = HCatClient.create(new Configuration(hcatConf));
+ final String dbName = "myDb";
+ final String tableName = "myTable";
+
+ client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ client.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ List<HCatFieldSchema> columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""),
+ new HCatFieldSchema("bar", Type.STRING, ""));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""),
+ new HCatFieldSchema("grid", Type.STRING, ""));
+
+ client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList<HCatFieldSchema>(partitionSchema)).build());
+
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2011_12_31");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2012_01_01");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "OB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "XB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+
+ Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("dt", "2012_01_01");
+
+ List<HCatPartition> partitions = client.getPartitions(dbName, tableName, partialPartitionSpec);
+ assertEquals("Unexpected number of partitions.", 3, partitions.size());
+ assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "AB"}, partitions.get(0).getValues().toArray());
+ assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "OB"}, partitions.get(1).getValues().toArray());
+ assertArrayEquals("Mismatched partition.", new String[]{"2012_01_01", "XB"}, partitions.get(2).getValues().toArray());
+
+ client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE);
+ }
+ catch (Exception unexpected) {
+ LOG.error("Unexpected exception!", unexpected);
+ assertTrue("Unexpected exception! " + unexpected.getMessage(), false);
+ }
+ }
+
+ @Test
+ public void testDropPartitionsWithPartialSpec() throws Exception {
+ try {
+ HCatClient client = HCatClient.create(new Configuration(hcatConf));
+ final String dbName = "myDb";
+ final String tableName = "myTable";
+
+ client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ client.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ List<HCatFieldSchema> columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""),
+ new HCatFieldSchema("bar", Type.STRING, ""));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(new HCatFieldSchema("dt", Type.STRING, ""),
+ new HCatFieldSchema("grid", Type.STRING, ""));
+
+ client.createTable(HCatCreateTableDesc.create(dbName, tableName, columnSchema).partCols(new ArrayList<HCatFieldSchema>(partitionSchema)).build());
+
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2011_12_31");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2012_01_01");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "OB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "XB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName, "", partitionSpec).build());
+
+ Map<String, String> partialPartitionSpec = new HashMap<String, String>();
+ partialPartitionSpec.put("dt", "2012_01_01");
+
+ client.dropPartitions(dbName, tableName, partialPartitionSpec, true);
+
+ List<HCatPartition> partitions = client.getPartitions(dbName, tableName);
+ assertEquals("Unexpected number of partitions.", 1, partitions.size());
+ assertArrayEquals("Mismatched partition.", new String[]{"2011_12_31", "AB"}, partitions.get(0).getValues().toArray());
+
+ client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE);
+ }
+ catch (Exception unexpected) {
+ LOG.error("Unexpected exception!", unexpected);
+ assertTrue("Unexpected exception! " + unexpected.getMessage(), false);
+ }
+ }
+
}