You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2015/03/02 19:17:03 UTC

svn commit: r1663386 - in /hive/branches/hbase-metastore: itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/ metastore/src/java/org/apache/hadoop/hive/metastore/hbase/ metastore/src/test/org/apache/hadoop/hive/metastore/hbase/

Author: gates
Date: Mon Mar  2 18:17:01 2015
New Revision: 1663386

URL: http://svn.apache.org/r1663386
Log:
HIVE-9641 Fill out remaining partition functions in HBaseStore (Alan Gates reviewed by Vaibhav Gumashta)

Modified:
    hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
    hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
    hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
    hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
    hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java Mon Mar  2 18:17:01 2015
@@ -354,6 +354,84 @@ public class TestHBaseStoreIntegration {
     Assert.assertEquals("fred", p.getValues().get(0));
   }
 
+  @Test
+  public void addPartitions() throws Exception {
+    String dbName = "default";
+    String tableName = "addParts";
+    int startTime = (int)(System.currentTimeMillis() / 1000);
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+        serde, null, null, emptyParameters);
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("pc", "string", ""));
+    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+        emptyParameters, null, null, null);
+    store.createTable(table);
+
+    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
+    List<Partition> partitions = new ArrayList<Partition>();
+    for (String val : partVals) {
+      List<String> vals = new ArrayList<String>();
+      vals.add(val);
+      StorageDescriptor psd = new StorageDescriptor(sd);
+      psd.setLocation("file:/tmp/pc=" + val);
+      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+          emptyParameters);
+      partitions.add(part);
+    }
+    store.addPartitions(dbName, tableName, partitions);
+
+    List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1);
+    Assert.assertEquals(5, partNames.size());
+    String[] names = partNames.toArray(new String[partNames.size()]);
+    Arrays.sort(names);
+    String[] canonicalNames = partVals.toArray(new String[partVals.size()]);
+    for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i];
+    Assert.assertArrayEquals(canonicalNames, names);
+  }
+
+  @Test
+  public void alterPartitions() throws Exception {
+    String dbName = "default";
+    String tableName = "alterParts";
+    int startTime = (int)(System.currentTimeMillis() / 1000);
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+        serde, null, null, emptyParameters);
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("pc", "string", ""));
+    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+        emptyParameters, null, null, null);
+    store.createTable(table);
+
+    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
+    List<Partition> partitions = new ArrayList<Partition>();
+    List<List<String>> allVals = new ArrayList<List<String>>();
+    for (String val : partVals) {
+      List<String> vals = new ArrayList<String>();
+      allVals.add(vals);
+      vals.add(val);
+      StorageDescriptor psd = new StorageDescriptor(sd);
+      psd.setLocation("file:/tmp/pc=" + val);
+      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+          emptyParameters);
+      partitions.add(part);
+    }
+    store.addPartitions(dbName, tableName, partitions);
+
+    for (Partition p : partitions) p.setLastAccessTime(startTime + 10);
+    store.alterPartitions(dbName, tableName, allVals, partitions);
+
+    partitions = store.getPartitions(dbName, tableName, -1);
+    for (Partition part : partitions) {
+      Assert.assertEquals(startTime + 10, part.getLastAccessTime());
+    }
+  }
+
   // TODO - Fix this and the next test.  They depend on test execution order and are bogus.
   @Test
   public void createManyPartitions() throws Exception {
@@ -489,6 +567,70 @@ public class TestHBaseStoreIntegration {
   }
 
   @Test
+  public void listPartitionsWithPs() throws Exception {
+    String dbName = "default";
+    String tableName = "listPartsPs";
+    int startTime = (int)(System.currentTimeMillis() / 1000);
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+        serde, null, null, emptyParameters);
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("ds", "string", ""));
+    partCols.add(new FieldSchema("region", "string", ""));
+    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+        emptyParameters, null, null, null);
+    store.createTable(table);
+
+    String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"},
+        {"tomorrow", "north america"}, {"tomorrow", "europe"}};
+    for (String[] pv : partVals) {
+      List<String> vals = new ArrayList<String>();
+      for (String v : pv) vals.add(v);
+      StorageDescriptor psd = new StorageDescriptor(sd);
+      psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
+      Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+          emptyParameters);
+      store.addPartition(part);
+    }
+
+    // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway.
+    // Test the case where we completely specify the partition
+    List<String> partitionNames =
+        store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1);
+    Assert.assertEquals(1, partitionNames.size());
+    Assert.assertEquals("ds=today/region=north america", partitionNames.get(0));
+
+    // Leave off the last value of the partition
+    partitionNames =
+        store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1);
+    Assert.assertEquals(2, partitionNames.size());
+    String[] names = partitionNames.toArray(new String[partitionNames.size()]);
+    Arrays.sort(names);
+    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+        "ds=today/region=north america"}, names);
+
+    // Put a star in the last value of the partition
+    partitionNames =
+        store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1);
+    Assert.assertEquals(2, partitionNames.size());
+    names = partitionNames.toArray(new String[partitionNames.size()]);
+    Arrays.sort(names);
+    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+        "ds=today/region=north america"}, names);
+
+    // Put a star in the first value of the partition
+    partitionNames =
+        store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1);
+    Assert.assertEquals(2, partitionNames.size());
+    names = partitionNames.toArray(new String[partitionNames.size()]);
+    Arrays.sort(names);
+    Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+        "ds=tomorrow/region=europe"}, names);
+  }
+
+  @Test
   public void dropPartition() throws Exception {
     String dbName = "default";
     String tableName = "myparttable2";

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java Mon Mar  2 18:17:01 2015
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.metastore.hbase;
 
 import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -46,6 +47,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Role;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -80,7 +82,7 @@ class HBaseReadWrite {
   @VisibleForTesting final static String NO_CACHE_CONF = "no.use.cache";
   private final static byte[] CATALOG_COL = "cat".getBytes(HBaseUtils.ENCODING);
   private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING);
-  private final static int tablesToCache = 10;
+  private final static int TABLES_TO_CACHE = 10;
 
   // TODO Add privileges as a second column in the CATALOG_CF
 
@@ -190,7 +192,7 @@ class HBaseReadWrite {
       partCache = new BogusPartitionCache();
       statsCache = StatsCache.getBogusStatsCache();
     } else {
-      tableCache = new ObjectCache<ObjectPair<String, String>, Table>(tablesToCache, tableHits,
+      tableCache = new ObjectCache<ObjectPair<String, String>, Table>(TABLES_TO_CACHE, tableHits,
           tableMisses, tableOverflows);
       sdCache = new ObjectCache<ByteArrayWrapper, StorageDescriptor>(sdsCacheSize, sdHits,
           sdMisses, sdOverflows);
@@ -334,6 +336,26 @@ class HBaseReadWrite {
   }
 
   /**
+   * Add a group of partitions
+   * @param partitions list of partitions to add
+   * @throws IOException
+   */
+  void putPartitions(List<Partition> partitions) throws IOException {
+    List<Put> puts = new ArrayList<Put>(partitions.size());
+    for (Partition partition : partitions) {
+      PartitionWritable part = new PartitionWritable(partition);
+      byte[] key = buildPartitionKey(part);
+      byte[] serialized = HBaseUtils.serialize(part);
+      Put p = new Put(key);
+      p.add(CATALOG_CF, CATALOG_COL, serialized);
+      puts.add(p);
+      partCache.put(partition.getDbName(), partition.getTableName(), partition);
+    }
+    getHTable(PART_TABLE).put(puts);
+    flush();
+  }
+
+  /**
    * Find all the partitions in a table.
    * @param dbName name of the database the table is in
    * @param tableName table name
@@ -360,45 +382,82 @@ class HBaseReadWrite {
    * Scan partitions based on partial key information.
    * @param dbName name of database, required
    * @param tableName name of table, required
-   * @param partVals partial specification of values.  Any values that are unknown can be left
-   *                 null in the list.  For example, if a table had two partition columns date
+   * @param partVals partial specification of values.  Any values that are unknown can instead be
+   *                 a '*'.  For example, if a table had two partition columns date
    *                 and region (in that order), and partitions ('today', 'na'), ('today', 'eu'),
-   *                 ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today'] would return
-   *                 ('today', 'na') and ('today', 'eu') while passing [null, 'eu'] would return
-   *                 ('today', 'eu') and ('tomorrow', 'eu')
+   *                 ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today', '*'] would return
+   *                 ('today', 'na') and ('today', 'eu') while passing ['*', 'eu'] would return
+   *                 ('today', 'eu') and ('tomorrow', 'eu').  Also the list can terminate early,
+   *                 which will be the equivalent of adding '*' for all non-included values.
+   *                 I.e. ['today'] is the same as ['today', '*'].
    * @param maxPartitions Maximum number of entries to return.
    * @return list of partitions that match the specified information
    * @throws IOException
+   * @throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException if the table containing
+   * the partitions can't be found.
    */
   List<Partition> scanPartitions(String dbName, String tableName, List<String> partVals,
-                                 int maxPartitions) throws IOException {
+                                 int maxPartitions) throws IOException, NoSuchObjectException {
+    // First, build as much of the key as we can so that we make the scan as tight as possible.
+    List<String> keyElements = new ArrayList<String>();
+    keyElements.add(dbName);
+    keyElements.add(tableName);
+
+    int firstStar = -1;
+    for (int i = 0; i < partVals.size(); i++) {
+      if ("*".equals(partVals.get(i))) {
+        firstStar = i;
+        break;
+      } else {
+        keyElements.add(partVals.get(i));
+      }
+    }
+
     byte[] keyPrefix;
-    if (partVals == null || partVals.size() == 0) {
-      keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName);
-      return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
-    }
-    int firstNull = 0;
-    for (; firstNull < partVals.size(); firstNull++) {
-      if (partVals.get(firstNull) == null) break;
-    }
-    if (firstNull == partVals.size()) {
-      keyPrefix = buildPartitionKey(dbName, tableName, partVals);
-      return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
-    }
-    keyPrefix = buildPartitionKey(dbName, tableName, partVals.subList(0, firstNull));
-    StringBuilder regex = new StringBuilder();
-    regex.append(dbName);
-    regex.append(':');
-    regex.append(tableName);
-    for (String val : partVals) {
-      regex.append(HBaseUtils.KEY_SEPARATOR);
-      if (val == null) regex.append("[^" + HBaseUtils.KEY_SEPARATOR + "]+"); // Will this do
-      // what I want?
-      else regex.append(val);
+    // We need to fetch the table to determine if the user fully specified the partitions or
+    // not, as it affects how we build the key.
+    Table table = getTable(dbName, tableName);
+    if (table == null) {
+      throw new NoSuchObjectException("Unable to find table " + dbName + "." + tableName);
+    }
+    if (partVals.size() == table.getPartitionKeys().size()) {
+      keyPrefix = HBaseUtils.buildKey(keyElements.toArray(new String[keyElements.size()]));
+    } else {
+      keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(keyElements.toArray(
+          new String[keyElements.size()]));
     }
 
-    Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL,
-        new RegexStringComparator(regex.toString()));
+    // Now, build a filter out of the remaining keys
+    String regex = null;
+    if (!(partVals.size() == table.getPartitionKeys().size() && firstStar == -1)) {
+      StringBuilder buf = new StringBuilder(".*");
+      for (int i = Math.max(0, firstStar);
+           i < table.getPartitionKeys().size() && i < partVals.size(); i++) {
+        buf.append(HBaseUtils.KEY_SEPARATOR);
+        if ("*".equals(partVals.get(i))) {
+          buf.append("[^");
+          buf.append(HBaseUtils.KEY_SEPARATOR);
+          buf.append("]+");
+        } else {
+          buf.append(partVals.get(i));
+        }
+      }
+      if (partVals.size() < table.getPartitionKeys().size()) {
+        buf.append(HBaseUtils.KEY_SEPARATOR);
+        buf.append(".*");
+      }
+      regex = buf.toString();
+    }
+
+    Filter filter = null;
+    if (regex != null) {
+      filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Scanning partitions with prefix <" + new String(keyPrefix) + "> and filter <" +
+          regex + ">");
+    }
 
     List<Partition> parts =
         scanPartitionsWithFilter(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions, filter);

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java Mon Mar  2 18:17:01 2015
@@ -161,7 +161,7 @@ public class HBaseSchemaTool {
         System.err.println("No such table: " + dbName + "." + tableName);
         return;
       }
-      String partName = HBaseStore.partName(table, partVals);
+      String partName = HBaseStore.buildExternalPartName(table, partVals);
       List<ColumnStatistics> stats = hrw.getPartitionStatistics(dbName, tableName,
           Arrays.asList(partName), Arrays.asList(partVals), colNames);
       if (stats == null) {

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java Mon Mar  2 18:17:01 2015
@@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUti
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.metastore.RawStore;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -212,7 +213,7 @@ public class HBaseStore implements RawSt
       return true;
     } catch (IOException e) {
       LOG.error("Unable to delete db" + e);
-      throw new MetaException("Unable to drop table " + tableName(dbName, tableName));
+      throw new MetaException("Unable to drop table " + tableNameForErrorMsg(dbName, tableName));
     }
   }
 
@@ -221,7 +222,7 @@ public class HBaseStore implements RawSt
     try {
       Table table = getHBase().getTable(dbName, tableName);
       if (table == null) {
-        LOG.debug("Unable to find table " + tableName(dbName, tableName));
+        LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
       }
       return table;
     } catch (IOException e) {
@@ -236,22 +237,18 @@ public class HBaseStore implements RawSt
       getHBase().putPartition(part);
       return true;
     } catch (IOException e) {
-      // TODO NOt sure what i should throw here
       LOG.error("Unable to add partition", e);
       throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
     }
   }
 
   @Override
-  public boolean addPartitions(String dbName, String tblName, List<Partition> parts) throws
-      InvalidObjectException, MetaException {
+  public boolean addPartitions(String dbName, String tblName, List<Partition> parts)
+      throws InvalidObjectException, MetaException {
     try {
-      for (Partition part : parts) {
-        getHBase().putPartition(part);
-      }
+      getHBase().putPartitions(parts);
       return true;
     } catch (IOException e) {
-      // TODO NOt sure what i should throw here
       LOG.error("Unable to add partitions", e);
       throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
     }
@@ -270,7 +267,7 @@ public class HBaseStore implements RawSt
       Partition part = getHBase().getPartition(dbName, tableName, part_vals);
       if (part == null) {
         throw new NoSuchObjectException("Unable to find partition " +
-            partName(dbName, tableName, part_vals));
+            partNameForErrorMsg(dbName, tableName, part_vals));
       }
       return part;
     } catch (IOException e) {
@@ -282,7 +279,12 @@ public class HBaseStore implements RawSt
   @Override
   public boolean doesPartitionExist(String dbName, String tableName, List<String> part_vals) throws
       MetaException, NoSuchObjectException {
-    throw new UnsupportedOperationException();
+    try {
+      return getHBase().getPartition(dbName, tableName, part_vals) != null;
+    } catch (IOException e) {
+      LOG.error("Unable to get partition", e);
+      throw new MetaException("Error reading partition " + e.getMessage());
+    }
   }
 
   @Override
@@ -293,7 +295,8 @@ public class HBaseStore implements RawSt
       return true;
     } catch (IOException e) {
       LOG.error("Unable to delete db" + e);
-      throw new MetaException("Unable to drop partition " + partName(dbName, tableName, part_vals));
+      throw new MetaException("Unable to drop partition " + partNameForErrorMsg(dbName, tableName,
+          part_vals));
     }
   }
 
@@ -315,8 +318,8 @@ public class HBaseStore implements RawSt
     try {
       getHBase().putTable(newTable);
     } catch (IOException e) {
-      LOG.error("Unable to alter table " + tableName(dbname, name), e);
-      throw new MetaException("Unable to alter table " + tableName(dbname, name));
+      LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e);
+      throw new MetaException("Unable to alter table " + tableNameForErrorMsg(dbname, name));
     }
   }
 
@@ -352,6 +355,7 @@ public class HBaseStore implements RawSt
   @Override
   public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables) throws
       MetaException, UnknownDBException {
+    // TODO needs to wait until we support pushing filters into HBase.
     throw new UnsupportedOperationException();
   }
 
@@ -364,7 +368,7 @@ public class HBaseStore implements RawSt
       List<String> names = new ArrayList<String>(parts.size());
       Table table = getHBase().getTable(db_name, tbl_name);
       for (Partition p : parts) {
-        names.add(partName(table, p));
+        names.add(buildExternalPartName(table, p));
       }
       return names;
     } catch (IOException e) {
@@ -376,20 +380,31 @@ public class HBaseStore implements RawSt
   @Override
   public List<String> listPartitionNamesByFilter(String db_name, String tbl_name, String filter,
                                                  short max_parts) throws MetaException {
+    // TODO needs to wait until we support pushing filters into HBase.
     throw new UnsupportedOperationException();
   }
 
   @Override
   public void alterPartition(String db_name, String tbl_name, List<String> part_vals,
                              Partition new_part) throws InvalidObjectException, MetaException {
-
+    try {
+      getHBase().putPartition(new_part);
+    } catch (IOException e) {
+      LOG.error("Unable to add partition", e);
+      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
+    }
   }
 
   @Override
   public void alterPartitions(String db_name, String tbl_name, List<List<String>> part_vals_list,
                               List<Partition> new_parts) throws InvalidObjectException,
       MetaException {
-    throw new UnsupportedOperationException();
+    try {
+      getHBase().putPartitions(new_parts);
+    } catch (IOException e) {
+      LOG.error("Unable to add partition", e);
+      throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
+    }
   }
 
   @Override
@@ -432,6 +447,7 @@ public class HBaseStore implements RawSt
   public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter,
                                                short maxParts) throws MetaException,
       NoSuchObjectException {
+    // TODO - Needs to wait for ability to push filters into HBase
     throw new UnsupportedOperationException();
   }
 
@@ -644,36 +660,46 @@ public class HBaseStore implements RawSt
 
   @Override
   public Partition getPartitionWithAuth(String dbName, String tblName, List<String> partVals,
-                                        String user_name, List<String> group_names) throws
-      MetaException, NoSuchObjectException, InvalidObjectException {
-    Partition p = getPartition(dbName, tblName, partVals);
-    // TODO check that user is authorized to see these partitions
-    return p;
+                                        String user_name, List<String> group_names)
+      throws MetaException, NoSuchObjectException, InvalidObjectException {
+    // We don't do authorization checks for partitions.
+    return getPartition(dbName, tblName, partVals);
   }
 
   @Override
   public List<Partition> getPartitionsWithAuth(String dbName, String tblName, short maxParts,
-                                               String userName, List<String> groupNames) throws
-      MetaException, NoSuchObjectException, InvalidObjectException {
-    List<Partition> parts = getPartitions(dbName, tblName, maxParts);
-    // TODO check that user is authorized;
-    return parts;
+                                               String userName, List<String> groupNames)
+      throws MetaException, NoSuchObjectException, InvalidObjectException {
+    // We don't do authorization checks for partitions.
+    return getPartitions(dbName, tblName, maxParts);
   }
 
   @Override
   public List<String> listPartitionNamesPs(String db_name, String tbl_name, List<String> part_vals,
-                                           short max_parts) throws MetaException,
-      NoSuchObjectException {
-    throw new UnsupportedOperationException();
+                                           short max_parts)
+      throws MetaException, NoSuchObjectException {
+    List<Partition> parts =
+        listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
+    List<String> partNames = new ArrayList<String>(parts.size());
+    for (Partition part : parts) {
+      partNames.add(buildExternalPartName(db_name, tbl_name, part.getValues()));
+    }
+    return partNames;
   }
 
 
   @Override
   public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name,
                                                   List<String> part_vals, short max_parts,
-                                                  String userName, List<String> groupNames) throws
-      MetaException, InvalidObjectException, NoSuchObjectException {
-    throw new UnsupportedOperationException();
+                                                  String userName, List<String> groupNames)
+      throws MetaException, NoSuchObjectException {
+    // We don't handle auth info with partitions
+    try {
+      return getHBase().scanPartitions(db_name, tbl_name, part_vals, max_parts);
+    } catch (IOException e) {
+      LOG.error("Unable to list partition names", e);
+      throw new MetaException("Failed to list part names, " + e.getMessage());
+    }
   }
 
   @Override
@@ -964,31 +990,37 @@ public class HBaseStore implements RawSt
     return hbase;
   }
 
-  private String tableName(String dbName, String tableName) {
+  // This is for building error messages only.  It does not look up anything in the metastore.
+  private String tableNameForErrorMsg(String dbName, String tableName) {
     return dbName + "." + tableName;
   }
 
-  private String partName(String dbName, String tableName, List<String> partVals) {
-    return tableName(dbName, tableName) + StringUtils.join(partVals, ':');
-  }
-
-  private String partName(Table table, Partition part) {
-    return partName(table, part.getValues());
-  }
-
-  static String partName(Table table, List<String> partVals) {
-    List<FieldSchema> partCols = table.getPartitionKeys();
-    StringBuilder builder = new StringBuilder();
-    if (partCols.size() != partVals.size()) {
-      throw new RuntimeException("Woh bad, different number of partition cols and vals!");
-    }
-    for (int i = 0; i < partCols.size(); i++) {
-      if (i != 0) builder.append('/');
-      builder.append(partCols.get(i).getName());
-      builder.append('=');
-      builder.append(partVals.get(i));
-    }
-    return builder.toString();
+  // This is for building error messages only.  It does not look up anything in the metastore as
+  // they may just throw another error.
+  private String partNameForErrorMsg(String dbName, String tableName, List<String> partVals) {
+    return tableNameForErrorMsg(dbName, tableName) + "." + StringUtils.join(partVals, ':');
+  }
+
+  private String buildExternalPartName(Table table, Partition part) {
+    return buildExternalPartName(table, part.getValues());
+  }
+
+  private String buildExternalPartName(String dbName, String tableName, List<String> partVals)
+      throws MetaException {
+    return buildExternalPartName(getTable(dbName, tableName), partVals);
+  }
+
+  /**
+   * Build a partition name for external use.  Necessary since HBase itself doesn't store
+   * partition names.
+   * @param table  table object
+   * @param partVals partition values.
+   * @return
+   */
+  static String buildExternalPartName(Table table, List<String> partVals) {
+    List<String> partCols = new ArrayList<String>();
+    for (FieldSchema pc : table.getPartitionKeys()) partCols.add(pc.getName());
+    return FileUtils.makePartName(partCols, partVals);
   }
 
   private List<String> partNameToVals(String name) {

Modified: hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java Mon Mar  2 18:17:01 2015
@@ -250,6 +250,55 @@ public class TestHBaseStore {
     Assert.assertEquals(tableName, p.getTableName());
     Assert.assertEquals(1, p.getValuesSize());
     Assert.assertEquals("fred", p.getValues().get(0));
+
+    Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals));
+    Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob")));
+  }
+
+  @Test
+  public void alterPartition() throws Exception {
+    String dbName = "default";
+    String tableName = "alterparttable";
+    int startTime = (int)(System.currentTimeMillis() / 1000);
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("col1", "int", "nocomment"));
+    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+        serde, null, null, emptyParameters);
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("pc", "string", ""));
+    Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+        emptyParameters, null, null, null);
+    store.createTable(table);
+
+    List<String> vals = Arrays.asList("fred");
+    StorageDescriptor psd = new StorageDescriptor(sd);
+    psd.setLocation("file:/tmp/pc=fred");
+    Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+        emptyParameters);
+    store.addPartition(part);
+
+    part.setLastAccessTime(startTime + 10);
+    store.alterPartition(dbName, tableName, vals, part);
+
+    Partition p = store.getPartition(dbName, tableName, vals);
+    Assert.assertEquals(1, p.getSd().getColsSize());
+    Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
+    Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
+    Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
+    Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
+    Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
+    Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
+    Assert.assertEquals("input", p.getSd().getInputFormat());
+    Assert.assertEquals("output", p.getSd().getOutputFormat());
+    Assert.assertEquals(dbName, p.getDbName());
+    Assert.assertEquals(tableName, p.getTableName());
+    Assert.assertEquals(1, p.getValuesSize());
+    Assert.assertEquals("fred", p.getValues().get(0));
+    Assert.assertEquals(startTime + 10, p.getLastAccessTime());
+
+    Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals));
+    Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob")));
   }
 
   @Test