You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2015/03/02 19:17:03 UTC
svn commit: r1663386 - in /hive/branches/hbase-metastore:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/
metastore/src/java/org/apache/hadoop/hive/metastore/hbase/
metastore/src/test/org/apache/hadoop/hive/metastore/hbase/
Author: gates
Date: Mon Mar 2 18:17:01 2015
New Revision: 1663386
URL: http://svn.apache.org/r1663386
Log:
HIVE-9641 Fill out remaining partition functions in HBaseStore (Alan Gates reviewed by Vaibhav Gumashta)
Modified:
hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java Mon Mar 2 18:17:01 2015
@@ -354,6 +354,84 @@ public class TestHBaseStoreIntegration {
Assert.assertEquals("fred", p.getValues().get(0));
}
+ @Test
+ public void addPartitions() throws Exception {
+ String dbName = "default";
+ String tableName = "addParts";
+ int startTime = (int)(System.currentTimeMillis() / 1000);
+ List<FieldSchema> cols = new ArrayList<FieldSchema>();
+ cols.add(new FieldSchema("col1", "int", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, emptyParameters);
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+ partCols.add(new FieldSchema("pc", "string", ""));
+ Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+ emptyParameters, null, null, null);
+ store.createTable(table);
+
+ List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
+ List<Partition> partitions = new ArrayList<Partition>();
+ for (String val : partVals) {
+ List<String> vals = new ArrayList<String>();
+ vals.add(val);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/pc=" + val);
+ Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+ emptyParameters);
+ partitions.add(part);
+ }
+ store.addPartitions(dbName, tableName, partitions);
+
+ List<String> partNames = store.listPartitionNames(dbName, tableName, (short) -1);
+ Assert.assertEquals(5, partNames.size());
+ String[] names = partNames.toArray(new String[partNames.size()]);
+ Arrays.sort(names);
+ String[] canonicalNames = partVals.toArray(new String[partVals.size()]);
+ for (int i = 0; i < canonicalNames.length; i++) canonicalNames[i] = "pc=" + canonicalNames[i];
+ Assert.assertArrayEquals(canonicalNames, names);
+ }
+
+ @Test
+ public void alterPartitions() throws Exception {
+ String dbName = "default";
+ String tableName = "alterParts";
+ int startTime = (int)(System.currentTimeMillis() / 1000);
+ List<FieldSchema> cols = new ArrayList<FieldSchema>();
+ cols.add(new FieldSchema("col1", "int", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, emptyParameters);
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+ partCols.add(new FieldSchema("pc", "string", ""));
+ Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+ emptyParameters, null, null, null);
+ store.createTable(table);
+
+ List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
+ List<Partition> partitions = new ArrayList<Partition>();
+ List<List<String>> allVals = new ArrayList<List<String>>();
+ for (String val : partVals) {
+ List<String> vals = new ArrayList<String>();
+ allVals.add(vals);
+ vals.add(val);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/pc=" + val);
+ Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+ emptyParameters);
+ partitions.add(part);
+ }
+ store.addPartitions(dbName, tableName, partitions);
+
+ for (Partition p : partitions) p.setLastAccessTime(startTime + 10);
+ store.alterPartitions(dbName, tableName, allVals, partitions);
+
+ partitions = store.getPartitions(dbName, tableName, -1);
+ for (Partition part : partitions) {
+ Assert.assertEquals(startTime + 10, part.getLastAccessTime());
+ }
+ }
+
// TODO - Fix this and the next test. They depend on test execution order and are bogus.
@Test
public void createManyPartitions() throws Exception {
@@ -489,6 +567,70 @@ public class TestHBaseStoreIntegration {
}
@Test
+ public void listPartitionsWithPs() throws Exception {
+ String dbName = "default";
+ String tableName = "listPartsPs";
+ int startTime = (int)(System.currentTimeMillis() / 1000);
+ List<FieldSchema> cols = new ArrayList<FieldSchema>();
+ cols.add(new FieldSchema("col1", "int", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, emptyParameters);
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+ partCols.add(new FieldSchema("ds", "string", ""));
+ partCols.add(new FieldSchema("region", "string", ""));
+ Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+ emptyParameters, null, null, null);
+ store.createTable(table);
+
+ String[][] partVals = new String[][]{{"today", "north america"}, {"today", "europe"},
+ {"tomorrow", "north america"}, {"tomorrow", "europe"}};
+ for (String[] pv : partVals) {
+ List<String> vals = new ArrayList<String>();
+ for (String v : pv) vals.add(v);
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/ds=" + pv[0] + "/region=" + pv[1]);
+ Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+ emptyParameters);
+ store.addPartition(part);
+ }
+
+ // We only test listPartitionNamesPs since it calls listPartitionsPsWithAuth anyway.
+ // Test the case where we completely specify the partition
+ List<String> partitionNames =
+ store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0]), (short) -1);
+ Assert.assertEquals(1, partitionNames.size());
+ Assert.assertEquals("ds=today/region=north america", partitionNames.get(0));
+
+ // Leave off the last value of the partition
+ partitionNames =
+ store.listPartitionNamesPs(dbName, tableName, Arrays.asList(partVals[0][0]), (short)-1);
+ Assert.assertEquals(2, partitionNames.size());
+ String[] names = partitionNames.toArray(new String[partitionNames.size()]);
+ Arrays.sort(names);
+ Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+ "ds=today/region=north america"}, names);
+
+ // Put a star in the last value of the partition
+ partitionNames =
+ store.listPartitionNamesPs(dbName, tableName, Arrays.asList("today", "*"), (short)-1);
+ Assert.assertEquals(2, partitionNames.size());
+ names = partitionNames.toArray(new String[partitionNames.size()]);
+ Arrays.sort(names);
+ Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+ "ds=today/region=north america"}, names);
+
+ // Put a star in the first value of the partition
+ partitionNames =
+ store.listPartitionNamesPs(dbName, tableName, Arrays.asList("*", "europe"), (short)-1);
+ Assert.assertEquals(2, partitionNames.size());
+ names = partitionNames.toArray(new String[partitionNames.size()]);
+ Arrays.sort(names);
+ Assert.assertArrayEquals(new String[] {"ds=today/region=europe",
+ "ds=tomorrow/region=europe"}, names);
+ }
+
+ @Test
public void dropPartition() throws Exception {
String dbName = "default";
String tableName = "myparttable2";
Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java Mon Mar 2 18:17:01 2015
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.metastore.hbase;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,6 +47,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
@@ -80,7 +82,7 @@ class HBaseReadWrite {
@VisibleForTesting final static String NO_CACHE_CONF = "no.use.cache";
private final static byte[] CATALOG_COL = "cat".getBytes(HBaseUtils.ENCODING);
private final static byte[] REF_COUNT_COL = "ref".getBytes(HBaseUtils.ENCODING);
- private final static int tablesToCache = 10;
+ private final static int TABLES_TO_CACHE = 10;
// TODO Add privileges as a second column in the CATALOG_CF
@@ -190,7 +192,7 @@ class HBaseReadWrite {
partCache = new BogusPartitionCache();
statsCache = StatsCache.getBogusStatsCache();
} else {
- tableCache = new ObjectCache<ObjectPair<String, String>, Table>(tablesToCache, tableHits,
+ tableCache = new ObjectCache<ObjectPair<String, String>, Table>(TABLES_TO_CACHE, tableHits,
tableMisses, tableOverflows);
sdCache = new ObjectCache<ByteArrayWrapper, StorageDescriptor>(sdsCacheSize, sdHits,
sdMisses, sdOverflows);
@@ -334,6 +336,26 @@ class HBaseReadWrite {
}
/**
+ * Add a group of partitions
+ * @param partitions list of partitions to add
+ * @throws IOException
+ */
+ void putPartitions(List<Partition> partitions) throws IOException {
+ List<Put> puts = new ArrayList<Put>(partitions.size());
+ for (Partition partition : partitions) {
+ PartitionWritable part = new PartitionWritable(partition);
+ byte[] key = buildPartitionKey(part);
+ byte[] serialized = HBaseUtils.serialize(part);
+ Put p = new Put(key);
+ p.add(CATALOG_CF, CATALOG_COL, serialized);
+ puts.add(p);
+ partCache.put(partition.getDbName(), partition.getTableName(), partition);
+ }
+ getHTable(PART_TABLE).put(puts);
+ flush();
+ }
+
+ /**
* Find all the partitions in a table.
* @param dbName name of the database the table is in
* @param tableName table name
@@ -360,45 +382,82 @@ class HBaseReadWrite {
* Scan partitions based on partial key information.
* @param dbName name of database, required
* @param tableName name of table, required
- * @param partVals partial specification of values. Any values that are unknown can be left
- * null in the list. For example, if a table had two partition columns date
+ * @param partVals partial specification of values. Any values that are unknown can instead be
+ * a '*'. For example, if a table had two partition columns date
* and region (in that order), and partitions ('today', 'na'), ('today', 'eu'),
- * ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today'] would return
- * ('today', 'na') and ('today', 'eu') while passing [null, 'eu'] would return
- * ('today', 'eu') and ('tomorrow', 'eu')
+ * ('tomorrow', 'na'), ('tomorrow', 'eu') then passing ['today', '*'] would return
+ * ('today', 'na') and ('today', 'eu') while passing ['*', 'eu'] would return
+ * ('today', 'eu') and ('tomorrow', 'eu'). Also the list can terminate early,
+ * which will be the equivalent of adding '*' for all non-included values.
+ * I.e. ['today'] is the same as ['today', '*'].
* @param maxPartitions Maximum number of entries to return.
* @return list of partitions that match the specified information
* @throws IOException
+ * @throws org.apache.hadoop.hive.metastore.api.NoSuchObjectException if the table containing
+ * the partitions can't be found.
*/
List<Partition> scanPartitions(String dbName, String tableName, List<String> partVals,
- int maxPartitions) throws IOException {
+ int maxPartitions) throws IOException, NoSuchObjectException {
+ // First, build as much of the key as we can so that we make the scan as tight as possible.
+ List<String> keyElements = new ArrayList<String>();
+ keyElements.add(dbName);
+ keyElements.add(tableName);
+
+ int firstStar = -1;
+ for (int i = 0; i < partVals.size(); i++) {
+ if ("*".equals(partVals.get(i))) {
+ firstStar = i;
+ break;
+ } else {
+ keyElements.add(partVals.get(i));
+ }
+ }
+
byte[] keyPrefix;
- if (partVals == null || partVals.size() == 0) {
- keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(dbName, tableName);
- return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
- }
- int firstNull = 0;
- for (; firstNull < partVals.size(); firstNull++) {
- if (partVals.get(firstNull) == null) break;
- }
- if (firstNull == partVals.size()) {
- keyPrefix = buildPartitionKey(dbName, tableName, partVals);
- return scanPartitions(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions);
- }
- keyPrefix = buildPartitionKey(dbName, tableName, partVals.subList(0, firstNull));
- StringBuilder regex = new StringBuilder();
- regex.append(dbName);
- regex.append(':');
- regex.append(tableName);
- for (String val : partVals) {
- regex.append(HBaseUtils.KEY_SEPARATOR);
- if (val == null) regex.append("[^" + HBaseUtils.KEY_SEPARATOR + "]+"); // Will this do
- // what I want?
- else regex.append(val);
+ // We need to fetch the table to determine if the user fully specified the partitions or
+ // not, as it affects how we build the key.
+ Table table = getTable(dbName, tableName);
+ if (table == null) {
+ throw new NoSuchObjectException("Unable to find table " + dbName + "." + tableName);
+ }
+ if (partVals.size() == table.getPartitionKeys().size()) {
+ keyPrefix = HBaseUtils.buildKey(keyElements.toArray(new String[keyElements.size()]));
+ } else {
+ keyPrefix = HBaseUtils.buildKeyWithTrailingSeparator(keyElements.toArray(
+ new String[keyElements.size()]));
}
- Filter filter = new RowFilter(CompareFilter.CompareOp.EQUAL,
- new RegexStringComparator(regex.toString()));
+ // Now, build a filter out of the remaining keys
+ String regex = null;
+ if (!(partVals.size() == table.getPartitionKeys().size() && firstStar == -1)) {
+ StringBuilder buf = new StringBuilder(".*");
+ for (int i = Math.max(0, firstStar);
+ i < table.getPartitionKeys().size() && i < partVals.size(); i++) {
+ buf.append(HBaseUtils.KEY_SEPARATOR);
+ if ("*".equals(partVals.get(i))) {
+ buf.append("[^");
+ buf.append(HBaseUtils.KEY_SEPARATOR);
+ buf.append("]+");
+ } else {
+ buf.append(partVals.get(i));
+ }
+ }
+ if (partVals.size() < table.getPartitionKeys().size()) {
+ buf.append(HBaseUtils.KEY_SEPARATOR);
+ buf.append(".*");
+ }
+ regex = buf.toString();
+ }
+
+ Filter filter = null;
+ if (regex != null) {
+ filter = new RowFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator(regex));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Scanning partitions with prefix <" + new String(keyPrefix) + "> and filter <" +
+ regex + ">");
+ }
List<Partition> parts =
scanPartitionsWithFilter(keyPrefix, CATALOG_CF, CATALOG_COL, maxPartitions, filter);
Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java Mon Mar 2 18:17:01 2015
@@ -161,7 +161,7 @@ public class HBaseSchemaTool {
System.err.println("No such table: " + dbName + "." + tableName);
return;
}
- String partName = HBaseStore.partName(table, partVals);
+ String partName = HBaseStore.buildExternalPartName(table, partVals);
List<ColumnStatistics> stats = hrw.getPartitionStatistics(dbName, tableName,
Arrays.asList(partName), Arrays.asList(partVals), colNames);
if (stats == null) {
Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java Mon Mar 2 18:17:01 2015
@@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.metastore.RawStore;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -212,7 +213,7 @@ public class HBaseStore implements RawSt
return true;
} catch (IOException e) {
LOG.error("Unable to delete db" + e);
- throw new MetaException("Unable to drop table " + tableName(dbName, tableName));
+ throw new MetaException("Unable to drop table " + tableNameForErrorMsg(dbName, tableName));
}
}
@@ -221,7 +222,7 @@ public class HBaseStore implements RawSt
try {
Table table = getHBase().getTable(dbName, tableName);
if (table == null) {
- LOG.debug("Unable to find table " + tableName(dbName, tableName));
+ LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
}
return table;
} catch (IOException e) {
@@ -236,22 +237,18 @@ public class HBaseStore implements RawSt
getHBase().putPartition(part);
return true;
} catch (IOException e) {
- // TODO NOt sure what i should throw here
LOG.error("Unable to add partition", e);
throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
}
}
@Override
- public boolean addPartitions(String dbName, String tblName, List<Partition> parts) throws
- InvalidObjectException, MetaException {
+ public boolean addPartitions(String dbName, String tblName, List<Partition> parts)
+ throws InvalidObjectException, MetaException {
try {
- for (Partition part : parts) {
- getHBase().putPartition(part);
- }
+ getHBase().putPartitions(parts);
return true;
} catch (IOException e) {
- // TODO NOt sure what i should throw here
LOG.error("Unable to add partitions", e);
throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
}
@@ -270,7 +267,7 @@ public class HBaseStore implements RawSt
Partition part = getHBase().getPartition(dbName, tableName, part_vals);
if (part == null) {
throw new NoSuchObjectException("Unable to find partition " +
- partName(dbName, tableName, part_vals));
+ partNameForErrorMsg(dbName, tableName, part_vals));
}
return part;
} catch (IOException e) {
@@ -282,7 +279,12 @@ public class HBaseStore implements RawSt
@Override
public boolean doesPartitionExist(String dbName, String tableName, List<String> part_vals) throws
MetaException, NoSuchObjectException {
- throw new UnsupportedOperationException();
+ try {
+ return getHBase().getPartition(dbName, tableName, part_vals) != null;
+ } catch (IOException e) {
+ LOG.error("Unable to get partition", e);
+ throw new MetaException("Error reading partition " + e.getMessage());
+ }
}
@Override
@@ -293,7 +295,8 @@ public class HBaseStore implements RawSt
return true;
} catch (IOException e) {
LOG.error("Unable to delete db" + e);
- throw new MetaException("Unable to drop partition " + partName(dbName, tableName, part_vals));
+ throw new MetaException("Unable to drop partition " + partNameForErrorMsg(dbName, tableName,
+ part_vals));
}
}
@@ -315,8 +318,8 @@ public class HBaseStore implements RawSt
try {
getHBase().putTable(newTable);
} catch (IOException e) {
- LOG.error("Unable to alter table " + tableName(dbname, name), e);
- throw new MetaException("Unable to alter table " + tableName(dbname, name));
+ LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e);
+ throw new MetaException("Unable to alter table " + tableNameForErrorMsg(dbname, name));
}
}
@@ -352,6 +355,7 @@ public class HBaseStore implements RawSt
@Override
public List<String> listTableNamesByFilter(String dbName, String filter, short max_tables) throws
MetaException, UnknownDBException {
+ // TODO needs to wait until we support pushing filters into HBase.
throw new UnsupportedOperationException();
}
@@ -364,7 +368,7 @@ public class HBaseStore implements RawSt
List<String> names = new ArrayList<String>(parts.size());
Table table = getHBase().getTable(db_name, tbl_name);
for (Partition p : parts) {
- names.add(partName(table, p));
+ names.add(buildExternalPartName(table, p));
}
return names;
} catch (IOException e) {
@@ -376,20 +380,31 @@ public class HBaseStore implements RawSt
@Override
public List<String> listPartitionNamesByFilter(String db_name, String tbl_name, String filter,
short max_parts) throws MetaException {
+ // TODO needs to wait until we support pushing filters into HBase.
throw new UnsupportedOperationException();
}
@Override
public void alterPartition(String db_name, String tbl_name, List<String> part_vals,
Partition new_part) throws InvalidObjectException, MetaException {
-
+ try {
+ getHBase().putPartition(new_part);
+ } catch (IOException e) {
+ LOG.error("Unable to add partition", e);
+ throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
+ }
}
@Override
public void alterPartitions(String db_name, String tbl_name, List<List<String>> part_vals_list,
List<Partition> new_parts) throws InvalidObjectException,
MetaException {
- throw new UnsupportedOperationException();
+ try {
+ getHBase().putPartitions(new_parts);
+ } catch (IOException e) {
+ LOG.error("Unable to add partition", e);
+ throw new MetaException("Unable to read from or write to hbase " + e.getMessage());
+ }
}
@Override
@@ -432,6 +447,7 @@ public class HBaseStore implements RawSt
public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter,
short maxParts) throws MetaException,
NoSuchObjectException {
+ // TODO - Needs to wait for ability to push filters into HBase
throw new UnsupportedOperationException();
}
@@ -644,36 +660,46 @@ public class HBaseStore implements RawSt
@Override
public Partition getPartitionWithAuth(String dbName, String tblName, List<String> partVals,
- String user_name, List<String> group_names) throws
- MetaException, NoSuchObjectException, InvalidObjectException {
- Partition p = getPartition(dbName, tblName, partVals);
- // TODO check that user is authorized to see these partitions
- return p;
+ String user_name, List<String> group_names)
+ throws MetaException, NoSuchObjectException, InvalidObjectException {
+ // We don't do authorization checks for partitions.
+ return getPartition(dbName, tblName, partVals);
}
@Override
public List<Partition> getPartitionsWithAuth(String dbName, String tblName, short maxParts,
- String userName, List<String> groupNames) throws
- MetaException, NoSuchObjectException, InvalidObjectException {
- List<Partition> parts = getPartitions(dbName, tblName, maxParts);
- // TODO check that user is authorized;
- return parts;
+ String userName, List<String> groupNames)
+ throws MetaException, NoSuchObjectException, InvalidObjectException {
+ // We don't do authorization checks for partitions.
+ return getPartitions(dbName, tblName, maxParts);
}
@Override
public List<String> listPartitionNamesPs(String db_name, String tbl_name, List<String> part_vals,
- short max_parts) throws MetaException,
- NoSuchObjectException {
- throw new UnsupportedOperationException();
+ short max_parts)
+ throws MetaException, NoSuchObjectException {
+ List<Partition> parts =
+ listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
+ List<String> partNames = new ArrayList<String>(parts.size());
+ for (Partition part : parts) {
+ partNames.add(buildExternalPartName(db_name, tbl_name, part.getValues()));
+ }
+ return partNames;
}
@Override
public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name,
List<String> part_vals, short max_parts,
- String userName, List<String> groupNames) throws
- MetaException, InvalidObjectException, NoSuchObjectException {
- throw new UnsupportedOperationException();
+ String userName, List<String> groupNames)
+ throws MetaException, NoSuchObjectException {
+ // We don't handle auth info with partitions
+ try {
+ return getHBase().scanPartitions(db_name, tbl_name, part_vals, max_parts);
+ } catch (IOException e) {
+ LOG.error("Unable to list partition names", e);
+ throw new MetaException("Failed to list part names, " + e.getMessage());
+ }
}
@Override
@@ -964,31 +990,37 @@ public class HBaseStore implements RawSt
return hbase;
}
- private String tableName(String dbName, String tableName) {
+ // This is for building error messages only. It does not look up anything in the metastore.
+ private String tableNameForErrorMsg(String dbName, String tableName) {
return dbName + "." + tableName;
}
- private String partName(String dbName, String tableName, List<String> partVals) {
- return tableName(dbName, tableName) + StringUtils.join(partVals, ':');
- }
-
- private String partName(Table table, Partition part) {
- return partName(table, part.getValues());
- }
-
- static String partName(Table table, List<String> partVals) {
- List<FieldSchema> partCols = table.getPartitionKeys();
- StringBuilder builder = new StringBuilder();
- if (partCols.size() != partVals.size()) {
- throw new RuntimeException("Woh bad, different number of partition cols and vals!");
- }
- for (int i = 0; i < partCols.size(); i++) {
- if (i != 0) builder.append('/');
- builder.append(partCols.get(i).getName());
- builder.append('=');
- builder.append(partVals.get(i));
- }
- return builder.toString();
+ // This is for building error messages only. It does not look up anything in the metastore as
+ // they may just throw another error.
+ private String partNameForErrorMsg(String dbName, String tableName, List<String> partVals) {
+ return tableNameForErrorMsg(dbName, tableName) + "." + StringUtils.join(partVals, ':');
+ }
+
+ private String buildExternalPartName(Table table, Partition part) {
+ return buildExternalPartName(table, part.getValues());
+ }
+
+ private String buildExternalPartName(String dbName, String tableName, List<String> partVals)
+ throws MetaException {
+ return buildExternalPartName(getTable(dbName, tableName), partVals);
+ }
+
+ /**
+ * Build a partition name for external use. Necessary since HBase itself doesn't store
+ * partition names.
+ * @param table table object
+ * @param partVals partition values.
+ * @return
+ */
+ static String buildExternalPartName(Table table, List<String> partVals) {
+ List<String> partCols = new ArrayList<String>();
+ for (FieldSchema pc : table.getPartitionKeys()) partCols.add(pc.getName());
+ return FileUtils.makePartName(partCols, partVals);
}
private List<String> partNameToVals(String name) {
Modified: hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java?rev=1663386&r1=1663385&r2=1663386&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseStore.java Mon Mar 2 18:17:01 2015
@@ -250,6 +250,55 @@ public class TestHBaseStore {
Assert.assertEquals(tableName, p.getTableName());
Assert.assertEquals(1, p.getValuesSize());
Assert.assertEquals("fred", p.getValues().get(0));
+
+ Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals));
+ Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob")));
+ }
+
+ @Test
+ public void alterPartition() throws Exception {
+ String dbName = "default";
+ String tableName = "alterparttable";
+ int startTime = (int)(System.currentTimeMillis() / 1000);
+ List<FieldSchema> cols = new ArrayList<FieldSchema>();
+ cols.add(new FieldSchema("col1", "int", "nocomment"));
+ SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
+ StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0,
+ serde, null, null, emptyParameters);
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+ partCols.add(new FieldSchema("pc", "string", ""));
+ Table table = new Table(tableName, dbName, "me", startTime, startTime, 0, sd, partCols,
+ emptyParameters, null, null, null);
+ store.createTable(table);
+
+ List<String> vals = Arrays.asList("fred");
+ StorageDescriptor psd = new StorageDescriptor(sd);
+ psd.setLocation("file:/tmp/pc=fred");
+ Partition part = new Partition(vals, dbName, tableName, startTime, startTime, psd,
+ emptyParameters);
+ store.addPartition(part);
+
+ part.setLastAccessTime(startTime + 10);
+ store.alterPartition(dbName, tableName, vals, part);
+
+ Partition p = store.getPartition(dbName, tableName, vals);
+ Assert.assertEquals(1, p.getSd().getColsSize());
+ Assert.assertEquals("col1", p.getSd().getCols().get(0).getName());
+ Assert.assertEquals("int", p.getSd().getCols().get(0).getType());
+ Assert.assertEquals("nocomment", p.getSd().getCols().get(0).getComment());
+ Assert.assertEquals("serde", p.getSd().getSerdeInfo().getName());
+ Assert.assertEquals("seriallib", p.getSd().getSerdeInfo().getSerializationLib());
+ Assert.assertEquals("file:/tmp/pc=fred", p.getSd().getLocation());
+ Assert.assertEquals("input", p.getSd().getInputFormat());
+ Assert.assertEquals("output", p.getSd().getOutputFormat());
+ Assert.assertEquals(dbName, p.getDbName());
+ Assert.assertEquals(tableName, p.getTableName());
+ Assert.assertEquals(1, p.getValuesSize());
+ Assert.assertEquals("fred", p.getValues().get(0));
+ Assert.assertEquals(startTime + 10, p.getLastAccessTime());
+
+ Assert.assertTrue(store.doesPartitionExist(dbName, tableName, vals));
+ Assert.assertFalse(store.doesPartitionExist(dbName, tableName, Arrays.asList("bob")));
}
@Test