You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/05/19 19:44:34 UTC
svn commit: r776392 - in /hadoop/hbase/branches/0.19: ./
src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/client/tableindexed/
src/java/org/apache/hadoop/hbase/regionserver/tableindexed/
Author: stack
Date: Tue May 19 17:44:33 2009
New Revision: 776392
URL: http://svn.apache.org/viewvc?rev=776392&view=rev
Log:
HBASE-1420 add abliity to add and remove (table) indexes on existing tables
Modified:
hadoop/hbase/branches/0.19/CHANGES.txt
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableAdmin.java
hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java
Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=776392&r1=776391&r2=776392&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Tue May 19 17:44:33 2009
@@ -24,6 +24,8 @@
long
HBASE-1401 close HLog (and open new one) if there hasnt been edits in
N minutes/hours
+ HBASE-1420 add abliity to add and remove (table) indexes on existing
+ tables (Clint Morgan via Stack)
Release 0.19.2 - May 9th, 2009
BUG FIXES
Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=776392&r1=776391&r2=776392&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/HTableDescriptor.java Tue May 19 17:44:33 2009
@@ -439,6 +439,10 @@
public void addIndex(IndexSpecification index) {
indexes.put(index.getIndexId(), index);
}
+
+ public void removeIndex(String indexId) {
+ indexes.remove(indexId);
+ }
/**
* Adds a column family.
Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableAdmin.java?rev=776392&r1=776391&r2=776392&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableAdmin.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableAdmin.java Tue May 19 17:44:33 2009
@@ -21,8 +21,13 @@
import java.io.IOException;
import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.Map.Entry;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.ColumnNameParseException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -30,6 +35,11 @@
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.regionserver.tableindexed.IndexMaintenanceUtils;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -38,6 +48,8 @@
*/
public class IndexedTableAdmin extends HBaseAdmin {
+ private static final Log LOG = LogFactory.getLog(IndexedTableAdmin.class);
+
/**
* Constructor
*
@@ -93,4 +105,49 @@
return indexTableDesc;
}
+
+ /** Remove an index for a table.
+ * @throws IOException
+ *
+ */
+ public void removeIndex(byte[] baseTableName, String indexId) throws IOException {
+ super.disableTable(baseTableName);
+ HTableDescriptor desc = super.getTableDescriptor(baseTableName);
+ IndexSpecification spec = desc.getIndex(indexId);
+ desc.removeIndex(indexId);
+ this.disableTable(spec.getIndexedTableName(baseTableName));
+ this.deleteTable(spec.getIndexedTableName(baseTableName));
+ super.modifyTable(baseTableName, desc);
+ super.enableTable(baseTableName);
+ }
+
+ /** Add an index to a table. */
+ public void addIndex(byte []baseTableName, IndexSpecification indexSpec) throws IOException {
+ LOG.warn("Adding index to existing table ["+Bytes.toString(baseTableName)+"], this may take a long time");
+ // TODO, make table read-only
+ LOG.warn("Not putting table in readonly, if its being written to, the index may get out of sync");
+ HTableDescriptor indexTableDesc = createIndexTableDesc(baseTableName, indexSpec);
+ super.createTable(indexTableDesc);
+ super.disableTable(baseTableName);
+ HTableDescriptor desc = super.getTableDescriptor(baseTableName);
+ desc.addIndex(indexSpec);
+ super.modifyTable(baseTableName, desc);
+ super.enableTable(baseTableName);
+ reIndexTable(baseTableName, indexSpec);
+ }
+
+ private void reIndexTable(byte[] baseTableName, IndexSpecification indexSpec) throws IOException {
+ HTable baseTable = new HTable(baseTableName);
+ HTable indexTable = new HTable(indexSpec.getIndexedTableName(baseTableName));
+ for (RowResult rowResult : baseTable.getScanner(indexSpec.getAllColumns())) {
+ SortedMap<byte[], byte[]> columnValues = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
+ for (Entry<byte[], Cell> entry : rowResult.entrySet()) {
+ columnValues.put(entry.getKey(), entry.getValue().getValue());
+ }
+ if (IndexMaintenanceUtils.doesApplyToIndex(indexSpec, columnValues)) {
+ BatchUpdate indexUpdate = IndexMaintenanceUtils.createIndexUpdate(indexSpec, rowResult.getRow(), columnValues);
+ indexTable.commit(indexUpdate);
+ }
+ }
+ }
}
Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java?rev=776392&r1=776391&r2=776392&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java Tue May 19 17:44:33 2009
@@ -138,7 +138,7 @@
Iterator<IndexSpecification> indexIterator = indexesToUpdate.iterator();
while (indexIterator.hasNext()) {
IndexSpecification indexSpec = indexIterator.next();
- if (!doesApplyToIndex(indexSpec, newColumnValues)) {
+ if (!IndexMaintenanceUtils.doesApplyToIndex(indexSpec, newColumnValues)) {
indexIterator.remove();
}
}
@@ -206,61 +206,20 @@
}
return false;
}
-
- /** Ask if this update does apply to the index.
- *
- * @param indexSpec
- * @param b
- * @return true if possibly apply.
- */
- private boolean doesApplyToIndex(IndexSpecification indexSpec, SortedMap<byte[], byte[]> columnValues) {
-
- for (byte [] neededCol : indexSpec.getIndexedColumns()) {
- if (! columnValues.containsKey(neededCol)) {
- LOG.debug("Index [" + indexSpec.getIndexId() + "] can't be updated because ["
- + Bytes.toString(neededCol) + "] is missing");
- return false;
- }
- }
- return true;
- }
+ // FIXME: This call takes place in an RPC, and requires an RPC. This makes for
+ // a likely deadlock if the number of RPCs we are trying to serve is >= the
+ // number of handler threads.
private void updateIndex(IndexSpecification indexSpec, byte[] row,
SortedMap<byte[], byte[]> columnValues) throws IOException {
- BatchUpdate indexUpdate = createIndexUpdate(indexSpec, row, columnValues);
- getIndexTable(indexSpec).commit(indexUpdate);
+ BatchUpdate indexUpdate = IndexMaintenanceUtils.createIndexUpdate(indexSpec, row, columnValues);
+ getIndexTable(indexSpec).commit(indexUpdate); // FIXME, this is synchronized
LOG.debug("Index [" + indexSpec.getIndexId() + "] adding new entry ["
+ Bytes.toString(indexUpdate.getRow()) + "] for row ["
+ Bytes.toString(row) + "]");
}
- private BatchUpdate createIndexUpdate(IndexSpecification indexSpec,
- byte[] row, SortedMap<byte[], byte[]> columnValues) {
- byte[] indexRow = indexSpec.getKeyGenerator().createIndexKey(row,
- columnValues);
- BatchUpdate update = new BatchUpdate(indexRow);
-
- update.put(IndexedTable.INDEX_BASE_ROW_COLUMN, row);
-
- for (byte[] col : indexSpec.getIndexedColumns()) {
- byte[] val = columnValues.get(col);
- if (val == null) {
- throw new RuntimeException("Unexpected missing column value. ["+Bytes.toString(col)+"]");
- }
- update.put(col, val);
- }
-
- for (byte [] col : indexSpec.getAdditionalColumns()) {
- byte[] val = columnValues.get(col);
- if (val != null) {
- update.put(col, val);
- }
- }
-
- return update;
- }
-
@Override
public void deleteAll(final byte[] row, final long ts, final Integer lockid)
throws IOException {
@@ -286,7 +245,7 @@
SortedMap<byte[], byte[]> currentColumnValues = convertToValueMap(currentColumnCells);
for (IndexSpecification indexSpec : getIndexes()) {
- if (doesApplyToIndex(indexSpec, currentColumnValues)) {
+ if (IndexMaintenanceUtils.doesApplyToIndex(indexSpec, currentColumnValues)) {
updateIndex(indexSpec, row, currentColumnValues);
}
}
@@ -331,7 +290,7 @@
SortedMap<byte[], byte[]> currentColumnValues = convertToValueMap(currentColumnCells);
for (IndexSpecification indexSpec : getIndexes()) {
- if (doesApplyToIndex(indexSpec, currentColumnValues)) {
+ if (IndexMaintenanceUtils.doesApplyToIndex(indexSpec, currentColumnValues)) {
updateIndex(indexSpec, row, currentColumnValues);
}
}