You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2007/12/08 01:39:33 UTC
svn commit: r602284 - in /lucene/hadoop/trunk/src/contrib/hbase: ./
src/java/org/apache/hadoop/hbase/
Author: stack
Date: Fri Dec 7 16:39:32 2007
New Revision: 602284
URL: http://svn.apache.org/viewvc?rev=602284&view=rev
Log:
HADOOP-1550 No means of deleting a'row'
Modified:
lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Fri Dec 7 16:39:32 2007
@@ -12,6 +12,7 @@
HADOOP-2068 RESTful interface (Bryan Duxbury via Stack)
HADOOP-2316 Run REST servlet outside of master
(Bryan Duxbury & Stack)
+ HADOOP-1550 No means of deleting a'row' (Bryan Duxbuery via Stack)
OPTIMIZATIONS
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Fri Dec 7 16:39:32 2007
@@ -1060,7 +1060,7 @@
*/
private List<HStoreKey> getKeys(final HStoreKey origin, final int versions)
throws IOException {
-
+
List<HStoreKey> keys = null;
Text colFamily = HStoreKey.extractFamily(origin.getColumn());
HStore targetStore = stores.get(colFamily);
@@ -1071,7 +1071,7 @@
}
return keys;
}
-
+
/**
* Return an iterator that scans over the HRegion, returning the indicated
* columns for only the rows that match the data filter. This Iterator must
@@ -1248,6 +1248,33 @@
releaseRowLock(row);
}
}
+
+ /**
+ * Delete all cells of the same age as the passed timestamp or older.
+ * @param row
+ * @param ts Delete all entries that have this timestamp or older
+ * @throws IOException
+ */
+ public void deleteAll(final Text row, final long ts)
+ throws IOException {
+
+ obtainRowLock(row);
+
+ try {
+ for(Map.Entry<Text, HStore> store : stores.entrySet()){
+ List<HStoreKey> keys = store.getValue().getKeys(new HStoreKey(row, ts), ALL_VERSIONS);
+
+ TreeMap<HStoreKey, byte []> edits = new TreeMap<HStoreKey, byte []>();
+ for (HStoreKey key: keys) {
+ edits.put(key, HLogEdit.deleteBytes.get());
+ }
+ update(edits);
+ }
+ } finally {
+ releaseRowLock(row);
+ }
+ }
+
/**
* Delete one or many cells.
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java Fri Dec 7 16:39:32 2007
@@ -134,6 +134,18 @@
*/
public void deleteAll(Text regionName, Text row, Text column, long timestamp)
throws IOException;
+
+ /**
+ * Delete all cells that match the passed row and whose
+ * timestamp is equal-to or older than the passed timestamp.
+ *
+ * @param regionName region name
+ * @param row row key
+ * @param timestamp Delete all entries that have this timestamp or older
+ * @throws IOException
+ */
+ public void deleteAll(Text regionName, Text row, long timestamp)
+ throws IOException;
//
// remote scanner interface
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Fri Dec 7 16:39:32 2007
@@ -1516,6 +1516,14 @@
region.deleteAll(row, column, timestamp);
}
+ /** {@inheritDoc} */
+ public void deleteAll(final Text regionName, final Text row,
+ final long timestamp)
+ throws IOException {
+ HRegion region = getRegion(regionName);
+ region.deleteAll(row, timestamp);
+ }
+
/**
* @return Info on this server.
*/
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Fri Dec 7 16:39:32 2007
@@ -268,7 +268,7 @@
(versions - results.size())));
}
return results;
-
+
} finally {
this.lock.readLock().unlock();
}
@@ -289,9 +289,28 @@
SortedMap<HStoreKey, byte []> tailMap = map.tailMap(origin);
for (Map.Entry<HStoreKey, byte []> es: tailMap.entrySet()) {
HStoreKey key = es.getKey();
- if (!key.matchesRowCol(origin)) {
- break;
+
+ // if there's no column name, then compare rows and timestamps
+ if (origin.getColumn().toString().equals("")) {
+ // if the current and origin row don't match, then we can jump
+ // out of the loop entirely.
+ if (!key.getRow().equals(origin.getRow())) {
+ break;
+ }
+ // if the rows match but the timestamp is newer, skip it so we can
+ // get to the ones we actually want.
+ if (key.getTimestamp() > origin.getTimestamp()) {
+ continue;
+ }
+ }
+ else{ // compare rows and columns
+ // if the key doesn't match the row and column, then we're done, since
+ // all the cells are ordered.
+ if (!key.matchesRowCol(origin)) {
+ break;
+ }
}
+
if (!HLogEdit.isDeleted(es.getValue())) {
result.add(key);
if (versions != HConstants.ALL_VERSIONS && result.size() >= versions) {
@@ -1429,6 +1448,7 @@
*/
void getFull(HStoreKey key, TreeMap<Text, byte []> results)
throws IOException {
+ Map<Text, List<Long>> deletes = new HashMap<Text, List<Long>>();
this.lock.readLock().lock();
memcache.getFull(key, results);
@@ -1447,7 +1467,7 @@
Text readcol = readkey.getColumn();
if (results.get(readcol) == null
&& key.matchesWithoutColumn(readkey)) {
- if(HLogEdit.isDeleted(readval.get())) {
+ if(isDeleted(readkey, readval.get(), true, deletes)) {
break;
}
results.put(new Text(readcol), readval.get());
@@ -1598,6 +1618,8 @@
MapFile.Reader map = maparray[i];
synchronized(map) {
map.reset();
+
+ // do the priming read
ImmutableBytesWritable readval = new ImmutableBytesWritable();
HStoreKey readkey = (HStoreKey)map.getClosest(origin, readval);
if (readkey == null) {
@@ -1607,30 +1629,79 @@
// BEFORE.
continue;
}
- if (!readkey.matchesRowCol(origin)) {
- continue;
- }
- if (!isDeleted(readkey, readval.get(), false, null) &&
- !keys.contains(readkey)) {
- keys.add(new HStoreKey(readkey));
- }
- for (readval = new ImmutableBytesWritable();
- map.next(readkey, readval) &&
- readkey.matchesRowCol(origin);
- readval = new ImmutableBytesWritable()) {
- if (!isDeleted(readkey, readval.get(), false, null) &&
- !keys.contains(readkey)) {
- keys.add(new HStoreKey(readkey));
- if (versions != ALL_VERSIONS && keys.size() >= versions) {
- break;
+
+ do{
+ // if the row matches, we might want this one.
+ if(rowMatches(origin, readkey)){
+ // if the cell matches, then we definitely want this key.
+ if (cellMatches(origin, readkey)) {
+ // store the key if it isn't deleted or superceeded by what's
+ // in the memcache
+ if (!isDeleted(readkey, readval.get(), false, null) &&
+ !keys.contains(readkey)) {
+ keys.add(new HStoreKey(readkey));
+
+ // if we've collected enough versions, then exit the loop.
+ if (versions != ALL_VERSIONS && keys.size() >= versions) {
+ break;
+ }
+ }
+ } else {
+ // the cell doesn't match, but there might be more with different
+ // timestamps, so move to the next key
+ continue;
}
+ } else{
+ // the row doesn't match, so we've gone too far.
+ break;
}
- }
+ }while(map.next(readkey, readval)); // advance to the next key
}
}
+
return keys;
} finally {
this.lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Test that the <i>target</i> matches the <i>origin</i>. If the
+ * <i>origin</i> has an empty column, then it's assumed to mean any column
+ * matches and only match on row and timestamp. Otherwise, it compares the
+ * keys with HStoreKey.matchesRowCol().
+ * @param origin The key we're testing against
+ * @param target The key we're testing
+ */
+ private boolean cellMatches(HStoreKey origin, HStoreKey target){
+ // if the origin's column is empty, then we're matching any column
+ if (origin.getColumn().equals(new Text())){
+ // if the row matches, then...
+ if (target.getRow().equals(origin.getRow())) {
+ // check the timestamp
+ return target.getTimestamp() <= origin.getTimestamp();
+ } else {
+ return false;
+ }
+ } else { // otherwise, we want to match on row and column
+ return target.matchesRowCol(origin);
+ }
+ }
+
+ /**
+ * Test that the <i>target</i> matches the <i>origin</i>. If the <i>origin</i>
+ * has an empty column, then it just tests row equivalence. Otherwise, it uses
+ * HStoreKey.matchesRowCol().
+ * @param origin Key we're testing against
+ * @param target Key we're testing
+ */
+ private boolean rowMatches(HStoreKey origin, HStoreKey target){
+ // if the origin's column is empty, then we're matching any column
+ if (origin.getColumn().equals(new Text())){
+ // if the row matches, then...
+ return target.getRow().equals(origin.getRow());
+ } else { // otherwise, we want to match on row and column
+ return target.matchesRowCol(origin);
}
}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?rev=602284&r1=602283&r2=602284&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Fri Dec 7 16:39:32 2007
@@ -742,6 +742,53 @@
}
}
+ /**
+ * Completely delete the row's cells of the same timestamp or older.
+ *
+ * @param row Key of the row you want to completely delete.
+ * @param ts Timestamp of cells to delete
+ */
+ public void deleteAll(final Text row, long ts)
+ throws IOException {
+ checkClosed();
+ for(int tries = 0; tries < numRetries; tries++) {
+ HRegionLocation r = getRegionLocation(row);
+ HRegionInterface server =
+ connection.getHRegionConnection(r.getServerAddress());
+ try {
+ server.deleteAll(r.getRegionInfo().getRegionName(), row, ts);
+ break;
+
+ } catch (IOException e) {
+ if (e instanceof RemoteException) {
+ e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
+ }
+ if (tries == numRetries - 1) {
+ throw e;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("reloading table servers because: " + e.getMessage());
+ }
+ tableServers = connection.reloadTableServers(tableName);
+ }
+ try {
+ Thread.sleep(this.pause);
+ } catch (InterruptedException x) {
+ // continue
+ }
+ }
+ }
+
+ /**
+ * Completely delete the row's cells.
+ *
+ * @param row Key of the row you want to completely delete.
+ */
+ public void deleteAll(final Text row)
+ throws IOException {
+ deleteAll(row, HConstants.LATEST_TIMESTAMP);
+ }
+
/**
* Abort a row mutation.
*