You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/06/16 07:26:59 UTC
svn commit: r785084 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ src/java/
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/mapreduce/
src/java/org/apache/hadoop/hbase/regionserver/ src/test/o...
Author: apurtell
Date: Tue Jun 16 05:26:58 2009
New Revision: 785084
URL: http://svn.apache.org/viewvc?rev=785084&view=rev
Log:
HBASE-1528, HBASE-1447
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Tue Jun 16 05:26:58 2009
@@ -21,8 +21,8 @@
HBASE-1411 Remove HLogEdit.
HBASE-1357 If one sets the hbase.master to 0.0.0.0 non local regionservers
can't find the master
- HBASE-1304 New client server implementation of how gets and puts are handled
- (holstad, jgray, rawson, stack)
+ HBASE-1304 New client server implementation of how gets and puts are
+ handled (holstad, jgray, rawson, stack)
BUG FIXES
HBASE-1140 "ant clean test" fails (Nitay Joffe via Stack)
@@ -186,6 +186,15 @@
HBASE-1523 NPE in BaseScanner
HBASE-1525 HTable.incrementColumnValue hangs()
HBASE-1526 mapreduce fixup
+ HBASE-1503 hbase-1304 dropped updating list of store files on flush
+ (jgray via stack)
+ HBASE-1480 compaction file not cleaned up after a crash/OOME server
+ (Evgeny Ryabitskiy via Stack)
+ HBASE-1529 familyMap not invalidated when a Result is (re)read as a
+ Writable
+ HBASE-1528 Ensure scanners work across memcache snapshot
+ HBASE-1447 Take last version of the hbase-1249 design doc. and make
+ documentation out of it
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java Tue Jun 16 05:26:58 2009
@@ -949,8 +949,7 @@
byte [] result = new byte[fl + 1 + ql];
System.arraycopy(this.bytes, fo, result, 0, fl);
result[fl] = COLUMN_FAMILY_DELIMITER;
- System.arraycopy(this.bytes, fo + fl, result,
- fl + 1, ql);
+ System.arraycopy(this.bytes, fo + fl, result, fl + 1, ql);
return result;
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java Tue Jun 16 05:26:58 2009
@@ -35,95 +35,93 @@
<div style="background-color: #cccccc; padding: 2px">
<blockquote><pre>
-REPLACE!!!!!!!!
import java.io.IOException;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Scanner;
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.io.Cell;
-import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-public class MyClient {
- public static void main(String args[]) throws IOException {
+// Class that has nothing but a main.
+// Does a Put, Get and a Scan against an hbase table.
+public class MyLittleHBaseClient {
+ public static void main(String[] args) throws IOException {
// You need a configuration object to tell the client where to connect.
- // But don't worry, the defaults are pulled from the local config file.
+ // When you create a HBaseConfiguration, it reads in whatever you've set
+ // into your hbase-site.xml and in hbase-default.xml, as long as these can
+ // be found on the CLASSPATH
HBaseConfiguration config = new HBaseConfiguration();
- // This instantiates an HTable object that connects you to the "myTable"
- // table.
- HTable table = new HTable(config, "myTable");
-
- // To do any sort of update on a row, you use an instance of the BatchUpdate
- // class. A BatchUpdate takes a row and optionally a timestamp which your
- // updates will affect. If no timestamp, the server applies current time
- // to the edits.
- BatchUpdate batchUpdate = new BatchUpdate("myRow");
-
- // The BatchUpdate#put method takes a byte [] (or String) that designates
- // what cell you want to put a value into, and a byte array that is the
- // value you want to store. Note that if you want to store Strings, you
- // have to getBytes() from the String for HBase to store it since HBase is
- // all about byte arrays. The same goes for primitives like ints and longs
- // and user-defined classes - you must find a way to reduce it to bytes.
- // The Bytes class from the hbase util package has utility for going from
- // String to utf-8 bytes and back again and help for other base types.
- batchUpdate.put("myColumnFamily:columnQualifier1",
- Bytes.toBytes("columnQualifier1 value!"));
-
- // Deletes are batch operations in HBase as well.
- batchUpdate.delete("myColumnFamily:cellIWantDeleted");
-
- // Once you've done all the puts you want, you need to commit the results.
- // The HTable#commit method takes the BatchUpdate instance you've been
- // building and pushes the batch of changes you made into HBase.
- table.commit(batchUpdate);
+ // This instantiates an HTable object that connects you to
+ // the "myLittleHBaseTable" table.
+ HTable table = new HTable(config, "myLittleHBaseTable");
+
+ // To add to a row, use Put. A Put constructor takes the name of the row
+ // you want to insert into as a byte array. In HBase, the Bytes class has
+ // utility for converting all kinds of java types to byte arrays. In the
+ // below, we are converting the String "myLittleRow" into a byte array to
+ // use as a row key for our update. Once you have a Put instance, you can
+ // adorn it by setting the names of columns you want to update on the row,
+ // the timestamp to use in your update, etc.If no timestamp, the server
+ // applies current time to the edits.
+ Put p = new Put(Bytes.toBytes("myLittleRow"));
+
+ // To set the value you'd like to update in the row 'myRow', specify the
+ // column family, column qualifier, and value of the table cell you'd like
+ // to update. The column family must already exist in your table schema.
+ // The qualifier can be anything. All must be specified as byte arrays as
+ // hbase is all about byte arrays. Lets pretend the table
+ // 'myLittleHBaseTable' was created with a family 'myLittleFamily'.
+ p.add(Bytes.toBytes("myLittleFamily"), Bytes.toBytes("someQualifier"),
+ Bytes.toBytes("Some Value"));
+
+ // Once you've adorned your Put instance with all the updates you want to
+ // make, to commit it do the following (The HTable#put method takes the
+ // Put instance you've been building and pushes the changes you made into
+ // hbase)
+ table.put(p);
// Now, to retrieve the data we just wrote. The values that come back are
- // Cell instances. A Cell is a combination of the value as a byte array and
- // the timestamp the value was stored with. If you happen to know that the
- // value contained is a string and want an actual string, then you must
- // convert it yourself.
- Cell cell = table.get("myRow", "myColumnFamily:columnQualifier1");
- // This could throw a NullPointerException if there was no value at the cell
- // location.
- String valueStr = Bytes.toString(cell.getValue());
-
+ // Result instances. Generally, a Result is an object that will package up
+ // the hbase return into the form you find most palatable.
+ Get g = new Get(Bytes.toBytes("myLittleRow"));
+ Result r = table.get(g);
+ byte [] value = r.getValue(Bytes.toBytes("myLittleFamily"),
+ Bytes.toBytes("someQualifier"));
+ // If we convert the value bytes, we should get back 'Some Value', the
+ // value we inserted at this location.
+ String valueStr = Bytes.toString(value);
+ System.out.println("GET: " + valueStr);
+
// Sometimes, you won't know the row you're looking for. In this case, you
// use a Scanner. This will give you cursor-like interface to the contents
- // of the table.
- Scanner scanner =
- // we want to get back only "myColumnFamily:columnQualifier1" when we iterate
- table.getScanner(new String[]{"myColumnFamily:columnQualifier1"});
-
-
- // Scanners return RowResult instances. A RowResult is like the
- // row key and the columns all wrapped up in a single Object.
- // RowResult#getRow gives you the row key. RowResult also implements
- // Map, so you can get to your column results easily.
-
- // Now, for the actual iteration. One way is to use a while loop like so:
- RowResult rowResult = scanner.next();
-
- while (rowResult != null) {
- // print out the row we found and the columns we were looking for
- System.out.println("Found row: " + Bytes.toString(rowResult.getRow()) +
- " with value: " + rowResult.get(Bytes.toBytes("myColumnFamily:columnQualifier1")));
- rowResult = scanner.next();
- }
-
- // The other approach is to use a foreach loop. Scanners are iterable!
- for (RowResult result : scanner) {
- // print out the row we found and the columns we were looking for
- System.out.println("Found row: " + Bytes.toString(rowResult.getRow()) +
- " with value: " + rowResult.get(Bytes.toBytes("myColumnFamily:columnQualifier1")));
+ // of the table. To set up a Scanner, do like you did above making a Put
+ // and a Get, create a Scan. Adorn it with column names, etc.
+ Scan s = new Scan();
+ s.addColumn(Bytes.toBytes("myLittleFamily"), Bytes.toBytes("someQualifier"));
+ ResultScanner scanner = table.getScanner(s);
+ try {
+ // Scanners return Result instances.
+ // Now, for the actual iteration. One way is to use a while loop like so:
+ for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
+ // print out the row we found and the columns we were looking for
+ System.out.println("Found row: " + rr);
+ }
+
+ // The other approach is to use a foreach loop. Scanners are iterable!
+ // for (Result rr : scanner) {
+ // System.out.println("Found row: " + rr);
+ // }
+ } finally {
+ // Make sure you close your scanners when you are done!
+ // Thats why we have it inside a try/finally clause
+ scanner.close();
}
-
- // Make sure you close your scanners when you are done!
- // Its probably best to put the iteration into a try/finally with the below
- // inside the finally clause.
- scanner.close();
}
}
</pre></blockquote>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java Tue Jun 16 05:26:58 2009
@@ -142,7 +142,7 @@
}
/**
- * @param inputColumns the columns to be placed in {@link RowResult}.
+ * @param inputColumns the columns to be placed in {@link Result}.
*/
public void setInputColumns(final byte [][] inputColumns) {
this.trrInputColumns = inputColumns;
@@ -304,7 +304,7 @@
}
/**
- * @param inputColumns to be passed in {@link RowResult} to the map task.
+ * @param inputColumns to be passed in {@link Result} to the map task.
*/
protected void setInputColumns(byte [][] inputColumns) {
this.inputColumns = inputColumns;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java Tue Jun 16 05:26:58 2009
@@ -47,7 +47,6 @@
/**
* Constructor
- * @param comparator
*/
public GetDeleteTracker() {}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Jun 16 05:26:58 2009
@@ -159,7 +159,7 @@
volatile boolean writesEnabled = true;
// Set if region is read-only
volatile boolean readOnly = false;
-
+
/**
* Set flags that make this region read-only.
*/
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Tue Jun 16 05:26:58 2009
@@ -88,9 +88,9 @@
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler;
import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
+import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.ipc.HBaseRPC.Server;
import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -167,7 +167,7 @@
// Server to handle client requests. Default access so can be accessed by
// unit tests.
- Server server;
+ HBaseServer server;
// Leases
private Leases leases;
@@ -1296,7 +1296,7 @@
master = (HMasterRegionInterface)HBaseRPC.waitForProxy(
HMasterRegionInterface.class, HBaseRPCProtocolVersion.versionID,
masterAddress.getInetSocketAddress(),
- this.conf, -1);
+ this.conf, -1, this.rpcTimeout);
} catch (IOException e) {
LOG.warn("Unable to connect to master. Retrying. Error was:", e);
sleeper.sleep();
@@ -1857,7 +1857,6 @@
checkOpen();
List<Result> results = new ArrayList<Result>();
try {
- long start = System.currentTimeMillis();
String scannerName = String.valueOf(scannerId);
InternalScanner s = scanners.get(scannerName);
if (s == null) {
@@ -1881,10 +1880,7 @@
throw convertThrowableToIOE(cleanup(t));
}
}
-
-
-
-
+
public void close(final long scannerId) throws IOException {
try {
checkOpen();
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/KeyValueHeap.java Tue Jun 16 05:26:58 2009
@@ -38,16 +38,13 @@
* <p>
* In the Region case, we also need InternalScanner.next(List), so this class
* also implements InternalScanner. WARNING: As is, if you try to use this
- * as an InternalScanner at the Store level, you will get runtime exceptions.
+ * as an InternalScanner at the Store level, you will get runtime exceptions.
*/
public class KeyValueHeap implements KeyValueScanner, InternalScanner {
-
private PriorityQueue<KeyValueScanner> heap;
-
private KeyValueScanner current = null;
-
private KVScannerComparator comparator;
-
+
/**
* Constructor
* @param scanners
@@ -91,7 +88,7 @@
}
return kvReturn;
}
-
+
/**
* Gets the next row of keys from the top-most scanner.
* <p>
@@ -194,4 +191,4 @@
public PriorityQueue<KeyValueScanner> getHeap() {
return this.heap;
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java Tue Jun 16 05:26:58 2009
@@ -539,12 +539,15 @@
}
/**
- * @return a scanner over the keys in the Memcache
+ * @return scanner on memcache and snapshot in this order.
*/
- KeyValueScanner getScanner() {
+ KeyValueScanner [] getScanners() {
this.lock.readLock().lock();
try {
- return new MemcacheScanner();
+ KeyValueScanner [] scanners = new KeyValueScanner[2];
+ scanners[0] = new MemcacheScanner(this.memcache);
+ scanners[1] = new MemcacheScanner(this.snapshot);
+ return scanners;
} finally {
this.lock.readLock().unlock();
}
@@ -553,7 +556,7 @@
//
// HBASE-880/1249/1304
//
-
+
/**
* Perform a single-row Get on the memcache and snapshot, placing results
* into the specified KV list.
@@ -618,71 +621,78 @@
return false;
}
- //////////////////////////////////////////////////////////////////////////////
- // MemcacheScanner implements the KeyValueScanner.
- // It lets the caller scan the contents of the Memcache.
- // This behaves as if it were a real scanner but does not maintain position
- // in the Memcache tree.
- //////////////////////////////////////////////////////////////////////////////
+ /*
+ * MemcacheScanner implements the KeyValueScanner.
+ * It lets the caller scan the contents of a memcache.
+ * This behaves as if it were a real scanner but does not maintain position
+ * in the passed memcache tree.
+ */
protected class MemcacheScanner implements KeyValueScanner {
+ private final NavigableSet<KeyValue> mc;
private KeyValue current = null;
private List<KeyValue> result = new ArrayList<KeyValue>();
private int idx = 0;
-
- MemcacheScanner() {}
-
+
+ MemcacheScanner(final NavigableSet<KeyValue> mc) {
+ this.mc = mc;
+ }
+
public boolean seek(KeyValue key) {
try {
- if(key == null) {
+ if (key == null) {
close();
return false;
}
- current = key;
+ this.current = key;
return cacheNextRow();
} catch(Exception e) {
close();
return false;
}
}
-
+
public KeyValue peek() {
- if(idx >= result.size()) {
- if(!cacheNextRow()) {
+ if (idx >= this.result.size()) {
+ if (!cacheNextRow()) {
return null;
}
return peek();
}
return result.get(idx);
}
-
+
public KeyValue next() {
- if(idx >= result.size()) {
- if(!cacheNextRow()) {
+ if (idx >= result.size()) {
+ if (!cacheNextRow()) {
return null;
}
return next();
}
- return result.get(idx++);
+ return this.result.get(idx++);
}
-
+
+ /**
+ * @return True if we successfully cached a NavigableSet aligned on
+ * next row.
+ */
boolean cacheNextRow() {
- NavigableSet<KeyValue> keys;
+ SortedSet<KeyValue> keys;
try {
- keys = memcache.tailSet(current);
- } catch(Exception e) {
+ keys = this.mc.tailSet(this.current);
+ } catch (Exception e) {
close();
return false;
}
- if(keys == null || keys.isEmpty()) {
+ if (keys == null || keys.isEmpty()) {
close();
return false;
}
- current = null;
+ this.current = null;
byte [] row = keys.first().getRow();
- for(KeyValue key : keys) {
- if(comparator.compareRows(key, row) != 0) {
- current = key;
+ for (KeyValue key: keys) {
+ if (comparator.compareRows(key, row) != 0) {
+ this.current = key;
break;
}
result.add(key);
@@ -693,7 +703,7 @@
public void close() {
current = null;
idx = 0;
- if(!result.isEmpty()) {
+ if (!result.isEmpty()) {
result.clear();
}
}
@@ -741,4 +751,4 @@
}
LOG.info("Exiting.");
}
-}
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MinorCompactingStoreScanner.java Tue Jun 16 05:26:58 2009
@@ -82,7 +82,7 @@
/**
* High performance merge scan.
* @param writer
- * @return
+ * @return True if more.
* @throws IOException
*/
public boolean next(HFile.Writer writer) throws IOException {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Jun 16 05:26:58 2009
@@ -126,7 +126,7 @@
// reflected in the TreeMaps).
private volatile long maxSeqId = -1;
- private final Path compactionDir;
+ private final Path regionCompactionDir;
private final Object compactLock = new Object();
private final int compactionThreshold;
private final int blocksize;
@@ -180,7 +180,8 @@
this.ttl *= 1000;
}
this.memcache = new Memcache(this.ttl, this.comparator);
- this.compactionDir = HRegion.getCompactionDir(basedir);
+ this.regionCompactionDir = new Path(HRegion.getCompactionDir(basedir),
+ Integer.toString(info.getEncodedName()));
this.storeName = this.family.getName();
this.storeNameStr = Bytes.toString(this.storeName);
@@ -653,8 +654,8 @@
(forceSplit || (filesToCompact.size() < compactionThreshold))) {
return checkSplit(forceSplit);
}
- if (!fs.exists(this.compactionDir) && !fs.mkdirs(this.compactionDir)) {
- LOG.warn("Mkdir on " + this.compactionDir.toString() + " failed");
+ if (!fs.exists(this.regionCompactionDir) && !fs.mkdirs(this.regionCompactionDir)) {
+ LOG.warn("Mkdir on " + this.regionCompactionDir.toString() + " failed");
return checkSplit(forceSplit);
}
@@ -707,7 +708,7 @@
}
// Step through them, writing to the brand-new file
- HFile.Writer writer = getWriter(this.compactionDir);
+ HFile.Writer writer = getWriter(this.regionCompactionDir);
if (LOG.isDebugEnabled()) {
LOG.debug("Started compaction of " + filesToCompact.size() + " file(s)" +
(references? ", hasReferences=true,": " ") + " into " +
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Tue Jun 16 05:26:58 2009
@@ -26,7 +26,6 @@
import java.util.Map;
import java.util.NavigableSet;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -38,19 +37,12 @@
* Scanner scans both the memcache and the HStore. Coaleace KeyValue stream
* into List<KeyValue> for a single row.
*/
-class StoreScanner implements KeyValueScanner, InternalScanner,
-ChangedReadersObserver {
+class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersObserver {
static final Log LOG = LogFactory.getLog(StoreScanner.class);
-
private Store store;
-
private ScanQueryMatcher matcher;
-
private KeyValueHeap heap;
- // Used around transition from no storefile to the first.
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-
// Used to indicate that the scanner has closed (see HBASE-1107)
private final AtomicBoolean closing = new AtomicBoolean(false);
@@ -63,8 +55,7 @@
columns, store.ttl, store.comparator.getRawComparator(),
store.versionsToReturn(scan.getMaxVersions()));
- List<KeyValueScanner> scanners = getStoreFileScanners();
- scanners.add(store.memcache.getScanner());
+ List<KeyValueScanner> scanners = getScanners();
// Seek all scanners to the initial key
for(KeyValueScanner scanner : scanners) {
@@ -79,10 +70,10 @@
}
// Constructor for testing.
- StoreScanner(Scan scan, byte [] colFamily,
- long ttl, KeyValue.KVComparator comparator,
+ StoreScanner(final Scan scan, final byte [] colFamily, final long ttl,
+ final KeyValue.KVComparator comparator,
final NavigableSet<byte[]> columns,
- KeyValueScanner [] scanners) {
+ final KeyValueScanner [] scanners) {
this.store = null;
this.matcher = new ScanQueryMatcher(scan, colFamily, columns, ttl,
comparator.getRawComparator(), scan.getMaxVersions());
@@ -91,12 +82,22 @@
for(KeyValueScanner scanner : scanners) {
scanner.seek(matcher.getStartKey());
}
+ heap = new KeyValueHeap(scanners, comparator);
+ }
- heap = new KeyValueHeap(
- scanners, comparator);
+ /*
+ * @return List of scanners ordered properly.
+ */
+ private List<KeyValueScanner> getScanners() {
+ List<KeyValueScanner> scanners = getStoreFileScanners();
+ KeyValueScanner [] memcachescanners = this.store.memcache.getScanners();
+ for (int i = memcachescanners.length - 1; i >= 0; i--) {
+ scanners.add(memcachescanners[i]);
+ }
+ return scanners;
}
- public KeyValue peek() {
+ public synchronized KeyValue peek() {
return this.heap.peek();
}
@@ -105,7 +106,7 @@
throw new RuntimeException("Never call StoreScanner.next()");
}
- public void close() {
+ public synchronized void close() {
this.closing.set(true);
// under test, we dont have a this.store
if (this.store != null)
@@ -113,8 +114,7 @@
this.heap.close();
}
- public boolean seek(KeyValue key) {
-
+ public synchronized boolean seek(KeyValue key) {
return this.heap.seek(key);
}
@@ -123,7 +123,7 @@
* @param result
* @return true if there are more rows, false if scanner is done
*/
- public boolean next(List<KeyValue> result) throws IOException {
+ public synchronized boolean next(List<KeyValue> result) throws IOException {
KeyValue peeked = this.heap.peek();
if (peeked == null) {
close();
@@ -132,12 +132,13 @@
matcher.setRow(peeked.getRow());
KeyValue kv;
while((kv = this.heap.peek()) != null) {
- QueryMatcher.MatchCode mc = matcher.match(kv);
- switch(mc) {
+ QueryMatcher.MatchCode qcode = matcher.match(kv);
+ switch(qcode) {
case INCLUDE:
KeyValue next = this.heap.next();
result.add(next);
continue;
+
case DONE:
// what happens if we have 0 results?
if (result.isEmpty()) {
@@ -159,14 +160,6 @@
return false;
case SEEK_NEXT_ROW:
- // TODO see comments in SEEK_NEXT_COL
- /*
- KeyValue rowToSeek =
- new KeyValue(kv.getRow(),
- 0,
- KeyValue.Type.Minimum);
- heap.seek(rowToSeek);
- */
heap.next();
break;
@@ -174,51 +167,15 @@
// TODO hfile needs 'hinted' seeking to prevent it from
// reseeking from the start of the block on every dang seek.
// We need that API and expose it the scanner chain.
- /*
- ColumnCount hint = matcher.getSeekColumn();
- KeyValue colToSeek;
- if (hint == null) {
- // seek to the 'last' key on this column, this is defined
- // as the key with the same row, fam, qualifier,
- // smallest timestamp, largest type.
- colToSeek =
- new KeyValue(kv.getRow(),
- kv.getFamily(),
- kv.getColumn(),
- Long.MIN_VALUE,
- KeyValue.Type.Minimum);
- } else {
- // This is ugmo. Move into KeyValue convience method.
- // First key on a column is:
- // same row, cf, qualifier, max_timestamp, max_type, no value.
- colToSeek =
- new KeyValue(kv.getRow(),
- 0,
- kv.getRow().length,
-
- kv.getFamily(),
- 0,
- kv.getFamily().length,
-
- hint.getBuffer(),
- hint.getOffset(),
- hint.getLength(),
-
- Long.MAX_VALUE,
- KeyValue.Type.Maximum,
- null,
- 0,
- 0);
- }
- heap.seek(colToSeek);
- */
-
heap.next();
break;
case SKIP:
this.heap.next();
break;
+
+ default:
+ throw new RuntimeException("UNEXPECTED");
}
}
if(result.size() > 0) {
@@ -245,18 +202,23 @@
}
// Implementation of ChangedReadersObserver
- public void updateReaders() throws IOException {
- if (this.closing.get()) {
- return;
- }
- this.lock.writeLock().lock();
- try {
- // Could do this pretty nicely with KeyValueHeap, but the existing
- // implementation of this method only updated if no existing storefiles?
- // Lets discuss.
- return;
- } finally {
- this.lock.writeLock().unlock();
+ public synchronized void updateReaders() throws IOException {
+ if (this.closing.get()) return;
+ KeyValue topKey = this.peek();
+ if (topKey == null) return;
+ List<KeyValueScanner> scanners = getScanners();
+
+ // Seek all scanners to the initial key
+ for(KeyValueScanner scanner : scanners) {
+ scanner.seek(topKey);
}
+
+ // Combine all seeked scanners with a heap
+ heap = new KeyValueHeap(
+ scanners.toArray(new KeyValueScanner[scanners.size()]), store.comparator);
+
+ // Reset the state of the Query Matcher and set to top row
+ matcher.reset();
+ matcher.setRow(heap.peek().getRow());
}
}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html Tue Jun 16 05:26:58 2009
@@ -35,6 +35,14 @@
ssh must be installed and sshd must be running to use Hadoop's
scripts to manage remote Hadoop daemons.
</li>
+ <li>HBase depends on <a href="http://hadoop.apache.org/zookeeper/">ZooKeeper</a> as of release 0.20.0.
+ Clients and Servers now must know where their ZooKeeper Quorum locations before
+ they can do anything else.
+ In basic standalone and pseudo-distributed modes, HBase manages a ZooKeeper instance
+ for you but it is required that you run a ZooKeeper Quorum when running HBase
+ fully distributed (More on this below). The Zookeeper addition changes
+ how some core HBase configuration is done.
+ </li>
<li>HBase currently is a file handle hog. The usual default of
1024 on *nix systems is insufficient if you are loading any significant
amount of data into regionservers. See the
@@ -46,11 +54,6 @@
<li>The clocks on cluster members should be in basic alignments. Some skew is tolerable but
wild skew can generate odd behaviors. Run <a href="http://en.wikipedia.org/wiki/Network_Time_Protocol">NTP</a>
on your cluster, or an equivalent.</li>
- <li>HBase depends on <a href="http://hadoop.apache.org/zookeeper/">ZooKeeper</a> as of release 0.20.0.
- In basic standalone and pseudo-distributed modes, HBase manages a ZooKeeper instance
- for you but it is required that you run a ZooKeeper Quorum when running HBase
- fully distributed (More on this below).
- </li>
<li>This is a list of patches we recommend you apply to your running Hadoop cluster:
<ul>
<li><a hef="https://issues.apache.org/jira/browse/HADOOP-4681">HADOOP-4681 <i>"DFSClient block read failures cause open DFSInputStream to become unusable"</i></a>. This patch will help with the ever-popular, "No live nodes contain current block".
@@ -86,6 +89,9 @@
see <a href="http://hadoop.apache.org/hbase/releases.html">Releases</a>, and are installing
for the first time. If upgrading your
HBase instance, see <a href="#upgrading">Upgrading</a>.
+If you have used HBase in the past,
+please read carefully. Some core configuration has changed in 0.20.x HBase.
+</p>
<p>Three modes are described: standalone, pseudo-distributed (where all servers are run on
a single host), and distributed. If new to hbase start by following the standalone instruction.
</p>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Tue Jun 16 05:26:58 2009
@@ -225,6 +225,7 @@
startKeyBytes = START_KEY_BYTES;
}
return addContent(new HRegionIncommon(r), Bytes.toString(column),
+ null,
startKeyBytes, endKey, -1);
}
@@ -237,11 +238,16 @@
* @throws IOException
* @return count of what we added.
*/
- protected static long addContent(final Incommon updater, final String column)
- throws IOException {
+ protected static long addContent(final Incommon updater,
+ final String column) throws IOException {
return addContent(updater, column, START_KEY_BYTES, null);
}
+ protected static long addContent(final Incommon updater, final String family,
+ final String column) throws IOException {
+ return addContent(updater, family, column, START_KEY_BYTES, null);
+ }
+
/**
* Add content to region <code>r</code> on the passed column
* <code>column</code>.
@@ -256,7 +262,13 @@
protected static long addContent(final Incommon updater, final String column,
final byte [] startKeyBytes, final byte [] endKey)
throws IOException {
- return addContent(updater, column, startKeyBytes, endKey, -1);
+ return addContent(updater, column, null, startKeyBytes, endKey, -1);
+ }
+
+ protected static long addContent(final Incommon updater, final String family,
+ final String column, final byte [] startKeyBytes,
+ final byte [] endKey) throws IOException {
+ return addContent(updater, family, column, startKeyBytes, endKey, -1);
}
/**
@@ -271,7 +283,8 @@
* @return count of what we added.
* @throws IOException
*/
- protected static long addContent(final Incommon updater, final String column,
+ protected static long addContent(final Incommon updater, final String columnFamily,
+ final String column,
final byte [] startKeyBytes, final byte [] endKey, final long ts)
throws IOException {
long count = 0;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MultiRegionTable.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MultiRegionTable.java Tue Jun 16 05:26:58 2009
@@ -56,7 +56,7 @@
Bytes.toBytes("yyy")
};
- protected final byte [] columnName;
+ protected final byte [] columnFamily;
protected HTableDescriptor desc;
/**
@@ -64,7 +64,7 @@
*/
public MultiRegionTable(final String columnName) {
super();
- this.columnName = Bytes.toBytes(columnName);
+ this.columnFamily = Bytes.toBytes(columnName);
// These are needed for the new and improved Map/Reduce framework
System.setProperty("hadoop.log.dir", conf.get("hadoop.log.dir"));
conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
@@ -101,7 +101,7 @@
private HRegion createARegion(byte [] startKey, byte [] endKey) throws IOException {
HRegion region = createNewHRegion(desc, startKey, endKey);
- addContent(region, this.columnName);
+ addContent(region, this.columnFamily);
closeRegionAndDeleteLog(region);
return region;
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java Tue Jun 16 05:26:58 2009
@@ -1,3 +1,23 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hbase.client;
import java.io.IOException;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java Tue Jun 16 05:26:58 2009
@@ -41,6 +41,8 @@
public class TestCompaction extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
private HRegion r = null;
+ private Path compactionDir = null;
+ private Path regionCompactionDir = null;
private static final byte [] COLUMN_FAMILY = fam1;
private final byte [] STARTROW = Bytes.toBytes(START_KEY);
private static final byte [] COLUMN_FAMILY_TEXT = COLUMN_FAMILY;
@@ -67,6 +69,9 @@
super.setUp();
HTableDescriptor htd = createTableDescriptor(getName());
this.r = createNewHRegion(htd, null, null);
+ this.compactionDir = HRegion.getCompactionDir(this.r.getBaseDir());
+ this.regionCompactionDir = new Path(this.compactionDir,
+ Integer.toString(this.r.getRegionInfo().getEncodedName()));
}
@Override
@@ -102,10 +107,14 @@
Result result = r.get(new Get(STARTROW).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
// Assert that I can get 3 versions since it is the max I should get
- assertEquals(3, result.size());
+ assertEquals(COMPACTION_THRESHOLD, result.size());
// assertEquals(cellValues.length, 3);
r.flushcache();
r.compactStores();
+ // check compaction dir is exists
+ assertTrue(this.cluster.getFileSystem().exists(this.compactionDir));
+ // check Compaction Dir for this Regions is cleaned up
+ assertTrue(!this.cluster.getFileSystem().exists(this.regionCompactionDir));
// Always 3 versions if that is what max versions is.
byte [] secondRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
// Increment the least significant character so we get to next row.
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java Tue Jun 16 05:26:58 2009
@@ -27,32 +27,26 @@
import java.util.Set;
import java.util.TreeSet;
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.KeyValueTestUtil;
import org.apache.hadoop.hbase.util.Bytes;
-import junit.framework.TestCase;
-
/** memcache test case */
public class TestMemcache extends TestCase {
-
+ private final Log LOG = LogFactory.getLog(this.getClass());
private Memcache memcache;
-
private static final int ROW_COUNT = 10;
-
private static final int QUALIFIER_COUNT = 10;
-
private static final byte [] FAMILY = Bytes.toBytes("column");
-
- private static final int FIRST_ROW = 1;
- private static final int NUM_VALS = 1000;
private static final byte [] CONTENTS_BASIC = Bytes.toBytes("contents:basic");
private static final String CONTENTSTR = "contentstr";
- private static final String ANCHORNUM = "anchor:anchornum-";
- private static final String ANCHORSTR = "anchorstr";
@Override
public void setUp() throws Exception {
@@ -60,6 +54,65 @@
this.memcache = new Memcache();
}
+ /**
+ * Test memcache snapshot happening while scanning.
+ * @throws IOException
+ */
+ public void testScanAcrossSnapshot() throws IOException {
+ int rowCount = addRows(this.memcache);
+ KeyValueScanner [] memcachescanners = this.memcache.getScanners();
+ Scan scan = new Scan();
+ List<KeyValue> result = new ArrayList<KeyValue>();
+ StoreScanner s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
+ this.memcache.comparator, null, memcachescanners);
+ int count = 0;
+ try {
+ while (s.next(result)) {
+ LOG.info(result);
+ count++;
+ result.clear();
+ }
+ } finally {
+ s.close();
+ }
+ assertEquals(rowCount, count);
+ // Now assert can count same number even if a snapshot mid-scan.
+ s = new StoreScanner(scan, null, HConstants.LATEST_TIMESTAMP,
+ this.memcache.comparator, null, memcachescanners);
+ count = 0;
+ try {
+ while (s.next(result)) {
+ LOG.info(result);
+ // Assert the stuff is coming out in right order.
+ assertTrue(Bytes.compareTo(Bytes.toBytes(count), result.get(0).getRow()) == 0);
+ count++;
+ if (count == 2) {
+ this.memcache.snapshot();
+ LOG.info("Snapshotted");
+ }
+ result.clear();
+ }
+ } finally {
+ s.close();
+ }
+ assertEquals(rowCount, count);
+ }
+
+ /**
+ * Test memcache snapshots
+ * @throws IOException
+ */
+ public void testSnapshotting() throws IOException {
+ final int snapshotCount = 5;
+ // Add some rows, run a snapshot. Do it a few times.
+ for (int i = 0; i < snapshotCount; i++) {
+ addRows(this.memcache);
+ runSnapshot(this.memcache);
+ Set<KeyValue> ss = this.memcache.getSnapshot();
+ assertEquals("History not being cleared", 0, ss.size());
+ }
+ }
+
public void testMultipleVersionsSimple() throws Exception {
Memcache m = new Memcache(HConstants.FOREVER, KeyValue.COMPARATOR);
byte [] row = Bytes.toBytes("testRow");
@@ -112,23 +165,6 @@
}
}
-
- /**
- * Test memcache snapshots
- * @throws IOException
- */
- public void testSnapshotting() throws IOException {
- final int snapshotCount = 5;
- // Add some rows, run a snapshot. Do it a few times.
- for (int i = 0; i < snapshotCount; i++) {
- addRows(this.memcache);
- runSnapshot(this.memcache);
- Set<KeyValue> ss = this.memcache.getSnapshot();
- assertEquals("History not being cleared", 0, ss.size());
- }
- }
-
-
//////////////////////////////////////////////////////////////////////////////
// Get tests
//////////////////////////////////////////////////////////////////////////////
@@ -218,7 +254,7 @@
InternalScanner scanner =
new StoreScanner(new Scan(Bytes.toBytes(startRowId)), FAMILY,
Integer.MAX_VALUE, this.memcache.comparator, null,
- new KeyValueScanner[]{memcache.getScanner()});
+ new KeyValueScanner[]{memcache.getScanners()[0]});
List<KeyValue> results = new ArrayList<KeyValue>();
for (int i = 0; scanner.next(results); i++) {
int rowId = startRowId + i;
@@ -550,9 +586,10 @@
/**
* Adds {@link #ROW_COUNT} rows and {@link #COLUMNS_COUNT}
* @param hmc Instance to add rows to.
+ * @return How many rows we added.
* @throws IOException
*/
- private void addRows(final Memcache hmc) {
+ private int addRows(final Memcache hmc) {
for (int i = 0; i < ROW_COUNT; i++) {
long timestamp = System.currentTimeMillis();
for (int ii = 0; ii < QUALIFIER_COUNT; ii++) {
@@ -561,6 +598,7 @@
hmc.add(new KeyValue(row, FAMILY, qf, timestamp, qf));
}
}
+ return ROW_COUNT;
}
private void runSnapshot(final Memcache hmc) throws UnexpectedException {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=785084&r1=785083&r2=785084&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Tue Jun 16 05:26:58 2009
@@ -361,20 +361,19 @@
}
/**
+ * Tests to do a sync flush during the middle of a scan. This is testing the StoreScanner
+ * update readers code essentially. This is not highly concurrent, since its all 1 thread.
* HBase-910.
* @throws Exception
*/
- public void testScanAndConcurrentFlush() throws Exception {
+ public void testScanAndSyncFlush() throws Exception {
this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
HRegionIncommon hri = new HRegionIncommon(r);
try {
- String columnString = Bytes.toString(HConstants.CATALOG_FAMILY) + ':' +
- Bytes.toString(HConstants.REGIONINFO_QUALIFIER);
- LOG.info("Added: " + addContent(hri, columnString));
- int count = count(hri, -1);
- assertEquals(count, count(hri, 100));
- assertEquals(count, count(hri, 0));
- assertEquals(count, count(hri, count - 1));
+ LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
+ Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
+ int count = count(hri, -1, false);
+ assertEquals(count, count(hri, 100, false)); // do a sync flush.
} catch (Exception e) {
LOG.error("Failed", e);
throw e;
@@ -384,26 +383,73 @@
shutdownDfs(cluster);
}
}
+
+ /**
+ * Tests to do a concurrent flush (using a 2nd thread) while scanning. This tests both
+ * the StoreScanner update readers and the transition from memcache -> snapshot -> store file.
+ *
+ * @throws Exception
+ */
+ public void testScanAndRealConcurrentFlush() throws Exception {
+ this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
+ HRegionIncommon hri = new HRegionIncommon(r);
+ try {
+ LOG.info("Added: " + addContent(hri, Bytes.toString(HConstants.CATALOG_FAMILY),
+ Bytes.toString(HConstants.REGIONINFO_QUALIFIER)));
+ int count = count(hri, -1, false);
+ assertEquals(count, count(hri, 100, true)); // do a true concurrent background thread flush
+ } catch (Exception e) {
+ LOG.error("Failed", e);
+ throw e;
+ } finally {
+ this.r.close();
+ this.r.getLog().closeAndDelete();
+ shutdownDfs(cluster);
+ }
+ }
+
/*
* @param hri Region
* @param flushIndex At what row we start the flush.
+ * @param concurrent if the flush should be concurrent or sync.
* @return Count of rows found.
* @throws IOException
*/
- private int count(final HRegionIncommon hri, final int flushIndex)
+ private int count(final HRegionIncommon hri, final int flushIndex,
+ boolean concurrent)
throws IOException {
LOG.info("Taking out counting scan");
ScannerIncommon s = hri.getScanner(HConstants.CATALOG_FAMILY, EXPLICIT_COLS,
HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
List<KeyValue> values = new ArrayList<KeyValue>();
int count = 0;
+ boolean justFlushed = false;
while (s.next(values)) {
+ if (justFlushed) {
+ LOG.info("after next() just after next flush");
+ justFlushed=false;
+ }
count++;
if (flushIndex == count) {
LOG.info("Starting flush at flush index " + flushIndex);
- hri.flushcache();
- LOG.info("Finishing flush");
+ Thread t = new Thread() {
+ public void run() {
+ try {
+ hri.flushcache();
+ LOG.info("Finishing flush");
+ } catch (IOException e) {
+ LOG.info("Failed flush cache");
+ }
+ }
+ };
+ if (concurrent) {
+ t.start(); // concurrently flush.
+ } else {
+ t.run(); // sync flush
+ }
+ LOG.info("Continuing on after kicking off background flush");
+ justFlushed = true;
}
}
s.close();