You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/06/21 00:29:05 UTC
svn commit: r786919 [2/2] - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ bin/
conf/ src/java/ src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/filter/
src/java/org/apache/hadoop/hbase/io/ src/ja...
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/ClassSize.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/ClassSize.java?rev=786919&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/ClassSize.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/ClassSize.java Sat Jun 20 22:29:03 2009
@@ -0,0 +1,195 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.io.HeapSize;
+
+/**
+ * Class for determining the "size" of a class, an attempt to calculate the
+ * actual bytes that an object of this class will occupy in memory
+ *
+ * The core of this class is taken from the Derby project
+ */
+public class ClassSize {
+ static final Log LOG = LogFactory.getLog(ClassSize.class);
+
+ private int refSize;
+ private int minObjectSize;
+
+ /**
+ * Constructor
+ * @throws Exception
+ */
+ public ClassSize() throws Exception{
+ // Figure out whether this is a 32 or 64 bit machine.
+ Runtime runtime = Runtime.getRuntime();
+ int loops = 10;
+ int sz = 0;
+ for(int i = 0; i < loops; i++) {
+ cleaner(runtime, i);
+ long memBase = runtime.totalMemory() - runtime.freeMemory();
+ Object[] junk = new Object[10000];
+ cleaner(runtime, i);
+ long memUsed = runtime.totalMemory() - runtime.freeMemory() - memBase;
+ sz = (int)((memUsed + junk.length/2)/junk.length);
+ if(sz > 0 ) {
+ break;
+ }
+ }
+
+ refSize = ( 4 > sz) ? 4 : sz;
+ minObjectSize = 4*refSize;
+ }
+
+ /**
+ * The estimate of the size of a class instance depends on whether the JVM
+ * uses 32 or 64 bit addresses, that is it depends on the size of an object
+ * reference. It is a linear function of the size of a reference, e.g.
+ * 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes).
+ *
+ * This method returns the coefficients of the linear function, e.g. {24, 5}
+ * in the above example.
+ *
+ * @param cl A class whose instance size is to be estimated
+ * @return an array of 3 integers. The first integer is the size of the
+ * primitives, the second the number of arrays and the third the number of
+ * references.
+ */
+ private int [] getSizeCoefficients(Class cl, boolean debug) {
+ int primitives = 0;
+ int arrays = 0;
+ int references = HeapSize.OBJECT / HeapSize.REFERENCE;
+
+ for( ; null != cl; cl = cl.getSuperclass()) {
+ Field[] field = cl.getDeclaredFields();
+ if( null != field) {
+ for( int i = 0; i < field.length; i++) {
+ if( ! Modifier.isStatic( field[i].getModifiers())) {
+ Class fieldClass = field[i].getType();
+ if( fieldClass.isArray()){
+ arrays++;
+ }
+ else if(! fieldClass.isPrimitive()){
+ references++;
+ }
+ else {// Is simple primitive
+ String name = fieldClass.getName();
+
+ if(name.equals("int") || name.equals( "I"))
+ primitives += Bytes.SIZEOF_INT;
+ else if(name.equals("long") || name.equals( "J"))
+ primitives += Bytes.SIZEOF_LONG;
+ else if(name.equals("boolean") || name.equals( "Z"))
+ primitives += Bytes.SIZEOF_BOOLEAN;
+ else if(name.equals("short") || name.equals( "S"))
+ primitives += Bytes.SIZEOF_SHORT;
+ else if(name.equals("byte") || name.equals( "B"))
+ primitives += Bytes.SIZEOF_BYTE;
+ else if(name.equals("char") || name.equals( "C"))
+ primitives += Bytes.SIZEOF_CHAR;
+ else if(name.equals("float") || name.equals( "F"))
+ primitives += Bytes.SIZEOF_FLOAT;
+ else if(name.equals("double") || name.equals( "D"))
+ primitives += Bytes.SIZEOF_DOUBLE;
+ }
+ if(debug) {
+ if (LOG.isDebugEnabled()) {
+ // Write out region name as string and its encoded name.
+ LOG.debug(field[i].getName()+ "\n\t" +field[i].getType());
+ }
+ }
+ }
+ }
+ }
+ }
+ return new int [] {primitives, arrays, references};
+ }
+
+ /**
+ * Estimate the static space taken up by a class instance given the
+ * coefficients returned by getSizeCoefficients.
+ *
+ * @param coeff the coefficients
+ *
+ * @return the size estimate, in bytes
+ */
+ private long estimateBaseFromCoefficients(int [] coeff, boolean debug) {
+ int size = coeff[0] + (coeff[1]*4 + coeff[2])*refSize;
+
+ // Round up to a multiple of 8
+ size = (int)alignSize(size);
+ if(debug) {
+ if (LOG.isDebugEnabled()) {
+ // Write out region name as string and its encoded name.
+ LOG.debug("Primitives " + coeff[0] + ", arrays " + coeff[1] +
+ ", references(inlcuding " + HeapSize.OBJECT +
+ ", for object overhead) " + coeff[2] + ", refSize " + refSize +
+ ", size " + size);
+ }
+ }
+ return (size < minObjectSize) ? minObjectSize : size;
+ }
+
+ /**
+ * Estimate the static space taken up by the fields of a class. This includes
+ * the space taken up by by references (the pointer) but not by the referenced
+ * object. So the estimated size of an array field does not depend on the size
+ * of the array. Similarly the size of an object (reference) field does not
+ * depend on the object.
+ *
+ * @return the size estimate in bytes.
+ */
+ public long estimateBase(Class cl, boolean debug) {
+ return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug);
+ }
+
+ /**
+ * Tries to clear all the memory used to estimate the reference size for the
+ * current JVM
+ * @param runtime
+ * @param i
+ * @throws Exception
+ */
+ private void cleaner(Runtime runtime, int i) throws Exception{
+ Thread.sleep(i*1000);
+ runtime.gc();runtime.gc(); runtime.gc();runtime.gc();runtime.gc();
+ runtime.runFinalization();
+ }
+
+
+ /**
+ * Aligns a number to 8.
+ * @param num number to align to 8
+ * @return smallest number >= input that is a multiple of 8
+ */
+ public static long alignSize(int num) {
+ int aligned = (num + 7)/8;
+ aligned *= 8;
+ return aligned;
+ }
+
+}
+
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html Sat Jun 20 22:29:03 2009
@@ -178,18 +178,9 @@
See the ZooKeeper <a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html"> Getting Started Guide</a>
for information about the format and options of that file. Specifically, look at the
<a href="http://hadoop.apache.org/zookeeper/docs/current/zookeeperStarted.html#sc_RunningReplicatedZooKeeper">Running Replicated ZooKeeper</a> section.
+</p>
-After configuring <code>zoo.cfg</code>, in <code>${HBASE_HOME}/conf/hbase-env.sh</code>,
-set the following to tell HBase to STOP managing its instance of ZooKeeper.
-<blockquote>
-<pre>
- ...
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
-</pre>
-</blockquote>
-</p>
<p>
Though not recommended, it can be convenient having HBase continue to manage
ZooKeeper even when in distributed mode (It can be good when testing or taking
@@ -216,6 +207,20 @@
on that node would render HBase <b>unusable</b>.
</p>
+<p>
+To tell HBase to stop managing a ZooKeeper instance, after configuring
+<code>zoo.cfg</code> to point at the ZooKeeper Quorum you'd like HBase to
+use, in <code>${HBASE_HOME}/conf/hbase-env.sh</code>,
+set the following to tell HBase to STOP managing its instance of ZooKeeper.
+<blockquote>
+<pre>
+ ...
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+</pre>
+</blockquote>
+</p>
+
<p>Of note, if you have made <i>HDFS client configuration</i> on your hadoop cluster, HBase will not
see this configuration unless you do one of the following:
<ul>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java Sat Jun 20 22:29:03 2009
@@ -62,10 +62,10 @@
hbaseCluster.startup();
} catch (BindException e) {
//this port is already in use. try to use another (for multiple testing)
- int port = conf.getInt("hbase.master.port", DEFAULT_MASTER_PORT);
+ int port = conf.getInt(MASTER_PORT, DEFAULT_MASTER_PORT);
LOG.info("Failed binding Master to port: " + port, e);
port++;
- conf.setInt("hbase.master.port", port);
+ conf.setInt(MASTER_PORT, port);
continue;
}
break;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestClient.java Sat Jun 20 22:29:03 2009
@@ -1032,9 +1032,254 @@
}
- public void XtestDeletes() throws Exception {
-
+ public void testDeletes() throws Exception {
+
+ byte [] TABLE = Bytes.toBytes("testDeletes");
+
+ byte [][] ROWS = makeNAscii(ROW, 6);
+ byte [][] FAMILIES = makeN(FAMILY, 3);
+ byte [][] VALUES = makeN(VALUE, 5);
+ long [] ts = {1000, 2000, 3000, 4000, 5000};
+
+ HTable ht = createTable(TABLE, FAMILIES);
+
+ Put put = new Put(ROW);
+ put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
+ put.add(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
+ ht.put(put);
+
+ Delete delete = new Delete(ROW);
+ delete.deleteFamily(FAMILIES[0], ts[0]);
+ ht.delete(delete);
+
+ Get get = new Get(ROW);
+ get.addFamily(FAMILIES[0]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Result result = ht.get(get);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[1]},
+ new byte[][] {VALUES[1]},
+ 0, 0);
+
+ Scan scan = new Scan(ROW);
+ scan.addFamily(FAMILIES[0]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[1]},
+ new byte[][] {VALUES[1]},
+ 0, 0);
+
// Test delete latest version
+ put = new Put(ROW);
+ put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
+ put.add(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
+ put.add(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
+ ht.put(put);
+
+ delete = new Delete(ROW);
+ delete.deleteColumn(FAMILIES[0], QUALIFIER);
+ ht.delete(delete);
+
+ get = new Get(ROW);
+ get.addFamily(FAMILIES[0]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[1], ts[2], ts[3]},
+ new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
+ 0, 2);
+
+ scan = new Scan(ROW);
+ scan.addFamily(FAMILIES[0]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[1], ts[2], ts[3]},
+ new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
+ 0, 2);
+
+ // Expected client behavior might be that you can re-put deleted values
+ // But alas, this is not to be. We can't put them back in either case.
+
+ put = new Put(ROW);
+ put.add(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
+ put.add(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
+ ht.put(put);
+
+ // The Get returns the latest value but then does not return the
+ // oldest, which was never deleted, ts[1].
+
+ get = new Get(ROW);
+ get.addFamily(FAMILIES[0]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[2], ts[3], ts[4]},
+ new byte[][] {VALUES[2], VALUES[3], VALUES[4]},
+ 0, 2);
+
+ // The Scanner returns the previous values, the expected-unexpected behavior
+
+ scan = new Scan(ROW);
+ scan.addFamily(FAMILIES[0]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
+ new long [] {ts[1], ts[2], ts[3]},
+ new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
+ 0, 2);
+
+ // Test deleting an entire family from one row but not the other various ways
+
+ put = new Put(ROWS[0]);
+ put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
+ put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
+ put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
+ put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
+ ht.put(put);
+
+ put = new Put(ROWS[1]);
+ put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
+ put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
+ put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
+ put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
+ ht.put(put);
+
+ put = new Put(ROWS[2]);
+ put.add(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
+ put.add(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
+ put.add(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
+ put.add(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
+ ht.put(put);
+
+ delete = new Delete(ROWS[0]);
+ delete.deleteFamily(FAMILIES[2]);
+ ht.delete(delete);
+
+ delete = new Delete(ROWS[1]);
+ delete.deleteColumns(FAMILIES[1], QUALIFIER);
+ ht.delete(delete);
+
+ delete = new Delete(ROWS[2]);
+ delete.deleteColumn(FAMILIES[1], QUALIFIER);
+ delete.deleteColumn(FAMILIES[1], QUALIFIER);
+ delete.deleteColumn(FAMILIES[2], QUALIFIER);
+ ht.delete(delete);
+
+ get = new Get(ROWS[0]);
+ get.addFamily(FAMILIES[1]);
+ get.addFamily(FAMILIES[2]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+ assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
+ new long [] {ts[0], ts[1]},
+ new byte[][] {VALUES[0], VALUES[1]},
+ 0, 1);
+
+ scan = new Scan(ROWS[0]);
+ scan.addFamily(FAMILIES[1]);
+ scan.addFamily(FAMILIES[2]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+ assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
+ new long [] {ts[0], ts[1]},
+ new byte[][] {VALUES[0], VALUES[1]},
+ 0, 1);
+
+ get = new Get(ROWS[1]);
+ get.addFamily(FAMILIES[1]);
+ get.addFamily(FAMILIES[2]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+
+ scan = new Scan(ROWS[1]);
+ scan.addFamily(FAMILIES[1]);
+ scan.addFamily(FAMILIES[2]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+
+ get = new Get(ROWS[2]);
+ get.addFamily(FAMILIES[1]);
+ get.addFamily(FAMILIES[2]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertTrue("Expected 1 key but received " + result.size(),
+ result.size() == 1);
+ assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
+ new long [] {ts[2]},
+ new byte[][] {VALUES[2]},
+ 0, 0);
+
+ scan = new Scan(ROWS[2]);
+ scan.addFamily(FAMILIES[1]);
+ scan.addFamily(FAMILIES[2]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ result = getSingleScanResult(ht, scan);
+ assertTrue("Expected 1 key but received " + result.size(),
+ result.size() == 1);
+ assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
+ new long [] {ts[2]},
+ new byte[][] {VALUES[2]},
+ 0, 0);
+
+ // Test if we delete the family first in one row (HBASE-1541)
+
+ delete = new Delete(ROWS[3]);
+ delete.deleteFamily(FAMILIES[1]);
+ ht.delete(delete);
+
+ put = new Put(ROWS[3]);
+ put.add(FAMILIES[2], QUALIFIER, VALUES[0]);
+ ht.put(put);
+
+ put = new Put(ROWS[4]);
+ put.add(FAMILIES[1], QUALIFIER, VALUES[1]);
+ put.add(FAMILIES[2], QUALIFIER, VALUES[2]);
+ ht.put(put);
+
+ get = new Get(ROWS[3]);
+ get.addFamily(FAMILIES[1]);
+ get.addFamily(FAMILIES[2]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertTrue("Expected 1 key but received " + result.size(),
+ result.size() == 1);
+
+ get = new Get(ROWS[4]);
+ get.addFamily(FAMILIES[1]);
+ get.addFamily(FAMILIES[2]);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ result = ht.get(get);
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+
+ scan = new Scan(ROWS[3]);
+ scan.addFamily(FAMILIES[1]);
+ scan.addFamily(FAMILIES[2]);
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner = ht.getScanner(scan);
+ result = scanner.next();
+ assertTrue("Expected 1 key but received " + result.size(),
+ result.size() == 1);
+ assertTrue(Bytes.equals(result.sorted()[0].getRow(), ROWS[3]));
+ assertTrue(Bytes.equals(result.sorted()[0].getValue(), VALUES[0]));
+ result = scanner.next();
+ assertTrue("Expected 2 keys but received " + result.size(),
+ result.size() == 2);
+ assertTrue(Bytes.equals(result.sorted()[0].getRow(), ROWS[4]));
+ assertTrue(Bytes.equals(result.sorted()[1].getRow(), ROWS[4]));
+ assertTrue(Bytes.equals(result.sorted()[0].getValue(), VALUES[1]));
+ assertTrue(Bytes.equals(result.sorted()[1].getValue(), VALUES[2]));
+ scanner.close();
}
/**
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestRowPrefixFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestRowPrefixFilter.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestRowPrefixFilter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestRowPrefixFilter.java Sat Jun 20 22:29:03 2009
@@ -54,6 +54,10 @@
prefixRowTests(mainFilter);
}
+ public void testPrefixOnRowInsideWhileMatchRow() throws Exception {
+ prefixRowTests(new RowWhileMatchFilter(this.mainFilter), true);
+ }
+
public void testSerialization() throws Exception {
// Decompose mainFilter to bytes.
ByteArrayOutputStream stream = new ByteArrayOutputStream();
@@ -72,20 +76,25 @@
}
private void prefixRowTests(Filter filter) throws Exception {
+ prefixRowTests(filter, false);
+ }
+
+ private void prefixRowTests(Filter filter, boolean lastFilterAllRemaining)
+ throws Exception {
for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) {
byte [] t = createRow(c);
- assertFalse("Failed with characer " + c, filter.filterRowKey(t, 0, t.length));
+ assertFalse("Failed with character " + c,
+ filter.filterRowKey(t, 0, t.length));
+ assertFalse(filter.filterAllRemaining());
}
String yahooSite = "com.yahoo.www";
byte [] yahooSiteBytes = Bytes.toBytes(yahooSite);
assertTrue("Failed with character " +
yahooSite, filter.filterRowKey(yahooSiteBytes, 0, yahooSiteBytes.length));
+ assertEquals(filter.filterAllRemaining(), lastFilterAllRemaining);
}
private byte [] createRow(final char c) {
return Bytes.toBytes(HOST_PREFIX + Character.toString(c));
}
-
-
-
-}
+}
\ No newline at end of file
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java?rev=786919&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java Sat Jun 20 22:29:03 2009
@@ -0,0 +1,72 @@
+package org.apache.hadoop.hbase.io;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+
+import junit.framework.TestCase;
+
+/**
+ * Testing the sizing that HeapSize offers and compares to the size given by
+ * ClassSize.
+ */
+public class TestHeapSize extends TestCase {
+ static final Log LOG = LogFactory.getLog(TestHeapSize.class);
+ // List of classes implementing HeapSize
+ // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
+ // KeyValue, LruBlockCache, LruHashMap<K,V>, Put, HLogKey
+
+ /**
+ * Testing the classes that implements HeapSize and are a part of 0.20.
+ * Some are not tested here for example BlockIndex which is tested in
+ * TestHFile since it is a non public class
+ */
+ public void testSizes() {
+ ClassSize cs = null;
+ Class cl = null;
+ long expected = 0L;
+ long actual = 0L;
+ try {
+ cs = new ClassSize();
+ } catch(Exception e) {}
+
+ //KeyValue
+ cl = KeyValue.class;
+ expected = cs.estimateBase(cl, false);
+ KeyValue kv = new KeyValue();
+ actual = kv.heapSize();
+ if(expected != actual) {
+ cs.estimateBase(cl, true);
+ assertEquals(expected, actual);
+ }
+
+ //LruBlockCache
+ cl = LruBlockCache.class;
+ expected = cs.estimateBase(cl, false);
+ LruBlockCache c = new LruBlockCache(1,1,200);
+ //Since minimum size for the for a LruBlockCache is 1
+ //we need to remove one reference from the heapsize
+ actual = c.heapSize() - HeapSize.REFERENCE;
+ if(expected != actual) {
+ cs.estimateBase(cl, true);
+ assertEquals(expected, actual);
+ }
+
+ //Put
+ cl = Put.class;
+ expected = cs.estimateBase(cl, false);
+ //The actual TreeMap is not included in the above calculation
+ expected += HeapSize.TREEMAP_SIZE;
+ Put put = new Put(Bytes.toBytes(""));
+ actual = put.heapSize();
+ if(expected != actual) {
+ cs.estimateBase(cl, true);
+ assertEquals(expected, actual);
+ }
+ }
+
+}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/hfile/TestHFile.java Sat Jun 20 22:29:03 2009
@@ -29,9 +29,12 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.HFile.BlockIndex;
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.io.RawComparator;
/**
@@ -244,4 +247,31 @@
writer.append("1".getBytes(), "0".getBytes());
writer.close();
}
+
+ /**
+ * Checks if the HeapSize calculator is within reason
+ */
+ public void testHeapSizeForBlockIndex() {
+ ClassSize cs = null;
+ Class cl = null;
+ long expected = 0L;
+ long actual = 0L;
+ try {
+ cs = new ClassSize();
+ } catch(Exception e) {}
+
+ //KeyValue
+ cl = BlockIndex.class;
+ expected = cs.estimateBase(cl, false);
+ BlockIndex bi = new BlockIndex(Bytes.BYTES_RAWCOMPARATOR);
+ actual = bi.heapSize();
+ //Since we have a [[]] in BlockIndex and the checker only sees the [] we
+ // miss a MULTI_ARRAY which is 4*Reference = 32 B
+ actual -= 32;
+ if(expected != actual) {
+ cs.estimateBase(cl, true);
+ assertEquals(expected, actual);
+ }
+ }
+
}
\ No newline at end of file
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java?rev=786919&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java Sat Jun 20 22:29:03 2009
@@ -0,0 +1,265 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Random;
+
+import junit.framework.TestSuite;
+import junit.textui.TestRunner;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MultiRegionTable;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MultiSearcher;
+import org.apache.lucene.search.Searchable;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.TermQuery;
+
+/**
+ * Test Map/Reduce job to build index over HBase table
+ */
+public class TestTableIndex extends MultiRegionTable {
+ private static final Log LOG = LogFactory.getLog(TestTableIndex.class);
+
+ static final String TABLE_NAME = "moretest";
+ static final String INPUT_COLUMN = "contents:";
+ static final byte [] TEXT_INPUT_COLUMN = Bytes.toBytes(INPUT_COLUMN);
+ static final String OUTPUT_COLUMN = "text:";
+ static final byte [] TEXT_OUTPUT_COLUMN = Bytes.toBytes(OUTPUT_COLUMN);
+ static final String ROWKEY_NAME = "key";
+ static final String INDEX_DIR = "testindex";
+ private static final byte [][] columns = new byte [][] {
+ TEXT_INPUT_COLUMN,
+ TEXT_OUTPUT_COLUMN
+ };
+
+ private JobConf jobConf = null;
+
+ /** default constructor */
+ public TestTableIndex() {
+ super(INPUT_COLUMN);
+ desc = new HTableDescriptor(TABLE_NAME);
+ desc.addFamily(new HColumnDescriptor(INPUT_COLUMN));
+ desc.addFamily(new HColumnDescriptor(OUTPUT_COLUMN));
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ if (jobConf != null) {
+ FileUtil.fullyDelete(new File(jobConf.get("hadoop.tmp.dir")));
+ }
+ }
+
+ /**
+ * Test HBase map/reduce
+ *
+ * @throws IOException
+ */
+ public void testTableIndex() throws IOException {
+ boolean printResults = false;
+ if (printResults) {
+ LOG.info("Print table contents before map/reduce");
+ }
+ scanTable(printResults);
+
+ MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
+
+ // set configuration parameter for index build
+ conf.set("hbase.index.conf", createIndexConfContent());
+
+ try {
+ jobConf = new JobConf(conf, TestTableIndex.class);
+ jobConf.setJobName("index column contents");
+ jobConf.setNumMapTasks(2);
+ // number of indexes to partition into
+ jobConf.setNumReduceTasks(1);
+
+ // use identity map (a waste, but just as an example)
+ IdentityTableMap.initJob(TABLE_NAME, INPUT_COLUMN,
+ IdentityTableMap.class, jobConf);
+
+ // use IndexTableReduce to build a Lucene index
+ jobConf.setReducerClass(IndexTableReduce.class);
+ FileOutputFormat.setOutputPath(jobConf, new Path(INDEX_DIR));
+ jobConf.setOutputFormat(IndexOutputFormat.class);
+
+ JobClient.runJob(jobConf);
+
+ } finally {
+ mrCluster.shutdown();
+ }
+
+ if (printResults) {
+ LOG.info("Print table contents after map/reduce");
+ }
+ scanTable(printResults);
+
+ // verify index results
+ verify();
+ }
+
+ private String createIndexConfContent() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("<configuration><column><property>" +
+ "<name>hbase.column.name</name><value>" + INPUT_COLUMN +
+ "</value></property>");
+ buffer.append("<property><name>hbase.column.store</name> " +
+ "<value>true</value></property>");
+ buffer.append("<property><name>hbase.column.index</name>" +
+ "<value>true</value></property>");
+ buffer.append("<property><name>hbase.column.tokenize</name>" +
+ "<value>false</value></property>");
+ buffer.append("<property><name>hbase.column.boost</name>" +
+ "<value>3</value></property>");
+ buffer.append("<property><name>hbase.column.omit.norms</name>" +
+ "<value>false</value></property></column>");
+ buffer.append("<property><name>hbase.index.rowkey.name</name><value>" +
+ ROWKEY_NAME + "</value></property>");
+ buffer.append("<property><name>hbase.index.max.buffered.docs</name>" +
+ "<value>500</value></property>");
+ buffer.append("<property><name>hbase.index.max.field.length</name>" +
+ "<value>10000</value></property>");
+ buffer.append("<property><name>hbase.index.merge.factor</name>" +
+ "<value>10</value></property>");
+ buffer.append("<property><name>hbase.index.use.compound.file</name>" +
+ "<value>true</value></property>");
+ buffer.append("<property><name>hbase.index.optimize</name>" +
+ "<value>true</value></property></configuration>");
+
+ IndexConfiguration c = new IndexConfiguration();
+ c.addFromXML(buffer.toString());
+ return c.toString();
+ }
+
+ private void scanTable(boolean printResults)
+ throws IOException {
+ HTable table = new HTable(conf, TABLE_NAME);
+ Scan scan = new Scan();
+ scan.addColumns(columns);
+ ResultScanner scanner = table.getScanner(scan);
+ try {
+ for (Result r : scanner) {
+ if (printResults) {
+ LOG.info("row: " + r.getRow());
+ }
+ for (Map.Entry<byte [], Cell> e : r.getRowResult().entrySet()) {
+ if (printResults) {
+ LOG.info(" column: " + e.getKey() + " value: "
+ + new String(e.getValue().getValue(), HConstants.UTF8_ENCODING));
+ }
+ }
+ }
+ } finally {
+ scanner.close();
+ }
+ }
+
+ private void verify() throws IOException {
+ // Force a cache flush for every online region to ensure that when the
+ // scanner takes its snapshot, all the updates have made it into the cache.
+ for (HRegion r : cluster.getRegionThreads().get(0).getRegionServer().
+ getOnlineRegions()) {
+ HRegionIncommon region = new HRegionIncommon(r);
+ region.flushcache();
+ }
+
+ Path localDir = new Path(getUnitTestdir(getName()), "index_" +
+ Integer.toString(new Random().nextInt()));
+ this.fs.copyToLocalFile(new Path(INDEX_DIR), localDir);
+ FileSystem localfs = FileSystem.getLocal(conf);
+ FileStatus [] indexDirs = localfs.listStatus(localDir);
+ Searcher searcher = null;
+ ResultScanner scanner = null;
+ try {
+ if (indexDirs.length == 1) {
+ searcher = new IndexSearcher((new File(indexDirs[0].getPath().
+ toUri())).getAbsolutePath());
+ } else if (indexDirs.length > 1) {
+ Searchable[] searchers = new Searchable[indexDirs.length];
+ for (int i = 0; i < indexDirs.length; i++) {
+ searchers[i] = new IndexSearcher((new File(indexDirs[i].getPath().
+ toUri()).getAbsolutePath()));
+ }
+ searcher = new MultiSearcher(searchers);
+ } else {
+ throw new IOException("no index directory found");
+ }
+
+ HTable table = new HTable(conf, TABLE_NAME);
+ Scan scan = new Scan();
+ scan.addColumns(columns);
+ scanner = table.getScanner(scan);
+
+ IndexConfiguration indexConf = new IndexConfiguration();
+ String content = conf.get("hbase.index.conf");
+ if (content != null) {
+ indexConf.addFromXML(content);
+ }
+ String rowkeyName = indexConf.getRowkeyName();
+
+ int count = 0;
+ for (Result r : scanner) {
+ String value = Bytes.toString(r.getRow());
+ Term term = new Term(rowkeyName, value);
+ int hitCount = searcher.search(new TermQuery(term)).length();
+ assertEquals("check row " + value, 1, hitCount);
+ count++;
+ }
+ LOG.debug("Searcher.maxDoc: " + searcher.maxDoc());
+ LOG.debug("IndexReader.numDocs: " + ((IndexSearcher)searcher).getIndexReader().numDocs());
+ int maxDoc = ((IndexSearcher)searcher).getIndexReader().numDocs();
+ assertEquals("check number of rows", maxDoc, count);
+ } finally {
+ if (null != searcher)
+ searcher.close();
+ if (null != scanner)
+ scanner.close();
+ }
+ }
+ /**
+ * @param args unused
+ */
+ public static void main(String[] args) {
+ TestRunner.run(new TestSuite(TestTableIndex.class));
+ }
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/OOMERegionServer.java Sat Jun 20 22:29:03 2009
@@ -40,11 +40,6 @@
public OOMERegionServer(HBaseConfiguration conf) throws IOException {
super(conf);
}
-
- public OOMERegionServer(HServerAddress address, HBaseConfiguration conf)
- throws IOException {
- super(address, conf);
- }
public void put(byte [] regionName, Put put)
throws IOException {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Sat Jun 20 22:29:03 2009
@@ -40,6 +40,7 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
@@ -71,8 +72,7 @@
// individual code pieces in the HRegion. Putting files locally in
// /tmp/testtable
//////////////////////////////////////////////////////////////////////////////
-
-
+
//////////////////////////////////////////////////////////////////////////////
// checkAndPut tests
//////////////////////////////////////////////////////////////////////////////
@@ -285,10 +285,45 @@
get = new Get(row).addColumn(fam, splitB);
result = region.get(get, null);
assertEquals(1, result.size());
+ }
-
+ public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
+ byte [] tableName = Bytes.toBytes("test_table");
+ byte [] fam1 = Bytes.toBytes("columnA");
+ byte [] fam2 = Bytes.toBytes("columnB");
+ initHRegion(tableName, getName(), fam1, fam2);
+
+ byte [] rowA = Bytes.toBytes("rowA");
+ byte [] rowB = Bytes.toBytes("rowB");
+
+ byte [] value = Bytes.toBytes("value");
+
+ Delete delete = new Delete(rowA);
+ delete.deleteFamily(fam1);
+
+ region.delete(delete, null, true);
+
+ // now create data.
+ Put put = new Put(rowA);
+ put.add(fam2, null, value);
+ region.put(put);
+
+ put = new Put(rowB);
+ put.add(fam1, null, value);
+ put.add(fam2, null, value);
+ region.put(put);
+
+ Scan scan = new Scan();
+ scan.addFamily(fam1).addFamily(fam2);
+ InternalScanner s = region.getScanner(scan);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ s.next(results);
+ assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
+
+ results.clear();
+ s.next(results);
+ assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
-
}
//Visual test, since the method doesn't return anything
@@ -340,7 +375,7 @@
}
assertFalse(true);
}
-
+
public void testGet_Basic() throws IOException {
byte [] tableName = Bytes.toBytes("testtable");
byte [] row1 = Bytes.toBytes("row1");
@@ -363,7 +398,7 @@
put.add(fam1, col4, null);
put.add(fam1, col5, null);
region.put(put);
-
+
Get get = new Get(row1);
get.addColumn(fam1, col2);
get.addColumn(fam1, col4);
@@ -371,10 +406,9 @@
KeyValue kv1 = new KeyValue(row1, fam1, col2);
KeyValue kv2 = new KeyValue(row1, fam1, col4);
KeyValue [] expected = {kv1, kv2};
-
+
//Test
Result res = region.get(get, null);
-
assertEquals(expected.length, res.size());
for(int i=0; i<res.size(); i++){
assertEquals(0,
@@ -385,8 +419,15 @@
Bytes.compareTo(
expected[i].getQualifier(), res.raw()[i].getQualifier()));
}
+
+ // Test using a filter on a Get
+ Get g = new Get(row1);
+ final int count = 2;
+ g.setFilter(new ColumnCountGetFilter(count));
+ res = region.get(g, null);
+ assertEquals(count, res.size());
}
-
+
public void testGet_Empty() throws IOException {
byte [] tableName = Bytes.toBytes("emptytable");
byte [] row = Bytes.toBytes("row");
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java Sat Jun 20 22:29:03 2009
@@ -296,7 +296,7 @@
long ttl = Long.MAX_VALUE;
QueryMatcher matcher =
- new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+ new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
List<KeyValue> result = new ArrayList<KeyValue>();
boolean res = memcache.get(matcher, result);
@@ -324,7 +324,7 @@
long ttl = Long.MAX_VALUE;
QueryMatcher matcher =
- new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+ new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
List<KeyValue> result = new ArrayList<KeyValue>();
boolean res = memcache.get(matcher, result);
@@ -349,7 +349,7 @@
long ttl = Long.MAX_VALUE;
QueryMatcher matcher =
- new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+ new QueryMatcher(get, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
//Setting up memcache
memcache.add(new KeyValue(row, fam ,qf1, val));
@@ -390,8 +390,8 @@
columns.add(qf3);
long ttl = Long.MAX_VALUE;
- QueryMatcher matcher =
- new QueryMatcher(get, row, fam, columns, ttl, KeyValue.KEY_COMPARATOR, 1);
+ QueryMatcher matcher = new QueryMatcher(get, fam, columns, ttl,
+ KeyValue.KEY_COMPARATOR, 1);
//Setting up expected
List<KeyValue> expected = new ArrayList<KeyValue>();
@@ -506,9 +506,12 @@
KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
+ KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
+
memcache.add(put1);
memcache.add(put2);
memcache.add(put3);
+ memcache.add(put4);
KeyValue del =
new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
@@ -516,8 +519,9 @@
List<KeyValue> expected = new ArrayList<KeyValue>();
expected.add(del);
+ expected.add(put4);
- assertEquals(1, memcache.memcache.size());
+ assertEquals(2, memcache.memcache.size());
int i=0;
for(KeyValue actual : memcache.memcache) {
assertEquals(expected.get(i++), actual);
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java Sat Jun 20 22:29:03 2009
@@ -97,7 +97,7 @@
expected.add(MatchCode.INCLUDE);
expected.add(MatchCode.DONE);
- QueryMatcher qm = new QueryMatcher(get, get.getRow(), fam2,
+ QueryMatcher qm = new QueryMatcher(get, fam2,
get.getFamilyMap().get(fam2), ttl, rowComparator, 1);
List<KeyValue> memCache = new ArrayList<KeyValue>();
@@ -140,8 +140,7 @@
expected.add(MatchCode.INCLUDE);
expected.add(MatchCode.NEXT);
- QueryMatcher qm = new QueryMatcher(get, get.getRow(), fam2, null,
- ttl, rowComparator, 1);
+ QueryMatcher qm = new QueryMatcher(get, fam2, null, ttl, rowComparator, 1);
List<KeyValue> memCache = new ArrayList<KeyValue>();
memCache.add(new KeyValue(row1, fam2, col1, data));
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Sat Jun 20 22:29:03 2009
@@ -39,9 +39,6 @@
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.StopRowFilter;
-import org.apache.hadoop.hbase.filter.WhileMatchRowFilter;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=786919&r1=786918&r2=786919&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Sat Jun 20 22:29:03 2009
@@ -140,8 +140,13 @@
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
+ assertEquals(0, results.size());
+
+ assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[2], results.get(0));
+
+ assertEquals(false, scan.next(results));
}
public void testDeleteVersionMaskingMultiplePuts() throws IOException {
@@ -272,8 +277,12 @@
null, scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
+ assertEquals(0, results.size());
+ assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[kvs.length-1], results.get(0));
+
+ assertEquals(false, scan.next(results));
}
public void testDeleteColumn() throws IOException {
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java?rev=786919&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java Sat Jun 20 22:29:03 2009
@@ -0,0 +1,397 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.thrift;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
+import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
+import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
+import org.apache.hadoop.hbase.thrift.generated.Mutation;
+import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Unit testing for ThriftServer.HBaseHandler, a part of the
+ * org.apache.hadoop.hbase.thrift package.
+ */
+public class TestThriftServer extends HBaseClusterTestCase {
+
+ // Static names for tables, columns, rows, and values
+ private static byte[] tableAname = Bytes.toBytes("tableA");
+ private static byte[] tableBname = Bytes.toBytes("tableB");
+ private static byte[] columnAname = Bytes.toBytes("columnA:");
+ private static byte[] columnBname = Bytes.toBytes("columnB:");
+ private static byte[] rowAname = Bytes.toBytes("rowA");
+ private static byte[] rowBname = Bytes.toBytes("rowB");
+ private static byte[] valueAname = Bytes.toBytes("valueA");
+ private static byte[] valueBname = Bytes.toBytes("valueB");
+ private static byte[] valueCname = Bytes.toBytes("valueC");
+ private static byte[] valueDname = Bytes.toBytes("valueD");
+
+ /**
+ * Runs all of the tests under a single JUnit test method. We
+ * consolidate all testing to one method because HBaseClusterTestCase
+ * is prone to OutOfMemoryExceptions when there are three or more
+ * JUnit test methods.
+ *
+ * @throws Exception
+ */
+ public void testAll() throws Exception {
+ // Run all tests
+ doTestTableCreateDrop();
+ doTestTableMutations();
+ doTestTableTimestampsAndColumns();
+ doTestTableScanners();
+ }
+
+ /**
+ * Tests for creating, enabling, disabling, and deleting tables. Also
+ * tests that creating a table with an invalid column name yields an
+ * IllegalArgument exception.
+ *
+ * @throws Exception
+ */
+ public void doTestTableCreateDrop() throws Exception {
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+
+ // Create/enable/disable/delete tables, ensure methods act correctly
+ assertEquals(handler.getTableNames().size(), 0);
+ handler.createTable(tableAname, getColumnDescriptors());
+ assertEquals(handler.getTableNames().size(), 1);
+ assertEquals(handler.getColumnDescriptors(tableAname).size(), 2);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.createTable(tableBname, new ArrayList<ColumnDescriptor>());
+ assertEquals(handler.getTableNames().size(), 2);
+ handler.disableTable(tableBname);
+ assertFalse(handler.isTableEnabled(tableBname));
+ handler.deleteTable(tableBname);
+ assertEquals(handler.getTableNames().size(), 1);
+ handler.disableTable(tableAname);
+ assertFalse(handler.isTableEnabled(tableAname));
+ handler.enableTable(tableAname);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Tests adding a series of Mutations and BatchMutations, including a
+ * delete mutation. Also tests data retrieval, and getting back multiple
+ * versions.
+ *
+ * @throws Exception
+ */
+ public void doTestTableMutations() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply a few Mutations to rowA
+ // mutations.add(new Mutation(false, columnAname, valueAname));
+ // mutations.add(new Mutation(false, columnBname, valueBname));
+ handler.mutateRow(tableAname, rowAname, getMutations());
+
+ // Assert that the changes were made
+ assertTrue(Bytes.equals(valueAname,
+ handler.get(tableAname, rowAname, columnAname).get(0).value));
+ TRowResult rowResult1 = handler.getRow(tableAname, rowAname).get(0);
+ assertTrue(Bytes.equals(rowAname, rowResult1.row));
+ assertTrue(Bytes.equals(valueBname,
+ rowResult1.columns.get(columnBname).value));
+
+ // Apply a few BatchMutations for rowA and rowB
+ // rowAmutations.add(new Mutation(true, columnAname, null));
+ // rowAmutations.add(new Mutation(false, columnBname, valueCname));
+ // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+ // Mutations to rowB
+ // rowBmutations.add(new Mutation(false, columnAname, valueCname));
+ // rowBmutations.add(new Mutation(false, columnBname, valueDname));
+ // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
+ handler.mutateRows(tableAname, getBatchMutations());
+
+ // Assert that changes were made to rowA
+ List<TCell> cells = handler.get(tableAname, rowAname, columnAname);
+ assertFalse(cells.size() > 0);
+ assertTrue(Bytes.equals(valueCname, handler.get(tableAname, rowAname, columnBname).get(0).value));
+ List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS);
+ assertTrue(Bytes.equals(valueCname, versions.get(0).value));
+ assertTrue(Bytes.equals(valueBname, versions.get(1).value));
+
+ // Assert that changes were made to rowB
+ TRowResult rowResult2 = handler.getRow(tableAname, rowBname).get(0);
+ assertTrue(Bytes.equals(rowBname, rowResult2.row));
+ assertTrue(Bytes.equals(valueCname, rowResult2.columns.get(columnAname).value));
+ assertTrue(Bytes.equals(valueDname, rowResult2.columns.get(columnBname).value));
+
+ // Apply some deletes
+ handler.deleteAll(tableAname, rowAname, columnBname);
+ handler.deleteAllRow(tableAname, rowBname);
+
+ // Assert that the deletes were applied
+ int size = handler.get(tableAname, rowAname, columnBname).size();
+ assertEquals(0, size);
+ size = handler.getRow(tableAname, rowBname).size();
+ assertEquals(0, size);
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Similar to testTableMutations(), except Mutations are applied with
+ * specific timestamps and data retrieval uses these timestamps to
+ * extract specific versions of data.
+ *
+ * @throws Exception
+ */
+ public void doTestTableTimestampsAndColumns() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ // Apply an overlapping timestamped mutation to rowB
+ handler.mutateRowTs(tableAname, rowBname, getMutations(), time2);
+
+ // the getVerTs is [inf, ts) so you need to increment one.
+ time1 += 1;
+ time2 += 2;
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
+ MAXVERSIONS).size());
+ assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
+ MAXVERSIONS).size());
+
+ TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0);
+ TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0);
+ // columnA was completely deleted
+ //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname));
+
+ // ColumnAname has been deleted, and will never be visible even with a getRowTs()
+ assertFalse(rowResult2.columns.containsKey(columnAname));
+
+ List<byte[]> columns = new ArrayList<byte[]>();
+ columns.add(columnBname);
+
+ rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueCname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ // Apply some timestamped deletes
+ // this actually deletes _everything_.
+ // nukes everything in columnB: forever.
+ handler.deleteAllTs(tableAname, rowAname, columnBname, time1);
+ handler.deleteAllRowTs(tableAname, rowBname, time2);
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size();
+ assertEquals(0, size);
+
+ size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS).size();
+ assertEquals(1, size);
+
+ // should be available....
+ assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname));
+
+ assertEquals(0, handler.getRow(tableAname, rowBname).size());
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Tests the four different scanner-opening methods (with and without
+ * a stoprow, with and without a timestamp).
+ *
+ * @throws Exception
+ */
+ public void doTestTableScanners() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ // Sleep to assure that 'time1' and 'time2' will be different even with a
+ // coarse grained system timer.
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ time1 += 1;
+ time2 += 1;
+
+ // Test a scanner on all rows and all columns, no timestamp
+ int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true));
+ TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1a.row, rowAname));
+ // This used to be '1'. I don't know why when we are asking for two columns
+ // and when the mutations above would seem to add two columns to the row.
+ // -- St.Ack 05/12/2009
+ assertEquals(rowResult1a.columns.size(), 1);
+ assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname));
+
+ TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1b.row, rowBname));
+ assertEquals(rowResult1b.columns.size(), 2);
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnAname).value, valueCname));
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnBname).value, valueDname));
+ closeScanner(scanner1, handler);
+
+ // Test a scanner on all rows and all columns, with timestamp
+ int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1);
+ TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
+ assertEquals(rowResult2a.columns.size(), 1);
+ // column A deleted, does not exist.
+ //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname));
+ closeScanner(scanner2, handler);
+
+ // Test a scanner on the first row and first column only, no timestamp
+ int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
+ getColumnList(true, false));
+ closeScanner(scanner3, handler);
+
+ // Test a scanner on the first row and second column only, with timestamp
+ int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
+ getColumnList(false, true), time1);
+ TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
+ assertEquals(rowResult4a.columns.size(), 1);
+ assertTrue(Bytes.equals(rowResult4a.columns.get(columnBname).value, valueBname));
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ *
+ * @return a List of ColumnDescriptors for use in creating a table. Has one
+ * default ColumnDescriptor and one ColumnDescriptor with fewer versions
+ */
+ private List<ColumnDescriptor> getColumnDescriptors() {
+ ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<ColumnDescriptor>();
+
+ // A default ColumnDescriptor
+ ColumnDescriptor cDescA = new ColumnDescriptor();
+ cDescA.name = columnAname;
+ cDescriptors.add(cDescA);
+
+ // A slightly customized ColumnDescriptor (only 2 versions)
+ ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
+ false, 2147483647, "NONE", 0, 0, false, -1);
+ cDescriptors.add(cDescB);
+
+ return cDescriptors;
+ }
+
+ /**
+ *
+ * @param includeA whether or not to include columnA
+ * @param includeB whether or not to include columnB
+ * @return a List of column names for use in retrieving a scanner
+ */
+ private List<byte[]> getColumnList(boolean includeA, boolean includeB) {
+ List<byte[]> columnList = new ArrayList<byte[]>();
+ if (includeA) columnList.add(columnAname);
+ if (includeB) columnList.add(columnBname);
+ return columnList;
+ }
+
+ /**
+ *
+ * @return a List of Mutations for a row, with columnA having valueA
+ * and columnB having valueB
+ */
+ private List<Mutation> getMutations() {
+ List<Mutation> mutations = new ArrayList<Mutation>();
+ mutations.add(new Mutation(false, columnAname, valueAname));
+ mutations.add(new Mutation(false, columnBname, valueBname));
+ return mutations;
+ }
+
+ /**
+ *
+ * @return a List of BatchMutations with the following effects:
+ * (rowA, columnA): delete
+ * (rowA, columnB): place valueC
+ * (rowB, columnA): place valueC
+ * (rowB, columnB): place valueD
+ */
+ private List<BatchMutation> getBatchMutations() {
+ List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
+
+ // Mutations to rowA. You can't mix delete and put anymore.
+ List<Mutation> rowAmutations = new ArrayList<Mutation>();
+ rowAmutations.add(new Mutation(true, columnAname, null));
+ batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+
+ rowAmutations = new ArrayList<Mutation>();
+ rowAmutations.add(new Mutation(false, columnBname, valueCname));
+ batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+
+ // Mutations to rowB
+ List<Mutation> rowBmutations = new ArrayList<Mutation>();
+ rowBmutations.add(new Mutation(false, columnAname, valueCname));
+ rowBmutations.add(new Mutation(false, columnBname, valueDname));
+ batchMutations.add(new BatchMutation(rowBname, rowBmutations));
+
+ return batchMutations;
+ }
+
+ /**
+ * Asserts that the passed scanner is exhausted, and then closes
+ * the scanner.
+ *
+ * @param scannerId the scanner to close
+ * @param handler the HBaseHandler interfacing to HBase
+ * @throws Exception
+ */
+ private void closeScanner(int scannerId, ThriftServer.HBaseHandler handler) throws Exception {
+ handler.scannerGet(scannerId);
+ handler.scannerClose(scannerId);
+ }
+}
Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/TestMergeTool.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/TestMergeTool.java?rev=786919&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/TestMergeTool.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/TestMergeTool.java Sat Jun 20 22:29:03 2009
@@ -0,0 +1,275 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.regionserver.HLog;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.util.ToolRunner;
+
+/** Test stand alone merge tool that can merge arbitrary regions */
+public class TestMergeTool extends HBaseTestCase {
+ static final Log LOG = LogFactory.getLog(TestMergeTool.class);
+// static final byte [] COLUMN_NAME = Bytes.toBytes("contents:");
+ static final byte [] FAMILY = Bytes.toBytes("contents");
+ static final byte [] QUALIFIER = Bytes.toBytes("dc");
+
+ private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
+ private final HRegion[] regions = new HRegion[5];
+ private HTableDescriptor desc;
+ private byte [][][] rows;
+ private MiniDFSCluster dfsCluster = null;
+
+ @Override
+ public void setUp() throws Exception {
+ this.conf.set("hbase.hstore.compactionThreshold", "2");
+
+ // Create table description
+ this.desc = new HTableDescriptor("TestMergeTool");
+ this.desc.addFamily(new HColumnDescriptor(FAMILY));
+
+ /*
+ * Create the HRegionInfos for the regions.
+ */
+ // Region 0 will contain the key range [row_0200,row_0300)
+ sourceRegions[0] = new HRegionInfo(this.desc, Bytes.toBytes("row_0200"),
+ Bytes.toBytes("row_0300"));
+
+ // Region 1 will contain the key range [row_0250,row_0400) and overlaps
+ // with Region 0
+ sourceRegions[1] =
+ new HRegionInfo(this.desc, Bytes.toBytes("row_0250"),
+ Bytes.toBytes("row_0400"));
+
+ // Region 2 will contain the key range [row_0100,row_0200) and is adjacent
+ // to Region 0 or the region resulting from the merge of Regions 0 and 1
+ sourceRegions[2] =
+ new HRegionInfo(this.desc, Bytes.toBytes("row_0100"),
+ Bytes.toBytes("row_0200"));
+
+ // Region 3 will contain the key range [row_0500,row_0600) and is not
+ // adjacent to any of Regions 0, 1, 2 or the merged result of any or all
+ // of those regions
+ sourceRegions[3] =
+ new HRegionInfo(this.desc, Bytes.toBytes("row_0500"),
+ Bytes.toBytes("row_0600"));
+
+ // Region 4 will have empty start and end keys and overlaps all regions.
+ sourceRegions[4] =
+ new HRegionInfo(this.desc, HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.EMPTY_BYTE_ARRAY);
+
+ /*
+ * Now create some row keys
+ */
+ this.rows = new byte [5][][];
+ this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
+ this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
+ "row_035" });
+ this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
+ "row_0175", "row_0175"});
+ this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
+ "row_0560", "row_0560", "row_0560"});
+ this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
+ "row_1000", "row_1000", "row_1000", "row_1000" });
+
+ // Start up dfs
+ this.dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
+ this.fs = this.dfsCluster.getFileSystem();
+ conf.set("fs.default.name", fs.getUri().toString());
+ Path parentdir = fs.getHomeDirectory();
+ conf.set(HConstants.HBASE_DIR, parentdir.toString());
+ fs.mkdirs(parentdir);
+ FSUtils.setVersion(fs, parentdir);
+
+ // Note: we must call super.setUp after starting the mini cluster or
+ // we will end up with a local file system
+
+ super.setUp();
+ try {
+ // Create root and meta regions
+ createRootAndMetaRegions();
+ /*
+ * Create the regions we will merge
+ */
+ for (int i = 0; i < sourceRegions.length; i++) {
+ regions[i] =
+ HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf);
+ /*
+ * Insert data
+ */
+ for (int j = 0; j < rows[i].length; j++) {
+ byte [] row = rows[i][j];
+ Put put = new Put(row);
+ put.add(FAMILY, QUALIFIER, row);
+ regions[i].put(put);
+ }
+ HRegion.addRegionToMETA(meta, regions[i]);
+ }
+ // Close root and meta regions
+ closeRootAndMeta();
+
+ } catch (Exception e) {
+ shutdownDfs(dfsCluster);
+ throw e;
+ }
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ shutdownDfs(dfsCluster);
+ }
+
+ /*
+ * @param msg Message that describes this merge
+ * @param regionName1
+ * @param regionName2
+ * @param log Log to use merging.
+ * @param upperbound Verifying, how high up in this.rows to go.
+ * @return Merged region.
+ * @throws Exception
+ */
+ private HRegion mergeAndVerify(final String msg, final String regionName1,
+ final String regionName2, final HLog log, final int upperbound)
+ throws Exception {
+ Merge merger = new Merge(this.conf);
+ LOG.info(msg);
+ int errCode = ToolRunner.run(merger,
+ new String[] {this.desc.getNameAsString(), regionName1, regionName2}
+ );
+ assertTrue("'" + msg + "' failed", errCode == 0);
+ HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
+
+ // Now verify that we can read all the rows from regions 0, 1
+ // in the new merged region.
+ HRegion merged =
+ HRegion.openHRegion(mergedInfo, this.testDir, log, this.conf);
+ verifyMerge(merged, upperbound);
+ merged.close();
+ LOG.info("Verified " + msg);
+ return merged;
+ }
+
+ private void verifyMerge(final HRegion merged, final int upperbound)
+ throws IOException {
+ //Test
+ Scan scan = new Scan();
+ scan.addFamily(FAMILY);
+ InternalScanner scanner = merged.getScanner(scan);
+ try {
+ List<KeyValue> testRes = null;
+ while (true) {
+ testRes = new ArrayList<KeyValue>();
+ boolean hasNext = scanner.next(testRes);
+ if (!hasNext) {
+ break;
+ }
+ }
+ } finally {
+ scanner.close();
+ }
+
+ //!Test
+
+ for (int i = 0; i < upperbound; i++) {
+ for (int j = 0; j < rows[i].length; j++) {
+ Get get = new Get(rows[i][j]);
+ get.addFamily(FAMILY);
+ Result result = merged.get(get, null);
+ assertEquals(1, result.size());
+ byte [] bytes = result.sorted()[0].getValue();
+ assertNotNull(rows[i][j].toString(), bytes);
+ assertTrue(Bytes.equals(bytes, rows[i][j]));
+ }
+ }
+ }
+
+ /**
+ * Test merge tool.
+ * @throws Exception
+ */
+ public void testMergeTool() throws Exception {
+ // First verify we can read the rows from the source regions and that they
+ // contain the right data.
+ for (int i = 0; i < regions.length; i++) {
+ for (int j = 0; j < rows[i].length; j++) {
+ Get get = new Get(rows[i][j]);
+ get.addFamily(FAMILY);
+ Result result = regions[i].get(get, null);
+ byte [] bytes = result.sorted()[0].getValue();
+ assertNotNull(bytes);
+ assertTrue(Bytes.equals(bytes, rows[i][j]));
+ }
+ // Close the region and delete the log
+ regions[i].close();
+ regions[i].getLog().closeAndDelete();
+ }
+
+ // Create a log that we can reuse when we need to open regions
+ Path logPath = new Path("/tmp", HConstants.HREGION_LOGDIR_NAME + "_" +
+ System.currentTimeMillis());
+ LOG.info("Creating log " + logPath.toString());
+ HLog log = new HLog(this.fs, logPath, this.conf, null);
+ try {
+ // Merge Region 0 and Region 1
+ HRegion merged = mergeAndVerify("merging regions 0 and 1",
+ this.sourceRegions[0].getRegionNameAsString(),
+ this.sourceRegions[1].getRegionNameAsString(), log, 2);
+
+ // Merge the result of merging regions 0 and 1 with region 2
+ merged = mergeAndVerify("merging regions 0+1 and 2",
+ merged.getRegionInfo().getRegionNameAsString(),
+ this.sourceRegions[2].getRegionNameAsString(), log, 3);
+
+ // Merge the result of merging regions 0, 1 and 2 with region 3
+ merged = mergeAndVerify("merging regions 0+1+2 and 3",
+ merged.getRegionInfo().getRegionNameAsString(),
+ this.sourceRegions[3].getRegionNameAsString(), log, 4);
+
+ // Merge the result of merging regions 0, 1, 2 and 3 with region 4
+ merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
+ merged.getRegionInfo().getRegionNameAsString(),
+ this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
+ } finally {
+ log.closeAndDelete();
+ }
+ }
+}
\ No newline at end of file