You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2009/06/18 22:47:04 UTC
svn commit: r786259 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/regionserver/
src/java/org/apache/hadoop/hbase/thrift/
src/test/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/...
Author: rawson
Date: Thu Jun 18 20:47:03 2009
New Revision: 786259
URL: http://svn.apache.org/viewvc?rev=786259&view=rev
Log:
HBASE-1541, 1488 (thrift server only)
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
Removed:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Delete.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Result.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Jun 18 20:47:03 2009
@@ -358,6 +358,9 @@
HBASE-1538 Up zookeeper timeout from 10 seconds to 30 seconds to cut down
on hbase-user traffic
HBASE-1539 prevent aborts due to missing zoo.cfg
+ HBASE-1488 Fix TestThriftServer and re-enable it
+ HBASE-1541 Scanning multiple column families in the presence of deleted
+ families results in bad scans
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Delete.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Delete.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Delete.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Delete.java Thu Jun 18 20:47:03 2009
@@ -105,6 +105,14 @@
}
/**
+ * Method to check if the familyMap is empty
+ * @return true if empty, false otherwise
+ */
+ public boolean isEmpty() {
+ return familyMap.isEmpty();
+ }
+
+ /**
* Delete all versions of all columns of the specified family.
* <p>
* Overrides previous calls to deleteColumn and deleteColumns for the
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Result.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/Result.java Thu Jun 18 20:47:03 2009
@@ -235,6 +235,8 @@
public Cell getCellValue(byte[] family, byte[] qualifier) {
Map.Entry<Long,byte[]> val = getKeyValue(family, qualifier);
+ if (val == null)
+ return null;
return new Cell(val.getValue(), val.getKey());
}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java Thu Jun 18 20:47:03 2009
@@ -89,8 +89,8 @@
int timeRes = Bytes.compareTo(memBuffer, tsOffset, Bytes.SIZEOF_LONG,
deleteBuffer, deleteTimeOffset, Bytes.SIZEOF_LONG);
- if(deleteType == KeyValue.Type.DeleteFamily.getCode()) {
- if(timeRes <= 0){
+ if (deleteType == KeyValue.Type.DeleteFamily.getCode()) {
+ if (timeRes <= 0) {
return DeleteCode.DELETE;
}
return DeleteCode.SKIP;
@@ -99,16 +99,16 @@
//Compare columns
res = Bytes.compareTo(memBuffer, memOffset, memQualifierLen,
deleteBuffer, deleteQualifierOffset, deleteQualifierLength);
- if(res < 0) {
+ if (res < 0) {
return DeleteCode.SKIP;
} else if(res > 0) {
return DeleteCode.DONE;
}
// same column, compare the time.
- if(timeRes == 0) {
+ if (timeRes == 0) {
return DeleteCode.DELETE;
} else if (timeRes < 0) {
- if(deleteType == KeyValue.Type.DeleteColumn.getCode()) {
+ if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
return DeleteCode.DELETE;
}
return DeleteCode.DONE;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Jun 18 20:47:03 2009
@@ -1710,7 +1710,7 @@
if(stopRow != null &&
comparator.compareRows(stopRow, 0, stopRow.length,
currentRow, 0, currentRow.length)
- <= 0){
+ <= 0) {
return false;
}
this.storeHeap.next(results);
@@ -1721,6 +1721,21 @@
}
byte [] row = kv.getRow();
if(!Bytes.equals(currentRow, row)) {
+ // Next row:
+
+ // what happens if there are _no_ results:
+ if (results.isEmpty()) {
+ // Continue on the next row:
+ currentRow = row;
+
+ // But did we pass the stop row?
+ if (stopRow != null &&
+ comparator.compareRows(stopRow, 0, stopRow.length,
+ currentRow, 0, currentRow.length) <= 0) {
+ return false;
+ }
+ continue;
+ }
return true;
}
this.storeHeap.next(results);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java Thu Jun 18 20:47:03 2009
@@ -251,13 +251,13 @@
byte deleteType = deleteBuffer[deleteOffset];
//Comparing with tail from memcache
- for(KeyValue mem : tailSet) {
+ for (KeyValue mem : tailSet) {
DeleteCode res = DeleteCompare.deleteCompare(mem, deleteBuffer,
deleteRowOffset, deleteRowLen, deleteQualifierOffset,
deleteQualifierLen, deleteTimestampOffset, deleteType,
comparator.getRawComparator());
- if(res == DeleteCode.DONE) {
+ if (res == DeleteCode.DONE) {
break;
} else if (res == DeleteCode.DELETE) {
deletes.add(mem);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Thu Jun 18 20:47:03 2009
@@ -123,7 +123,8 @@
* @param result
* @return true if there are more rows, false if scanner is done
*/
- public synchronized boolean next(List<KeyValue> result) throws IOException {
+ public synchronized boolean next(List<KeyValue> outResult) throws IOException {
+ List<KeyValue> results = new ArrayList<KeyValue>();
KeyValue peeked = this.heap.peek();
if (peeked == null) {
close();
@@ -136,27 +137,25 @@
switch(qcode) {
case INCLUDE:
KeyValue next = this.heap.next();
- result.add(next);
+ results.add(next);
continue;
case DONE:
- // what happens if we have 0 results?
- if (result.isEmpty()) {
- // try the next one.
- matcher.setRow(this.heap.peek().getRow());
- continue;
- }
if (matcher.filterEntireRow()) {
- // wow, well, um, reset the result and continue.
- result.clear();
- matcher.setRow(heap.peek().getRow());
- continue;
+ // nuke all results, and then return.
+ results.clear();
}
+ // copy jazz
+ outResult.addAll(results);
return true;
case DONE_SCAN:
close();
+
+ // copy jazz
+ outResult.addAll(results);
+
return false;
case SEEK_NEXT_ROW:
@@ -178,9 +177,14 @@
throw new RuntimeException("UNEXPECTED");
}
}
- if(result.size() > 0) {
+
+ if (!results.isEmpty()) {
+ // copy jazz
+ outResult.addAll(results);
+
return true;
}
+
// No more keys
close();
return false;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java Thu Jun 18 20:47:03 2009
@@ -298,12 +298,15 @@
HTable table = getTable(tableName);
Get get = new Get(row);
get.addColumn(family, qualifier);
- get.setTimeStamp(timestamp);
+ get.setTimeRange(Long.MIN_VALUE, timestamp);
get.setMaxVersions(numVersions);
Result result = table.get(get);
List<Cell> cells = new ArrayList<Cell>();
- for(KeyValue kv : result.sorted()) {
- cells.add(new Cell(kv.getValue(), kv.getTimestamp()));
+ KeyValue [] kvs = result.sorted();
+ if (kvs != null) {
+ for(KeyValue kv : kvs) {
+ cells.add(new Cell(kv.getValue(), kv.getTimestamp()));
+ }
}
return ThriftUtilities.cellFromHBase(cells.toArray(new Cell[0]));
} catch (IOException e) {
@@ -335,7 +338,7 @@
HTable table = getTable(tableName);
if (columns == null) {
Get get = new Get(row);
- get.setTimeStamp(timestamp);
+ get.setTimeRange(Long.MIN_VALUE, timestamp);
Result result = table.get(get);
return ThriftUtilities.rowResultFromHBase(result.getRowResult());
}
@@ -345,7 +348,7 @@
byte [][] famAndQf = KeyValue.parseColumn(column);
get.addColumn(famAndQf[0], famAndQf[1]);
}
- get.setTimeStamp(timestamp);
+ get.setTimeRange(Long.MIN_VALUE, timestamp);
Result result = table.get(get);
return ThriftUtilities.rowResultFromHBase(result.getRowResult());
} catch (IOException e) {
@@ -362,12 +365,12 @@
long timestamp) throws IOError {
try {
HTable table = getTable(tableName);
- Delete delete = new Delete(row, timestamp, null);
+ Delete delete = new Delete(row);
byte [][] famAndQf = KeyValue.parseColumn(column);
- if(famAndQf[1].length == 0){
- delete.deleteFamily(famAndQf[0]);
+ if (famAndQf[1].length == 0) {
+ delete.deleteFamily(famAndQf[0], timestamp);
} else {
- delete.deleteColumns(famAndQf[0], famAndQf[1]);
+ delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp);
}
table.delete(delete);
@@ -457,23 +460,36 @@
public void mutateRowsTs(byte[] tableName, List<BatchMutation> rowBatches, long timestamp)
throws IOError, IllegalArgument, TException {
List<Put> puts = new ArrayList<Put>();
-
+ List<Delete> deletes = new ArrayList<Delete>();
+
for (BatchMutation batch : rowBatches) {
byte[] row = batch.row;
List<Mutation> mutations = batch.mutations;
+ Delete delete = new Delete(row);
Put put = new Put(row);
put.setTimeStamp(timestamp);
for (Mutation m : mutations) {
byte [][] famAndQf = KeyValue.parseColumn(m.column);
- put.add(famAndQf[0], famAndQf[1], m.value);
- }
- puts.add(put);
+ if (m.isDelete) {
+ delete.deleteColumns(famAndQf[0], famAndQf[1]);
+ } else {
+ put.add(famAndQf[0], famAndQf[1], m.value);
+ }
+ }
+ if (!delete.isEmpty())
+ deletes.add(delete);
+ if (!put.isEmpty())
+ puts.add(put);
}
HTable table = null;
try {
table = getTable(tableName);
- table.put(puts);
+ if (!puts.isEmpty())
+ table.put(puts);
+ for (Delete del : deletes) {
+ table.delete(del);
+ }
} catch (IOException e) {
throw new IOError(e.getMessage());
} catch (IllegalArgumentException e) {
@@ -534,19 +550,19 @@
public int scannerOpen(byte[] tableName, byte[] startRow,
List<byte[]> columns) throws IOError {
try {
- HTable table = getTable(tableName);
- byte[][] columnsArray = null;
- if ((columns == null) || (columns.size() == 0)) {
- columnsArray = getAllColumns(table);
- } else {
- columnsArray = columns.toArray(new byte[0][]);
+ HTable table = getTable(tableName);
+ byte[][] columnsArray = null;
+ if ((columns == null) || (columns.size() == 0)) {
+ columnsArray = getAllColumns(table);
+ } else {
+ columnsArray = columns.toArray(new byte[0][]);
+ }
+ Scan scan = new Scan(startRow);
+ scan.addColumns(columnsArray);
+ return addScanner(table.getScanner(scan));
+ } catch (IOException e) {
+ throw new IOError(e.getMessage());
}
- Scan scan = new Scan(startRow);
- scan.addColumns(columnsArray);
- return addScanner(table.getScanner(scan));
- } catch (IOException e) {
- throw new IOError(e.getMessage());
- }
}
public int scannerOpenWithStop(byte[] tableName, byte[] startRow,
@@ -579,7 +595,7 @@
}
Scan scan = new Scan(startRow);
scan.addColumns(columnsArray);
- scan.setTimeRange(0, timestamp);
+ scan.setTimeRange(Long.MIN_VALUE, timestamp);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());
@@ -599,7 +615,7 @@
}
Scan scan = new Scan(startRow, stopRow);
scan.addColumns(columnsArray);
- scan.setTimeRange(0, timestamp);
+ scan.setTimeRange(Long.MIN_VALUE, timestamp);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Thu Jun 18 20:47:03 2009
@@ -285,10 +285,45 @@
get = new Get(row).addColumn(fam, splitB);
result = region.get(get, null);
assertEquals(1, result.size());
+ }
-
+ public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
+ byte [] tableName = Bytes.toBytes("test_table");
+ byte [] fam1 = Bytes.toBytes("columnA");
+ byte [] fam2 = Bytes.toBytes("columnB");
+ initHRegion(tableName, getName(), fam1, fam2);
+
+ byte [] rowA = Bytes.toBytes("rowA");
+ byte [] rowB = Bytes.toBytes("rowB");
+
+ byte [] value = Bytes.toBytes("value");
+
+ Delete delete = new Delete(rowA);
+ delete.deleteFamily(fam1);
+
+ region.delete(delete, null, true);
+
+ // now create data.
+ Put put = new Put(rowA);
+ put.add(fam2, null, value);
+ region.put(put);
+
+ put = new Put(rowB);
+ put.add(fam1, null, value);
+ put.add(fam2, null, value);
+ region.put(put);
+
+ Scan scan = new Scan();
+ scan.addFamily(fam1).addFamily(fam2);
+ InternalScanner s = region.getScanner(scan);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ s.next(results);
+ assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
+
+ results.clear();
+ s.next(results);
+ assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
-
}
//Visual test, since the method doesn't return anything
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java Thu Jun 18 20:47:03 2009
@@ -506,9 +506,12 @@
KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
+ KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
+
memcache.add(put1);
memcache.add(put2);
memcache.add(put3);
+ memcache.add(put4);
KeyValue del =
new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
@@ -516,8 +519,9 @@
List<KeyValue> expected = new ArrayList<KeyValue>();
expected.add(del);
+ expected.add(put4);
- assertEquals(1, memcache.memcache.size());
+ assertEquals(2, memcache.memcache.size());
int i=0;
for(KeyValue actual : memcache.memcache) {
assertEquals(expected.get(i++), actual);
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=786259&r1=786258&r2=786259&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java Thu Jun 18 20:47:03 2009
@@ -140,8 +140,13 @@
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
+ assertEquals(0, results.size());
+
+ assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[2], results.get(0));
+
+ assertEquals(false, scan.next(results));
}
public void testDeleteVersionMaskingMultiplePuts() throws IOException {
@@ -272,8 +277,12 @@
null, scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
+ assertEquals(0, results.size());
+ assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[kvs.length-1], results.get(0));
+
+ assertEquals(false, scan.next(results));
}
public void testDeleteColumn() throws IOException {
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java?rev=786259&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java Thu Jun 18 20:47:03 2009
@@ -0,0 +1,397 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.thrift;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
+import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
+import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
+import org.apache.hadoop.hbase.thrift.generated.Mutation;
+import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Unit testing for ThriftServer.HBaseHandler, a part of the
+ * org.apache.hadoop.hbase.thrift package.
+ */
+public class TestThriftServer extends HBaseClusterTestCase {
+
+ // Static names for tables, columns, rows, and values
+ private static byte[] tableAname = Bytes.toBytes("tableA");
+ private static byte[] tableBname = Bytes.toBytes("tableB");
+ private static byte[] columnAname = Bytes.toBytes("columnA:");
+ private static byte[] columnBname = Bytes.toBytes("columnB:");
+ private static byte[] rowAname = Bytes.toBytes("rowA");
+ private static byte[] rowBname = Bytes.toBytes("rowB");
+ private static byte[] valueAname = Bytes.toBytes("valueA");
+ private static byte[] valueBname = Bytes.toBytes("valueB");
+ private static byte[] valueCname = Bytes.toBytes("valueC");
+ private static byte[] valueDname = Bytes.toBytes("valueD");
+
+ /**
+ * Runs all of the tests under a single JUnit test method. We
+ * consolidate all testing to one method because HBaseClusterTestCase
+ * is prone to OutOfMemoryExceptions when there are three or more
+ * JUnit test methods.
+ *
+ * @throws Exception
+ */
+ public void testAll() throws Exception {
+ // Run all tests
+ doTestTableCreateDrop();
+ doTestTableMutations();
+ doTestTableTimestampsAndColumns();
+ doTestTableScanners();
+ }
+
+ /**
+ * Tests for creating, enabling, disabling, and deleting tables. Also
+ * tests that creating a table with an invalid column name yields an
+ * IllegalArgument exception.
+ *
+ * @throws Exception
+ */
+ public void doTestTableCreateDrop() throws Exception {
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+
+ // Create/enable/disable/delete tables, ensure methods act correctly
+ assertEquals(handler.getTableNames().size(), 0);
+ handler.createTable(tableAname, getColumnDescriptors());
+ assertEquals(handler.getTableNames().size(), 1);
+ assertEquals(handler.getColumnDescriptors(tableAname).size(), 2);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.createTable(tableBname, new ArrayList<ColumnDescriptor>());
+ assertEquals(handler.getTableNames().size(), 2);
+ handler.disableTable(tableBname);
+ assertFalse(handler.isTableEnabled(tableBname));
+ handler.deleteTable(tableBname);
+ assertEquals(handler.getTableNames().size(), 1);
+ handler.disableTable(tableAname);
+ assertFalse(handler.isTableEnabled(tableAname));
+ handler.enableTable(tableAname);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Tests adding a series of Mutations and BatchMutations, including a
+ * delete mutation. Also tests data retrieval, and getting back multiple
+ * versions.
+ *
+ * @throws Exception
+ */
+ public void doTestTableMutations() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply a few Mutations to rowA
+ // mutations.add(new Mutation(false, columnAname, valueAname));
+ // mutations.add(new Mutation(false, columnBname, valueBname));
+ handler.mutateRow(tableAname, rowAname, getMutations());
+
+ // Assert that the changes were made
+ assertTrue(Bytes.equals(valueAname,
+ handler.get(tableAname, rowAname, columnAname).get(0).value));
+ TRowResult rowResult1 = handler.getRow(tableAname, rowAname).get(0);
+ assertTrue(Bytes.equals(rowAname, rowResult1.row));
+ assertTrue(Bytes.equals(valueBname,
+ rowResult1.columns.get(columnBname).value));
+
+ // Apply a few BatchMutations for rowA and rowB
+ // rowAmutations.add(new Mutation(true, columnAname, null));
+ // rowAmutations.add(new Mutation(false, columnBname, valueCname));
+ // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+ // Mutations to rowB
+ // rowBmutations.add(new Mutation(false, columnAname, valueCname));
+ // rowBmutations.add(new Mutation(false, columnBname, valueDname));
+ // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
+ handler.mutateRows(tableAname, getBatchMutations());
+
+ // Assert that changes were made to rowA
+ List<TCell> cells = handler.get(tableAname, rowAname, columnAname);
+ assertFalse(cells.size() > 0);
+ assertTrue(Bytes.equals(valueCname, handler.get(tableAname, rowAname, columnBname).get(0).value));
+ List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS);
+ assertTrue(Bytes.equals(valueCname, versions.get(0).value));
+ assertTrue(Bytes.equals(valueBname, versions.get(1).value));
+
+ // Assert that changes were made to rowB
+ TRowResult rowResult2 = handler.getRow(tableAname, rowBname).get(0);
+ assertTrue(Bytes.equals(rowBname, rowResult2.row));
+ assertTrue(Bytes.equals(valueCname, rowResult2.columns.get(columnAname).value));
+ assertTrue(Bytes.equals(valueDname, rowResult2.columns.get(columnBname).value));
+
+ // Apply some deletes
+ handler.deleteAll(tableAname, rowAname, columnBname);
+ handler.deleteAllRow(tableAname, rowBname);
+
+ // Assert that the deletes were applied
+ int size = handler.get(tableAname, rowAname, columnBname).size();
+ assertEquals(0, size);
+ size = handler.getRow(tableAname, rowBname).size();
+ assertEquals(0, size);
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Similar to testTableMutations(), except Mutations are applied with
+ * specific timestamps and data retrieval uses these timestamps to
+ * extract specific versions of data.
+ *
+ * @throws Exception
+ */
+ public void doTestTableTimestampsAndColumns() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ // Apply an overlapping timestamped mutation to rowB
+ handler.mutateRowTs(tableAname, rowBname, getMutations(), time2);
+
+ // the getVerTs is [inf, ts) so you need to increment one.
+ time1 += 1;
+ time2 += 2;
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
+ MAXVERSIONS).size());
+ assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
+ MAXVERSIONS).size());
+
+ TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0);
+ TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0);
+ // columnA was completely deleted
+ //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname));
+
+ // ColumnAname has been deleted, and will never be visible even with a getRowTs()
+ assertFalse(rowResult2.columns.containsKey(columnAname));
+
+ List<byte[]> columns = new ArrayList<byte[]>();
+ columns.add(columnBname);
+
+ rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueCname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ // Apply some timestamped deletes
+ // this actually deletes _everything_.
+ // nukes everything in columnB: forever.
+ handler.deleteAllTs(tableAname, rowAname, columnBname, time1);
+ handler.deleteAllRowTs(tableAname, rowBname, time2);
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size();
+ assertEquals(0, size);
+
+ size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS).size();
+ assertEquals(1, size);
+
+ // should be available....
+ assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname));
+
+ assertEquals(0, handler.getRow(tableAname, rowBname).size());
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Tests the four different scanner-opening methods (with and without
+ * a stoprow, with and without a timestamp).
+ *
+ * @throws Exception
+ */
+ public void doTestTableScanners() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ // Sleep to assure that 'time1' and 'time2' will be different even with a
+ // coarse grained system timer.
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ time1 += 1;
+ time2 += 1;
+
+ // Test a scanner on all rows and all columns, no timestamp
+ int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true));
+ TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1a.row, rowAname));
+ // This used to be '1'. I don't know why when we are asking for two columns
+ // and when the mutations above would seem to add two columns to the row.
+ // -- St.Ack 05/12/2009
+ assertEquals(rowResult1a.columns.size(), 1);
+ assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname));
+
+ TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1b.row, rowBname));
+ assertEquals(rowResult1b.columns.size(), 2);
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnAname).value, valueCname));
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnBname).value, valueDname));
+ closeScanner(scanner1, handler);
+
+ // Test a scanner on all rows and all columns, with timestamp
+ int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1);
+ TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
+ assertEquals(rowResult2a.columns.size(), 1);
+ // column A deleted, does not exist.
+ //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname));
+ closeScanner(scanner2, handler);
+
+ // Test a scanner on the first row and first column only, no timestamp
+ int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
+ getColumnList(true, false));
+ closeScanner(scanner3, handler);
+
+ // Test a scanner on the first row and second column only, with timestamp
+ int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
+ getColumnList(false, true), time1);
+ TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
+ assertEquals(rowResult4a.columns.size(), 1);
+ assertTrue(Bytes.equals(rowResult4a.columns.get(columnBname).value, valueBname));
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ *
+ * @return a List of ColumnDescriptors for use in creating a table. Has one
+ * default ColumnDescriptor and one ColumnDescriptor with fewer versions
+ */
+ private List<ColumnDescriptor> getColumnDescriptors() {
+ ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<ColumnDescriptor>();
+
+ // A default ColumnDescriptor
+ ColumnDescriptor cDescA = new ColumnDescriptor();
+ cDescA.name = columnAname;
+ cDescriptors.add(cDescA);
+
+ // A slightly customized ColumnDescriptor (only 2 versions)
+ ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
+ false, 2147483647, "NONE", 0, 0, false, -1);
+ cDescriptors.add(cDescB);
+
+ return cDescriptors;
+ }
+
+ /**
+ *
+ * @param includeA whether or not to include columnA
+ * @param includeB whether or not to include columnB
+ * @return a List of column names for use in retrieving a scanner
+ */
+ private List<byte[]> getColumnList(boolean includeA, boolean includeB) {
+ List<byte[]> columnList = new ArrayList<byte[]>();
+ if (includeA) columnList.add(columnAname);
+ if (includeB) columnList.add(columnBname);
+ return columnList;
+ }
+
+ /**
+ *
+ * @return a List of Mutations for a row, with columnA having valueA
+ * and columnB having valueB
+ */
+ private List<Mutation> getMutations() {
+ List<Mutation> mutations = new ArrayList<Mutation>();
+ mutations.add(new Mutation(false, columnAname, valueAname));
+ mutations.add(new Mutation(false, columnBname, valueBname));
+ return mutations;
+ }
+
+ /**
+ *
+ * @return a List of BatchMutations with the following effects:
+ * (rowA, columnA): delete
+ * (rowA, columnB): place valueC
+ * (rowB, columnA): place valueC
+ * (rowB, columnB): place valueD
+ */
+ private List<BatchMutation> getBatchMutations() {
+ List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
+
+ // Mutations to rowA. You can't mix delete and put anymore.
+ List<Mutation> rowAmutations = new ArrayList<Mutation>();
+ rowAmutations.add(new Mutation(true, columnAname, null));
+ batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+
+ rowAmutations = new ArrayList<Mutation>();
+ rowAmutations.add(new Mutation(false, columnBname, valueCname));
+ batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+
+ // Mutations to rowB
+ List<Mutation> rowBmutations = new ArrayList<Mutation>();
+ rowBmutations.add(new Mutation(false, columnAname, valueCname));
+ rowBmutations.add(new Mutation(false, columnBname, valueDname));
+ batchMutations.add(new BatchMutation(rowBname, rowBmutations));
+
+ return batchMutations;
+ }
+
+ /**
+ * Asserts that the passed scanner is exhausted, and then closes
+ * the scanner.
+ *
+ * @param scannerId the scanner to close
+ * @param handler the HBaseHandler interfacing to HBase
+ * @throws Exception
+ */
+ private void closeScanner(int scannerId, ThriftServer.HBaseHandler handler) throws Exception {
+ handler.scannerGet(scannerId);
+ handler.scannerClose(scannerId);
+ }
+}