You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/05/13 02:08:12 UTC
svn commit: r774145 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java
src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
Author: stack
Date: Wed May 13 00:08:12 2009
New Revision: 774145
URL: http://svn.apache.org/viewvc?rev=774145&view=rev
Log:
HBASE-1323 hbase-1234 broke TestThriftServer; fix and reenable
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
Removed:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=774145&r1=774144&r2=774145&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Wed May 13 00:08:12 2009
@@ -125,6 +125,7 @@
(Nitay Joffe via Stack)
HBASE-1391 NPE in TableInputFormatBase$TableRecordReader.restart if zoo.cfg
is wrong or missing on task trackers
+ HBASE-1323 hbase-1234 broke TestThriftServer; fix and reenable
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java?rev=774145&r1=774144&r2=774145&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java Wed May 13 00:08:12 2009
@@ -105,11 +105,14 @@
* @return Thrift TCell array
*/
static public List<TCell> cellFromHBase(Cell[] in) {
- List<TCell> list = new ArrayList<TCell>(in.length);
+ List<TCell> list = null;
if (in != null) {
+ list = new ArrayList<TCell>(in.length);
for (int i = 0; i < in.length; i++) {
list.add(new TCell(in[i].getValue(), in[i].getTimestamp()));
}
+ } else {
+ list = new ArrayList<TCell>(0);
}
return list;
}
Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java?rev=774145&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java Wed May 13 00:08:12 2009
@@ -0,0 +1,380 @@
+/**
+ * Copyright 2008-2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.thrift;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
+import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
+import org.apache.hadoop.hbase.thrift.generated.IllegalArgument;
+import org.apache.hadoop.hbase.thrift.generated.Mutation;
+import org.apache.hadoop.hbase.thrift.generated.TCell;
+import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Unit testing for ThriftServer.HBaseHandler, a part of the
+ * org.apache.hadoop.hbase.thrift package.
+ */
+public class TestThriftServer extends HBaseClusterTestCase {
+
+ // Static names for tables, columns, rows, and values
+ private static byte[] tableAname = Bytes.toBytes("tableA");
+ private static byte[] tableBname = Bytes.toBytes("tableB");
+ private static byte[] columnAname = Bytes.toBytes("columnA:");
+ private static byte[] columnBname = Bytes.toBytes("columnB:");
+ private static byte[] badColumnName = Bytes.toBytes("forgotColon");
+ private static byte[] rowAname = Bytes.toBytes("rowA");
+ private static byte[] rowBname = Bytes.toBytes("rowB");
+ private static byte[] valueAname = Bytes.toBytes("valueA");
+ private static byte[] valueBname = Bytes.toBytes("valueB");
+ private static byte[] valueCname = Bytes.toBytes("valueC");
+ private static byte[] valueDname = Bytes.toBytes("valueD");
+
+ /**
+ * Runs all of the tests under a single JUnit test method. We
+ * consolidate all testing to one method because HBaseClusterTestCase
+ * is prone to OutOfMemoryExceptions when there are three or more
+ * JUnit test methods.
+ *
+ * @throws Exception
+ */
+ public void testAll() throws Exception {
+ // Run all tests
+ doTestTableCreateDrop();
+ doTestTableMutations();
+ doTestTableTimestampsAndColumns();
+ doTestTableScanners();
+ }
+
+ /**
+ * Tests for creating, enabling, disabling, and deleting tables. Also
+ * tests that creating a table with an invalid column name yields an
+ * IllegalArgument exception.
+ *
+ * @throws Exception
+ */
+ public void doTestTableCreateDrop() throws Exception {
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+
+ // Create/enable/disable/delete tables, ensure methods act correctly
+ assertEquals(handler.getTableNames().size(), 0);
+ handler.createTable(tableAname, getColumnDescriptors());
+ assertEquals(handler.getTableNames().size(), 1);
+ assertEquals(handler.getColumnDescriptors(tableAname).size(), 2);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.createTable(tableBname, new ArrayList<ColumnDescriptor>());
+ assertEquals(handler.getTableNames().size(), 2);
+ handler.disableTable(tableBname);
+ assertFalse(handler.isTableEnabled(tableBname));
+ handler.deleteTable(tableBname);
+ assertEquals(handler.getTableNames().size(), 1);
+ handler.disableTable(tableAname);
+ assertFalse(handler.isTableEnabled(tableAname));
+ handler.enableTable(tableAname);
+ assertTrue(handler.isTableEnabled(tableAname));
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+
+ // Make sure that trying to create a table with a bad column name creates
+ // an IllegalArgument exception.
+ List<ColumnDescriptor> cDescriptors = new ArrayList<ColumnDescriptor>();
+ ColumnDescriptor badDescriptor = new ColumnDescriptor();
+ badDescriptor.name = badColumnName;
+ cDescriptors.add(badDescriptor);
+ String message = null;
+ try {
+ handler.createTable(tableBname, cDescriptors);
+ } catch (IllegalArgument ia) {
+ message = ia.message;
+ }
+ assertEquals("Family names must end in a colon: " + new String(badColumnName), message);
+ }
+
+ /**
+ * Tests adding a series of Mutations and BatchMutations, including a
+ * delete mutation. Also tests data retrieval, and getting back multiple
+ * versions.
+ *
+ * @throws Exception
+ */
+ public void doTestTableMutations() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply a few Mutations to rowA
+ handler.mutateRow(tableAname, rowAname, getMutations());
+
+ // Assert that the changes were made
+ assertTrue(Bytes.equals(valueAname, handler.get(tableAname, rowAname, columnAname).get(0).value));
+ TRowResult rowResult1 = handler.getRow(tableAname, rowAname).get(0);
+ assertTrue(Bytes.equals(rowAname, rowResult1.row));
+ assertTrue(Bytes.equals(valueBname, rowResult1.columns.get(columnBname).value));
+
+ // Apply a few BatchMutations for rowA and rowB
+ handler.mutateRows(tableAname, getBatchMutations());
+
+ // Assert that changes were made to rowA
+ assertFalse(handler.get(tableAname, rowAname, columnAname).size() > 0);
+ assertTrue(Bytes.equals(valueCname, handler.get(tableAname, rowAname, columnBname).get(0).value));
+ List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS);
+ assertTrue(Bytes.equals(valueCname, versions.get(0).value));
+ assertTrue(Bytes.equals(valueBname, versions.get(1).value));
+
+ // Assert that changes were made to rowB
+ TRowResult rowResult2 = handler.getRow(tableAname, rowBname).get(0);
+ assertTrue(Bytes.equals(rowBname, rowResult2.row));
+ assertTrue(Bytes.equals(valueCname, rowResult2.columns.get(columnAname).value));
+ assertTrue(Bytes.equals(valueDname, rowResult2.columns.get(columnBname).value));
+
+ // Apply some deletes
+ handler.deleteAll(tableAname, rowAname, columnBname);
+ handler.deleteAllRow(tableAname, rowBname);
+
+ // Assert that the deletes were applied
+ int size = handler.get(tableAname, rowAname, columnBname).size();
+ assertEquals(0, size);
+ size = handler.getRow(tableAname, rowBname).size();
+ assertEquals(0, size);
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Similar to testTableMutations(), except Mutations are applied with
+ * specific timestamps and data retrieval uses these timestamps to
+ * extract specific versions of data.
+ *
+ * @throws Exception
+ */
+ public void doTestTableTimestampsAndColumns() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ // Sleep to assure that 'time1' and 'time2' will be different even with a
+ // coarse grained system timer.
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ // Apply an overlapping timestamped mutation to rowB
+ handler.mutateRowTs(tableAname, rowBname, getMutations(), time2);
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time2,
+ MAXVERSIONS).size(), 2);
+ assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time1,
+ MAXVERSIONS).size(), 1);
+
+ TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0);
+ TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname));
+
+ // Maybe I'd reading this wrong but at line #187 above, the BatchMutations
+ // are adding a columnAname at time2 so the below should be true not false
+ // -- St.Ack
+ assertTrue(rowResult2.columns.containsKey(columnAname));
+
+ List<byte[]> columns = new ArrayList<byte[]>();
+ columns.add(columnBname);
+
+ rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueCname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1).get(0);
+ assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
+ assertFalse(rowResult1.columns.containsKey(columnAname));
+
+ // Apply some timestamped deletes
+ handler.deleteAllTs(tableAname, rowAname, columnBname, time1);
+ handler.deleteAllRowTs(tableAname, rowBname, time2);
+
+ // Assert that the timestamp-related methods retrieve the correct data
+ int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size();
+ assertFalse(size > 0);
+ assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname));
+ assertFalse(handler.getRow(tableAname, rowBname).size() > 0);
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ * Tests the four different scanner-opening methods (with and without
+ * a stoprow, with and without a timestamp).
+ *
+ * @throws Exception
+ */
+ public void doTestTableScanners() throws Exception {
+ // Setup
+ ThriftServer.HBaseHandler handler = new ThriftServer.HBaseHandler();
+ handler.createTable(tableAname, getColumnDescriptors());
+
+ // Apply timestamped Mutations to rowA
+ long time1 = System.currentTimeMillis();
+ handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
+
+ // Sleep to assure that 'time1' and 'time2' will be different even with a
+ // coarse grained system timer.
+ Thread.sleep(1000);
+
+ // Apply timestamped BatchMutations for rowA and rowB
+ long time2 = System.currentTimeMillis();
+ handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
+
+ // Test a scanner on all rows and all columns, no timestamp
+ int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true));
+ TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1a.row, rowAname));
+ // This used to be '1'. I don't know why when we are asking for two columns
+ // and when the mutations above would seem to add two columns to the row.
+ // -- St.Ack 05/12/2009
+ assertEquals(rowResult1a.columns.size(), 2);
+ assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname));
+ TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
+ assertTrue(Bytes.equals(rowResult1b.row, rowBname));
+ assertEquals(rowResult1b.columns.size(), 2);
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnAname).value, valueCname));
+ assertTrue(Bytes.equals(rowResult1b.columns.get(columnBname).value, valueDname));
+ closeScanner(scanner1, handler);
+
+ // Test a scanner on all rows and all columns, with timestamp
+ int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1);
+ TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
+ assertEquals(rowResult2a.columns.size(), 2);
+ assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
+ assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname));
+ closeScanner(scanner2, handler);
+
+ // Test a scanner on the first row and first column only, no timestamp
+ int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
+ getColumnList(true, false));
+ closeScanner(scanner3, handler);
+
+ // Test a scanner on the first row and second column only, with timestamp
+ int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
+ getColumnList(false, true), time1);
+ TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
+ assertEquals(rowResult4a.columns.size(), 1);
+ assertTrue(Bytes.equals(rowResult4a.columns.get(columnBname).value, valueBname));
+
+ // Teardown
+ handler.disableTable(tableAname);
+ handler.deleteTable(tableAname);
+ }
+
+ /**
+ *
+ * @return a List of ColumnDescriptors for use in creating a table. Has one
+ * default ColumnDescriptor and one ColumnDescriptor with fewer versions
+ */
+ private List<ColumnDescriptor> getColumnDescriptors() {
+ ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<ColumnDescriptor>();
+
+ // A default ColumnDescriptor
+ ColumnDescriptor cDescA = new ColumnDescriptor();
+ cDescA.name = columnAname;
+ cDescriptors.add(cDescA);
+
+ // A slightly customized ColumnDescriptor (only 2 versions)
+ ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
+ false, 2147483647, "NONE", 0, 0, false, -1);
+ cDescriptors.add(cDescB);
+
+ return cDescriptors;
+ }
+
+ /**
+ *
+ * @param includeA whether or not to include columnA
+ * @param includeB whether or not to include columnB
+ * @return a List of column names for use in retrieving a scanner
+ */
+ private List<byte[]> getColumnList(boolean includeA, boolean includeB) {
+ List<byte[]> columnList = new ArrayList<byte[]>();
+ if (includeA) columnList.add(columnAname);
+ if (includeB) columnList.add(columnBname);
+ return columnList;
+ }
+
+ /**
+ *
+ * @return a List of Mutations for a row, with columnA having valueA
+ * and columnB having valueB
+ */
+ private List<Mutation> getMutations() {
+ List<Mutation> mutations = new ArrayList<Mutation>();
+ mutations.add(new Mutation(false, columnAname, valueAname));
+ mutations.add(new Mutation(false, columnBname, valueBname));
+ return mutations;
+ }
+
+ /**
+ *
+ * @return a List of BatchMutations with the following effects:
+ * (rowA, columnA): delete
+ * (rowA, columnB): place valueC
+ * (rowB, columnA): place valueC
+ * (rowB, columnB): place valueD
+ */
+ private List<BatchMutation> getBatchMutations() {
+ List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
+ // Mutations to rowA
+ List<Mutation> rowAmutations = new ArrayList<Mutation>();
+ rowAmutations.add(new Mutation(true, columnAname, null));
+ rowAmutations.add(new Mutation(false, columnBname, valueCname));
+ batchMutations.add(new BatchMutation(rowAname, rowAmutations));
+ // Mutations to rowB
+ List<Mutation> rowBmutations = new ArrayList<Mutation>();
+ rowBmutations.add(new Mutation(false, columnAname, valueCname));
+ rowBmutations.add(new Mutation(false, columnBname, valueDname));
+ batchMutations.add(new BatchMutation(rowBname, rowBmutations));
+ return batchMutations;
+ }
+
+ /**
+ * Asserts that the passed scanner is exhausted, and then closes
+ * the scanner.
+ *
+ * @param scannerId the scanner to close
+ * @param handler the HBaseHandler interfacing to HBase
+ * @throws Exception
+ */
+ private void closeScanner(int scannerId, ThriftServer.HBaseHandler handler) throws Exception {
+ handler.scannerGet(scannerId);
+ handler.scannerClose(scannerId);
+ }
+}