You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/07/24 07:21:01 UTC
svn commit: r797323 [1/2] - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ bin/
src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/
src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/
src/contrib/tr...
Author: apurtell
Date: Fri Jul 24 05:20:59 2009
New Revision: 797323
URL: http://svn.apache.org/viewvc?rev=797323&view=rev
Log:
HBASE-1215, HBASE-1689, HBASE-1680, HBASE-1607, HBASE-1688, HBABSE-1694
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase
hadoop/hbase/trunk_on_hadoop-0.18.3/build.xml
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableDescriptor.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/JtaXAResource.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionalTable.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexMaintenanceUtils.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/THLog.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionState.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegionServer.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/tableindexed/TestIndexedTable.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/transactional/TestTransactions.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerAddress.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnection.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/FilterList.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/io/HBaseMapFile.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/BloomFilter.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/CountingBloomFilter.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/DynamicBloomFilter.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Filter.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/HashFunction.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Key.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/RetouchedBloomFilter.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/regionserver/HStoreFile.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/package-info.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/FSUtils.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/MetaUtils.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Migrate.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/overview.html
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/util/TestBytes.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/webapps/regionserver/regionserver.jsp
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Fri Jul 24 05:20:59 2009
@@ -277,6 +277,13 @@
HBASE-1669 need dynamic extensibility of HBaseRPC code maps and interface
lists (Clint Morgan via Stack)
HBASE-1359 After a large truncating table HBase becomes unresponsive
+ HBASE-1215 0.19.0 -> 0.20.0 migration (hfile, HCD changes, HSK changes)
+ HBASE-1689 Fix javadoc warnings and add overview on client classes to
+ client package
+ HBASE-1680 FilterList writable only works for HBaseObjectWritable
+ defined types (Clint Morgan via Stack and Jon Gray)
+ HBASE-1607 transactions / indexing fixes: trx deletes not handeled, index
+ scan can't specify stopRow (Clint Morgan via Stack)
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -498,6 +505,9 @@
HBASE-1609 We wait on leases to expire before regionserver goes down.
Rather, just let client fail
HBASE-1655 Usability improvements to HTablePool (Ken Weiner via jgray)
+ HBASE-1688 Improve javadocs in Result and KeyValue
+ HBASE-1694 Add TOC to 'Getting Started', add references to THBase and
+ ITHBase
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/bin/hbase Fri Jul 24 05:20:59 2009
@@ -134,7 +134,7 @@
CLASSPATH=${CLASSPATH}:$f;
done
-for f in $HBASE_HOME/lib/jetty-ext/*.jar; do
+for f in $HBASE_HOME/lib/jsp-2.1/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/build.xml?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/build.xml (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/build.xml Fri Jul 24 05:20:59 2009
@@ -516,9 +516,7 @@
</subant>
</target>
- <!-- REENABLE!!! <target name="test" depends="test-core, test-contrib"
- -->
- <target name="test" depends="test-core"
+ <target name="test" depends="test-core, test-contrib"
description="Run core, contrib unit tests">
</target>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTable.java Fri Jul 24 05:20:59 2009
@@ -30,7 +30,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HStoreKey;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
@@ -39,14 +38,12 @@
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.transactional.TransactionalTable;
import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.io.Cell;
-import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.util.Bytes;
/** HTable extended with indexed support. */
public class IndexedTable extends TransactionalTable {
- // FIXME, these belong elsewhere
+ // TODO move these schema constants elsewhere
public static final byte[] INDEX_COL_FAMILY_NAME = Bytes.toBytes("__INDEX__");
public static final byte[] INDEX_COL_FAMILY = Bytes.add(
INDEX_COL_FAMILY_NAME, new byte[] { HStoreKey.COLUMN_FAMILY_DELIMITER });
@@ -79,6 +76,7 @@
*
* @param indexId the id of the index to use
* @param indexStartRow (created from the IndexKeyGenerator)
+ * @param indexStopRow (created from the IndexKeyGenerator)
* @param indexColumns in the index table
* @param indexFilter filter to run on the index'ed table. This can only use
* columns that have been added to the index.
@@ -87,7 +85,7 @@
* @throws IOException
* @throws IndexNotFoundException
*/
- public ResultScanner getIndexedScanner(String indexId, final byte[] indexStartRow,
+ public ResultScanner getIndexedScanner(String indexId, final byte[] indexStartRow, final byte[] indexStopRow,
byte[][] indexColumns, final Filter indexFilter,
final byte[][] baseColumns) throws IOException, IndexNotFoundException {
IndexSpecification indexSpec = this.indexedTableDescriptor.getIndex(indexId);
@@ -114,9 +112,15 @@
allIndexColumns[allColumns.length] = INDEX_BASE_ROW_COLUMN;
}
- Scan indexScan = new Scan(indexStartRow);
- //indexScan.setFilter(filter); // FIXME
+ Scan indexScan = new Scan();
+ indexScan.setFilter(indexFilter);
indexScan.addColumns(allIndexColumns);
+ if (indexStartRow != null) {
+ indexScan.setStartRow(indexStartRow);
+ }
+ if (indexStopRow != null) {
+ indexScan.setStopRow(indexStopRow);
+ }
ResultScanner indexScanner = indexTable.getScanner(indexScan);
return new ScannerWrapper(indexScanner, baseColumns);
@@ -173,8 +177,6 @@
byte[] baseRow = row.getValue(INDEX_BASE_ROW_COLUMN);
LOG.debug("next index row [" + Bytes.toString(row.getRow())
+ "] -> base row [" + Bytes.toString(baseRow) + "]");
- HbaseMapWritable<byte[], Cell> colValues =
- new HbaseMapWritable<byte[], Cell>();
Result baseResult = null;
if (columns != null && columns.length > 0) {
LOG.debug("Going to base table for remaining columns");
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableDescriptor.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableDescriptor.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/tableindexed/IndexedTableDescriptor.java Fri Jul 24 05:20:59 2009
@@ -28,11 +28,8 @@
import java.util.HashMap;
import java.util.Map;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Writable;
public class IndexedTableDescriptor {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/JtaXAResource.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/JtaXAResource.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/JtaXAResource.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/JtaXAResource.java Fri Jul 24 05:20:59 2009
@@ -65,7 +65,9 @@
} catch (CommitUnsuccessfulException e) {
throw new XAException(XAException.XA_RBROLLBACK);
} catch (IOException e) {
- throw new XAException(XAException.XA_RBPROTO); // FIXME correct code?
+ XAException xae = new XAException(XAException.XAER_RMERR);
+ xae.initCause(e);
+ throw xae;
} finally {
threadLocalTransactionState.remove();
}
@@ -85,7 +87,9 @@
try {
transactionManager.abort(state);
} catch (IOException e) {
- throw new RuntimeException(e); // FIXME, should be an XAException?
+ XAException xae = new XAException(XAException.XAER_RMERR);
+ xae.initCause(e);
+ throw xae;
}
}
}
@@ -108,9 +112,13 @@
try {
status = this.transactionManager.prepareCommit(state);
} catch (CommitUnsuccessfulException e) {
- throw new XAException(XAException.XA_HEURRB); // FIXME correct code?
+ XAException xae = new XAException(XAException.XA_HEURRB);
+ xae.initCause(e);
+ throw xae;
} catch (IOException e) {
- throw new XAException(XAException.XA_RBPROTO); // FIXME correct code?
+ XAException xae = new XAException(XAException.XAER_RMERR);
+ xae.initCause(e);
+ throw xae;
}
switch (status) {
@@ -119,7 +127,7 @@
case TransactionalRegionInterface.COMMIT_OK_READ_ONLY:
return XAResource.XA_RDONLY;
default:
- throw new XAException(XAException.XA_RBPROTO); // FIXME correct code?
+ throw new XAException(XAException.XA_RBPROTO);
}
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionManager.java Fri Jul 24 05:20:59 2009
@@ -36,6 +36,10 @@
*
*/
public class TransactionManager {
+ static {
+ TransactionalRPC.initialize();
+ }
+
static final Log LOG = LogFactory.getLog(TransactionManager.class);
private final HConnection connection;
@@ -123,7 +127,6 @@
LOG.debug("Commit of transaction [" + transactionState.getTransactionId()
+ "] was unsucsessful", e);
// This happens on a NSRE that is triggered by a split
- // FIXME, but then abort fails
try {
abort(transactionState);
} catch (Exception abortException) {
@@ -177,7 +180,6 @@
LOG.debug("Commit of transaction [" + transactionState.getTransactionId()
+ "] was unsucsessful", e);
// This happens on a NSRE that is triggered by a split
- // FIXME, but then abort fails
try {
abort(transactionState);
} catch (Exception abortException) {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionalTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionalTable.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionalTable.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/client/transactional/TransactionalTable.java Fri Jul 24 05:20:59 2009
@@ -41,10 +41,9 @@
*
*/
public class TransactionalTable extends HTable {
-
- private static final byte RPC_CODE = 100;
+
static {
- HBaseRPC.addToMap(TransactionalRegionInterface.class, RPC_CODE);
+ TransactionalRPC.initialize();
}
/**
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexMaintenanceUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexMaintenanceUtils.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexMaintenanceUtils.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexMaintenanceUtils.java Fri Jul 24 05:20:59 2009
@@ -28,7 +28,6 @@
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.tableindexed.IndexSpecification;
import org.apache.hadoop.hbase.client.tableindexed.IndexedTable;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.util.Bytes;
/**
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/tableindexed/IndexedRegion.java Fri Jul 24 05:20:59 2009
@@ -31,7 +31,6 @@
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -49,7 +48,6 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.tableindexed.IndexSpecification;
import org.apache.hadoop.hbase.client.tableindexed.IndexedTableDescriptor;
-import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.regionserver.FlushRequester;
import org.apache.hadoop.hbase.regionserver.HLog;
import org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegion;
@@ -148,24 +146,6 @@
}
}
- private void updateIndexes(Delete delete) {
- // FIXME
- // Handle delete batch updates. Go back and get the next older values
-// for (BatchOperation op : batchUpdate) {
-// if (!op.isPut()) {
-// Cell current = oldColumnCells.get(op.getColumn());
-// if (current != null) {
-// // TODO: Fix this profligacy!!! St.Ack
-// Cell [] older = Cell.createSingleCellArray(super.get(batchUpdate.getRow(),
-// op.getColumn(), current.getTimestamp(), 1));
-// if (older != null && older.length > 0) {
-// newColumnValues.put(op.getColumn(), older[0].getValue());
-// }
-// }
-// }
-// }
- }
-
/** Return the columns needed for the update. */
private NavigableSet<byte[]> getColumnsForIndexes(Collection<IndexSpecification> indexes) {
NavigableSet<byte[]> neededColumns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/THLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/THLog.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/THLog.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/THLog.java Fri Jul 24 05:20:59 2009
@@ -70,7 +70,7 @@
public void writeDeleteToLog(HRegionInfo regionInfo, final long transactionId, final Delete delete)
throws IOException {
- // FIXME
+ this.append(regionInfo, delete, transactionId);
}
public void writeCommitToLog(HRegionInfo regionInfo, final long transactionId) throws IOException {
@@ -112,7 +112,7 @@
public void append(HRegionInfo regionInfo, Put update, long transactionId)
throws IOException {
- long commitTime = System.currentTimeMillis(); // FIXME ?
+ long commitTime = System.currentTimeMillis();
THLogKey key = new THLogKey(regionInfo.getRegionName(), regionInfo
.getTableDesc().getName(), -1, commitTime, THLogKey.TrxOp.OP,
@@ -123,6 +123,30 @@
}
}
+ /**
+ * Write a transactional delete to the log.
+ *
+ * @param regionInfo
+ * @param now
+ * @param update
+ * @param transactionId
+ * @throws IOException
+ */
+ public void append(HRegionInfo regionInfo, Delete delete, long transactionId)
+ throws IOException {
+
+ long commitTime = System.currentTimeMillis();
+
+ THLogKey key = new THLogKey(regionInfo.getRegionName(), regionInfo
+ .getTableDesc().getName(), -1, commitTime, THLogKey.TrxOp.OP,
+ transactionId);
+
+ for (KeyValue value : convertToKeyValues(delete)) {
+ super.append(regionInfo, key, value);
+ }
+ }
+
+
private List<KeyValue> convertToKeyValues(Put update) {
List<KeyValue> edits = new ArrayList<KeyValue>();
@@ -133,4 +157,15 @@
}
return edits;
}
+
+ private List<KeyValue> convertToKeyValues(Delete delete) {
+ List<KeyValue> edits = new ArrayList<KeyValue>();
+
+ for (List<KeyValue> kvs : delete.getFamilyMap().values()) {
+ for (KeyValue kv : kvs) {
+ edits.add(kv);
+ }
+ }
+ return edits;
+ }
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionState.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionState.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionState.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionState.java Fri Jul 24 05:20:59 2009
@@ -19,7 +19,9 @@
*/
package org.apache.hadoop.hbase.regionserver.transactional;
+import java.io.IOException;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.NavigableSet;
@@ -30,6 +32,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
@@ -37,10 +40,14 @@
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.util.Bytes;
/**
- * Holds the state of a transaction.
+ * Holds the state of a transaction. This includes a buffer of all writes, a
+ * record of all reads / scans, and information about which other transactions
+ * we need to check against.
*/
class TransactionState {
@@ -70,8 +77,8 @@
protected byte[] endRow;
public ScanRange(byte[] startRow, byte[] endRow) {
- this.startRow = startRow;
- this.endRow = endRow;
+ this.startRow = startRow == HConstants.EMPTY_START_ROW ? null : startRow;
+ this.endRow = endRow == HConstants.EMPTY_END_ROW ? null : endRow;
}
/**
@@ -104,8 +111,9 @@
private Status status;
private SortedSet<byte[]> readSet = new TreeSet<byte[]>(
Bytes.BYTES_COMPARATOR);
- private List<Put> writeSet = new LinkedList<Put>();
- private List<ScanRange> scanSet = new LinkedList<ScanRange>();
+ private List<Put> puts = new LinkedList<Put>();
+ private List<ScanRange> scans = new LinkedList<ScanRange>();
+ private List<Delete> deletes = new LinkedList<Delete>();
private Set<TransactionState> transactionsToCheck = new HashSet<TransactionState>();
private int startSequenceNumber;
private Integer sequenceNumber;
@@ -128,15 +136,19 @@
}
void addWrite(final Put write) {
- writeSet.add(write);
+ puts.add(write);
+ }
+
+ boolean hasWrite() {
+ return puts.size() > 0 || deletes.size() > 0;
}
- List<Put> getWriteSet() {
- return writeSet;
+ List<Put> getPuts() {
+ return puts;
}
void addDelete(final Delete delete) {
- throw new UnsupportedOperationException("NYI");
+ deletes.add(delete);
}
/**
@@ -149,9 +161,11 @@
*/
Result localGet(Get get) {
+ // TODO take deletes into account as well
+
List<KeyValue> localKVs = new LinkedList<KeyValue>();
- for (Put put : writeSet) {
+ for (Put put : puts) {
if (!Bytes.equals(get.getRow(), put.getRow())) {
continue;
}
@@ -203,7 +217,7 @@
return false; // Cannot conflict with aborted transactions
}
- for (Put otherUpdate : checkAgainst.getWriteSet()) {
+ for (Put otherUpdate : checkAgainst.getPuts()) {
if (this.getReadSet().contains(otherUpdate.getRow())) {
LOG.debug("Transaction [" + this.toString()
+ "] has read which conflicts with [" + checkAgainst.toString()
@@ -211,7 +225,7 @@
+ Bytes.toString(otherUpdate.getRow()) + "]");
return true;
}
- for (ScanRange scanRange : this.scanSet) {
+ for (ScanRange scanRange : this.scans) {
if (scanRange.contains(otherUpdate.getRow())) {
LOG.debug("Transaction [" + this.toString()
+ "] has scan which conflicts with [" + checkAgainst.toString()
@@ -289,9 +303,9 @@
result.append(" read Size: ");
result.append(readSet.size());
result.append(" scan Size: ");
- result.append(scanSet.size());
+ result.append(scans.size());
result.append(" write Size: ");
- result.append(writeSet.size());
+ result.append(puts.size());
result.append(" startSQ: ");
result.append(startSequenceNumber);
if (sequenceNumber != null) {
@@ -328,7 +342,7 @@
scanRange.startRow == null ? "null" : Bytes
.toString(scanRange.startRow), scanRange.endRow == null ? "null"
: Bytes.toString(scanRange.endRow)));
- scanSet.add(scanRange);
+ scans.add(scanRange);
}
int getCommitPendingWaits() {
@@ -338,4 +352,106 @@
void incrementCommitPendingWaits() {
this.commitPendingWaits++;
}
+
+ /** Get deleteSet.
+ * @return deleteSet
+ */
+ List<Delete> getDeleteSet() {
+ return deletes;
+ }
+
+ /** Set deleteSet.
+ * @param deleteSet the deleteSet to set
+ */
+ void setDeleteSet(List<Delete> deleteSet) {
+ this.deletes = deleteSet;
+ }
+
+ /** Get a scanner to go through the puts from this transaction. Used to weave together the local trx puts with the global state.
+ *
+ * @return scanner
+ */
+ KeyValueScanner getScanner() {
+ return new PutScanner();
+ }
+
+ /** Scanner of the puts that occur during this transaction.
+ *
+ * @author clint.morgan
+ *
+ */
+ private class PutScanner implements KeyValueScanner, InternalScanner {
+
+ private NavigableSet<KeyValue> kvSet;
+ private Iterator<KeyValue> iterator;
+ private boolean didHasNext = false;
+ private KeyValue next = null;
+
+
+ PutScanner() {
+ kvSet = new TreeSet<KeyValue>(KeyValue.COMPARATOR);
+ for (Put put : puts) {
+ for (List<KeyValue> putKVs : put.getFamilyMap().values()) {
+ kvSet.addAll(putKVs);
+ }
+ }
+ iterator = kvSet.iterator();
+ }
+
+ public void close() {
+ // Nothing to close
+ }
+
+ public KeyValue next() {
+ getNext();
+ didHasNext = false;
+ return next;
+ }
+
+ public KeyValue peek() {
+ getNext();
+ return next;
+ }
+
+ public boolean seek(KeyValue key) {
+ iterator = kvSet.headSet(key).iterator();
+
+ getNext();
+ return next != null;
+ }
+
+ private KeyValue getNext() {
+ if (didHasNext) {
+ return next;
+ }
+ didHasNext = true;
+ if (iterator.hasNext()) {
+ next = iterator.next(); }
+ else {
+ next= null;
+ }
+ return next;
+ }
+
+ public boolean next(List<KeyValue> results) throws IOException {
+ KeyValue peek = this.peek();
+ if (peek == null) {
+ return false;
+ }
+ byte [] row = peek.getRow();
+ results.add(peek);
+ while (true){
+ if (this.peek() == null) {
+ break;
+ }
+ if (!Bytes.equals(row, this.peek().getRow())) {
+ break;
+ }
+ results.add(this.next());
+ }
+ return true;
+
+ }
+
+ }
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegion.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegion.java Fri Jul 24 05:20:59 2009
@@ -57,6 +57,7 @@
import org.apache.hadoop.hbase.regionserver.HLog;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.transactional.TransactionState.Status;
import org.apache.hadoop.util.Progressable;
@@ -201,13 +202,12 @@
TransactionState state = new TransactionState(transactionId, super.getLog()
.getSequenceNumber(), super.getRegionInfo());
- // Order is important here ...
- List<TransactionState> commitPendingCopy = new LinkedList<TransactionState>(
+ state.setStartSequenceNumber(nextSequenceId.get());
+ List<TransactionState> commitPendingCopy = new ArrayList<TransactionState>(
commitPendingTransactions);
for (TransactionState commitPending : commitPendingCopy) {
state.addTransactionToCheck(commitPending);
}
- state.setStartSequenceNumber(nextSequenceId.get());
synchronized (transactionsById) {
transactionsById.put(key, state);
@@ -271,7 +271,9 @@
TransactionState state = getTransactionState(transactionId);
state.addScan(scan);
- return new ScannerWrapper(transactionId, super.getScanner(scan));
+ List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>(1);
+ scanners.add(state.getScanner());
+ return super.getScanner(scan, scanners);
}
/**
@@ -310,7 +312,6 @@
/**
* Add a delete to the transaction. Does not get applied until commit process.
- * FIXME, not sure about this approach
*
* @param transactionId
* @param delete
@@ -350,7 +351,7 @@
+ ". Voting for commit");
// If there are writes we must keep record off the transaction
- if (state.getWriteSet().size() > 0) {
+ if (state.hasWrite()) {
// Order is important
state.setStatus(Status.COMMIT_PENDING);
commitPendingTransactions.add(state);
@@ -403,20 +404,19 @@
* @throws IOException
*/
public void commit(final long transactionId) throws IOException {
- // Not checking closing...
TransactionState state;
try {
state = getTransactionState(transactionId);
} catch (UnknownTransactionException e) {
LOG.fatal("Asked to commit unknown transaction: " + transactionId
+ " in region " + super.getRegionInfo().getRegionNameAsString());
- // FIXME Write to the transaction log that this transaction was corrupted
+ // TODO. Anything to handle here?
throw e;
}
if (!state.getStatus().equals(Status.COMMIT_PENDING)) {
LOG.fatal("Asked to commit a non pending transaction");
- // FIXME Write to the transaction log that this transaction was corrupted
+ // TODO. Anything to handle here?
throw new IOException("commit failure");
}
@@ -461,17 +461,21 @@
this.hlog.writeCommitToLog(super.getRegionInfo(), state.getTransactionId());
- for (Put update : state.getWriteSet()) {
+ for (Put update : state.getPuts()) {
this.put(update, false); // Don't need to WAL these
- // FIME, maybe should be walled so we don't need to look so far back.
+ }
+
+ for (Delete delete : state.getDeleteSet()) {
+ this.delete(delete, null, false);
}
state.setStatus(Status.COMMITED);
- if (state.getWriteSet().size() > 0
+ if (state.hasWrite()
&& !commitPendingTransactions.remove(state)) {
LOG
.fatal("Commiting a non-query transaction that is not in commitPendingTransactions");
- throw new IOException("commit failure"); // FIXME, how to handle?
+ // Something has gone really wrong.
+ throw new IOException("commit failure");
}
retireTransaction(state);
}
@@ -480,11 +484,11 @@
public List<StoreFile> close(boolean abort) throws IOException {
prepareToClose();
if (!commitPendingTransactions.isEmpty()) {
- // FIXME, better way to handle?
LOG.warn("Closing transactional region ["
+ getRegionInfo().getRegionNameAsString() + "], but still have ["
+ commitPendingTransactions.size()
- + "] transactions that are pending commit");
+ + "] transactions that are pending commit.");
+ // TODO resolve from the Global Trx Log.
}
return super.close(abort);
}
@@ -495,7 +499,7 @@
}
boolean closing = false;
-
+ private static final int CLOSE_WAIT_ON_COMMIT_PENDING = 1000;
/**
* Get ready to close.
*
@@ -511,10 +515,10 @@
+ commitPendingTransactions.size()
+ "] transactions that are pending commit. Sleeping");
for (TransactionState s : commitPendingTransactions) {
- LOG.info(s.toString());
+ LOG.info("commit pending: " + s.toString());
}
try {
- Thread.sleep(200);
+ Thread.sleep(CLOSE_WAIT_ON_COMMIT_PENDING);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -700,33 +704,4 @@
}
}
}
-
- /** Wrapper which keeps track of rows returned by scanner. */
- private class ScannerWrapper implements InternalScanner {
- private long transactionId;
- private InternalScanner scanner;
-
- /**
- * @param transactionId
- * @param scanner
- * @throws UnknownTransactionException
- */
- public ScannerWrapper(final long transactionId,
- final InternalScanner scanner) throws UnknownTransactionException {
-
- this.transactionId = transactionId;
- this.scanner = scanner;
- }
-
- public void close() throws IOException {
- scanner.close();
- }
-
- public boolean next(List<KeyValue> results) throws IOException {
- boolean result = scanner.next(results);
- TransactionState state = getTransactionState(transactionId);
- // FIXME need to weave in new stuff from this transaction too.
- return result;
- }
- }
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegionServer.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegionServer.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/java/org/apache/hadoop/hbase/regionserver/transactional/TransactionalRegionServer.java Fri Jul 24 05:20:59 2009
@@ -36,6 +36,7 @@
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.transactional.TransactionalRPC;
import org.apache.hadoop.hbase.ipc.HBaseRPCProtocolVersion;
import org.apache.hadoop.hbase.ipc.TransactionalRegionInterface;
import org.apache.hadoop.hbase.regionserver.HLog;
@@ -53,6 +54,11 @@
*/
public class TransactionalRegionServer extends HRegionServer implements
TransactionalRegionInterface {
+
+ static {
+ TransactionalRPC.initialize();
+ }
+
private static final String LEASE_TIME = "hbase.transaction.leasetime";
private static final int DEFAULT_LEASE_TIME = 60 * 1000;
private static final int LEASE_CHECK_FREQUENCY = 1000;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/tableindexed/TestIndexedTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/tableindexed/TestIndexedTable.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/tableindexed/TestIndexedTable.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/tableindexed/TestIndexedTable.java Fri Jul 24 05:20:59 2009
@@ -73,8 +73,7 @@
IndexedTableDescriptor indexDesc = new IndexedTableDescriptor(desc);
// Create a new index that does lexicographic ordering on COL_A
- IndexSpecification colAIndex = new IndexSpecification(INDEX_COL_A,
- COL_A);
+ IndexSpecification colAIndex = new IndexSpecification(INDEX_COL_A, COL_A);
indexDesc.addIndex(colAIndex);
admin = new IndexedTableAdmin(conf);
@@ -98,10 +97,11 @@
writeInitalRows();
assertRowsInOrder(NUM_ROWS);
}
-
- private void assertRowsInOrder(int numRowsExpected) throws IndexNotFoundException, IOException {
- ResultScanner scanner = table.getIndexedScanner(INDEX_COL_A,
- HConstants.EMPTY_START_ROW, null, null, null);
+
+ private void assertRowsInOrder(int numRowsExpected)
+ throws IndexNotFoundException, IOException {
+ ResultScanner scanner = table.getIndexedScanner(INDEX_COL_A, null, null,
+ null, null, null);
int numRows = 0;
byte[] lastColA = null;
for (Result rowResult : scanner) {
@@ -115,7 +115,7 @@
numRows++;
}
scanner.close();
- Assert.assertEquals(numRowsExpected, numRows);
+ Assert.assertEquals(numRowsExpected, numRows);
}
public void testMultipleWrites() throws IOException {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/transactional/TestTransactions.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/transactional/TestTransactions.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/transactional/TestTransactions.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/contrib/transactional/src/test/org/apache/hadoop/hbase/client/transactional/TestTransactions.java Fri Jul 24 05:20:59 2009
@@ -21,6 +21,8 @@
import java.io.IOException;
+import junit.framework.Assert;
+
import org.apache.hadoop.hbase.HBaseClusterTestCase;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -29,6 +31,8 @@
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.ipc.TransactionalRegionInterface;
import org.apache.hadoop.hbase.regionserver.transactional.TransactionalRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
@@ -95,7 +99,7 @@
transactionManager.tryCommit(transactionState2);
}
- public void TestTwoTransactionsWithConflict() throws IOException,
+ public void testTwoTransactionsWithConflict() throws IOException,
CommitUnsuccessfulException {
TransactionState transactionState1 = makeTransaction1();
TransactionState transactionState2 = makeTransaction2();
@@ -110,14 +114,73 @@
}
}
+ public void testGetAfterPut() throws IOException {
+ TransactionState transactionState = transactionManager.beginTransaction();
+
+ int originalValue = Bytes.toInt(table.get(transactionState,
+ new Get(ROW1).addColumn(COL_A)).value());
+ int newValue = originalValue + 1;
+
+ table.put(transactionState, new Put(ROW1).add(FAMILY, QUAL_A, Bytes
+ .toBytes(newValue)));
+
+ Result row1_A = table.get(transactionState, new Get(ROW1).addColumn(COL_A));
+ Assert.assertEquals(newValue, Bytes.toInt(row1_A.value()));
+ }
+
+ public void testScanAfterUpdatePut() throws IOException {
+ TransactionState transactionState = transactionManager.beginTransaction();
+
+ int originalValue = Bytes.toInt(table.get(transactionState,
+ new Get(ROW1).addColumn(COL_A)).value());
+ int newValue = originalValue + 1;
+ table.put(transactionState, new Put(ROW1).add(FAMILY, QUAL_A, Bytes
+ .toBytes(newValue)));
+
+ ResultScanner scanner = table.getScanner(transactionState, new Scan()
+ .addFamily(FAMILY));
+
+ Result result = scanner.next();
+ Assert.assertNotNull(result);
+
+ Assert.assertEquals(Bytes.toString(ROW1), Bytes.toString(result.getRow()));
+ Assert.assertEquals(newValue, Bytes.toInt(result.value()));
+
+ result = scanner.next();
+ Assert.assertNull(result);
+
+ }
+
+ public void testScanAfterNewPut() throws IOException {
+ TransactionState transactionState = transactionManager.beginTransaction();
+
+ int row2Value = 199;
+ table.put(transactionState, new Put(ROW2).add(FAMILY, QUAL_A, Bytes
+ .toBytes(row2Value)));
+
+ ResultScanner scanner = table.getScanner(transactionState, new Scan()
+ .addFamily(FAMILY));
+
+ Result result = scanner.next();
+ Assert.assertNotNull(result);
+ Assert.assertEquals(Bytes.toString(ROW1), Bytes.toString(result.getRow()));
+
+ result = scanner.next();
+ Assert.assertNotNull(result);
+ Assert.assertEquals(Bytes.toString(ROW2), Bytes.toString(result.getRow()));
+ Assert.assertEquals(row2Value, Bytes.toInt(result.value()));
+ }
+
// Read from ROW1,COL_A and put it in ROW2_COLA and ROW3_COLA
private TransactionState makeTransaction1() throws IOException {
TransactionState transactionState = transactionManager.beginTransaction();
Result row1_A = table.get(transactionState, new Get(ROW1).addColumn(COL_A));
- table.put(new Put(ROW2).add(FAMILY, QUAL_A, row1_A.getValue(COL_A)));
- table.put(new Put(ROW3).add(FAMILY, QUAL_A, row1_A.getValue(COL_A)));
+ table.put(transactionState, new Put(ROW2).add(FAMILY, QUAL_A, row1_A
+ .getValue(COL_A)));
+ table.put(transactionState, new Put(ROW3).add(FAMILY, QUAL_A, row1_A
+ .getValue(COL_A)));
return transactionState;
}
@@ -130,7 +193,8 @@
int value = Bytes.toInt(row1_A.getValue(COL_A));
- table.put(new Put(ROW1).add(FAMILY, QUAL_A, Bytes.toBytes(value + 1)));
+ table.put(transactionState, new Put(ROW1).add(FAMILY, QUAL_A, Bytes
+ .toBytes(value + 1)));
return transactionState;
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java Fri Jul 24 05:20:59 2009
@@ -381,6 +381,13 @@
}
/**
+ * @param key Key whose key and value we're to remove from HCD parameters.
+ */
+ public void remove(final byte [] key) {
+ values.remove(new ImmutableBytesWritable(key));
+ }
+
+ /**
* @param key The key.
* @param value The value.
*/
@@ -391,7 +398,8 @@
/** @return compression type being used for the column family */
@TOJSON
public Compression.Algorithm getCompression() {
- return Compression.Algorithm.valueOf(getValue(COMPRESSION));
+ String n = getValue(COMPRESSION);
+ return Compression.Algorithm.valueOf(n.toUpperCase());
}
/** @return maximum number of versions */
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerAddress.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerAddress.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerAddress.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerAddress.java Fri Jul 24 05:20:59 2009
@@ -163,7 +163,7 @@
}
public void write(DataOutput out) throws IOException {
- if(address == null) {
+ if (address == null) {
out.writeUTF("");
out.writeInt(0);
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerInfo.java Fri Jul 24 05:20:59 2009
@@ -38,7 +38,7 @@
private long startCode;
private HServerLoad load;
private int infoPort;
- private transient volatile String serverName = null;
+ private String serverName = null;
private String name;
/** default constructor - used by Writable */
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java Fri Jul 24 05:20:59 2009
@@ -348,14 +348,14 @@
}
/**
- * @returns the amount of heap in use, in MB
+ * @return the amount of heap in use, in MB
*/
public int getUsedHeapMB() {
return usedHeapMB;
}
/**
- * @returns the maximum allowable heap size, in MB
+ * @return the maximum allowable heap size, in MB
*/
public int getMaxHeapMB() {
return maxHeapMB;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java Fri Jul 24 05:20:59 2009
@@ -351,6 +351,13 @@
}
/**
+ * @param key Key whose key and value we're to remove from HTD parameters.
+ */
+ public void remove(final byte [] key) {
+ values.remove(new ImmutableBytesWritable(key));
+ }
+
+ /**
* @return true if all columns in the table should be read only
*/
public boolean isReadOnly() {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java Fri Jul 24 05:20:59 2009
@@ -33,8 +33,15 @@
import org.apache.hadoop.io.Writable;
/**
- * An HBase Key/Value. Instances of this class are immutable. They are not
- * comparable but Comparators are provided Comparators change with context,
+ * An HBase Key/Value.
+ *
+ * <p>If being used client-side, the primary methods to access individual fields
+ * are {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()},
+ * {@link #getTimestamp()}, and {@link #getValue()}. These methods allocate new
+ * byte arrays and return copies so they should be avoided server-side.
+ *
+ * <p>Instances of this class are immutable. They are not
+ * comparable but Comparators are provided. Comparators change with context,
* whether user table or a catalog table comparison context. Its
* important that you use the appropriate comparator comparing rows in
* particular. There are Comparators for KeyValue instances and then for
@@ -613,6 +620,9 @@
//---------------------------------------------------------------------------
public String toString() {
+ if (this.bytes == null || this.bytes.length == 0) {
+ return "empty";
+ }
return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) +
"/vlen=" + getValueLength();
}
@@ -860,7 +870,11 @@
//---------------------------------------------------------------------------
/**
- * @return Copy of the key portion only. Used compacting and testing.
+ * Do not use unless you have to. Used internally for compacting and testing.
+ *
+ * Use {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, and
+ * {@link #getValue()} if accessing a KeyValue client-side.
+ * @return Copy of the key portion only.
*/
public byte [] getKey() {
int keylength = getKeyLength();
@@ -883,8 +897,11 @@
}
/**
- * Do not use this unless you have to.
- * Use {@link #getBuffer()} with appropriate offsets and lengths instead.
+ * Primarily for use client-side. Returns the row of this KeyValue in a new
+ * byte array.<p>
+ *
+ * If server-side, use {@link #getBuffer()} with appropriate offsets and
+ * lengths instead.
* @return Row in a new byte array.
*/
public byte [] getRow() {
@@ -896,6 +913,7 @@
}
/**
+ *
* @return Timestamp
*/
public long getTimestamp() {
@@ -941,8 +959,11 @@
}
/**
- * Do not use this unless you have to.
- * Use {@link #getBuffer()} with appropriate offsets and lengths instead.
+ * Primarily for use client-side. Returns the column of this KeyValue in the
+ * deprecated format: <i>family:qualifier</i>, and in a new byte array.<p>
+ *
+ * If server-side, use {@link #getBuffer()} with appropriate offsets and
+ * lengths instead.
* @return Returns column. Makes a copy. Inserts delimiter.
*/
public byte [] getColumn() {
@@ -957,8 +978,11 @@
}
/**
- * Do not use this unless you have to.
- * Use {@link #getBuffer()} with appropriate offsets and lengths instead.
+ * Primarily for use client-side. Returns the family of this KeyValue in a
+ * new byte array.<p>
+ *
+ * If server-side, use {@link #getBuffer()} with appropriate offsets and
+ * lengths instead.
* @return Returns family. Makes a copy.
*/
public byte [] getFamily() {
@@ -970,7 +994,11 @@
}
/**
- * Do not use this unless you have to.
+ * Primarily for use client-side. Returns the column qualifier of this
+ * KeyValue in a new byte array.<p>
+ *
+ * If server-side, use {@link #getBuffer()} with appropriate offsets and
+ * lengths instead.
* Use {@link #getBuffer()} with appropriate offsets and lengths instead.
* @return Returns qualifier. Makes a copy.
*/
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Fri Jul 24 05:20:59 2009
@@ -195,10 +195,12 @@
while (regionServerThread.isAlive()) {
try {
LOG.info("Waiting on " +
- regionServerThread.getRegionServer().getServerInfo().toString());
+ regionServerThread.getRegionServer().getHServerInfo().toString());
regionServerThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
}
}
return regionServerThread.getName();
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java Fri Jul 24 05:20:59 2009
@@ -38,8 +38,29 @@
import org.apache.hadoop.io.Writable;
/**
- * Single row result of a {@link Get} or {@link Scan} query.
- * Backed by array of KeyValues.
+ * Single row result of a {@link Get} or {@link Scan} query.<p>
+ *
+ * Convenience methods are available that return various {@link Map}
+ * structures and values directly.<p>
+ *
+ * To get a complete mapping of all cells in the Result, which can include
+ * multiple families and multiple versions, use {@link #getMap()}.<p>
+ *
+ * To get a mapping of each family to its columns (qualifiers and values),
+ * including only the latest version of each, use {@link #getNoVersionMap()}.
+ *
+ * To get a mapping of qualifiers to latest values for an individual family use
+ * {@link #getFamilyMap(byte[])}.<p>
+ *
+ * To get the latest value for a specific family and qualifier use {@link #getValue(byte[], byte[])}.
+ *
+ * A Result is backed by an array of {@link KeyValue} objects, each representing
+ * an HBase cell defined by the row, family, qualifier, timestamp, and value.<p>
+ *
+ * The underlying {@link KeyValue} objects can be accessed through the methods
+ * {@link #sorted()} and {@link #list()}. Each KeyValue can then be accessed
+ * through {@link KeyValue#getRow()}, {@link KeyValue#getFamily()}, {@link KeyValue#getQualifier()},
+ * {@link KeyValue#getTimestamp()}, and {@link KeyValue#getValue()}.
*/
public class Result implements Writable {
private KeyValue [] kvs = null;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnection.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnection.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnection.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnection.java Fri Jul 24 05:20:59 2009
@@ -27,7 +27,6 @@
* cluster to be up to get a connection.
*/
public interface ServerConnection extends HConnection {
-
/**
* Set root region location in connection
* @param rootRegion
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ServerConnectionManager.java Fri Jul 24 05:20:59 2009
@@ -24,7 +24,7 @@
/**
* Used by server processes to expose HServerConnection method
- * do can call {@link HConnectionManager#setRootRegionLocation}
+ * do can call HConnectionManager#setRootRegionLocation
*/
public class ServerConnectionManager extends HConnectionManager {
/*
@@ -41,4 +41,4 @@
public static ServerConnection getConnection(HBaseConfiguration conf) {
return (ServerConnection) HConnectionManager.getConnection(conf);
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java Fri Jul 24 05:20:59 2009
@@ -27,7 +27,29 @@
</ul>
<h2><a name="overview">Overview</a></h2>
- <p>
+ <p>To administer HBase, create and drop tables, list and alter tables,
+ use {@link org.apache.hadoop.hbase.client.HBaseAdmin}. Once created, table access is via an instance
+ of {@link org.apache.hadoop.hbase.client.HTable}. You add content to a table a row at a time. To insert,
+ create an instance of a {@link org.apache.hadoop.hbase.client.Put} object and after setting it appropriately,
+ commit your update using {@link org.apache.hadoop.hbase.client.HTable#put(Put)}.
+ To fetch your inserted
+ value, use {@link org.apache.hadoop.hbase.client.Get}. The Get can be specified to be broad -- get all
+ on a particular row -- or narrow; return only a single cell value.
+ When finished with your Get settings, invoke {@link org.apache.hadoop.hbase.client.HTable#get(Get)}. Use
+ {@link org.apache.hadoop.hbase.client.Scan} to set up a scanner -- a Cursor-like access. After
+ configuring your Scan instance, call {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} and then
+ invoke next on the returned object. Both {@link org.apache.hadoop.hbase.client.HTable#get(Get)} and
+ {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} return a
+{@link org.apache.hadoop.hbase.client.Result}.
+A Result is a List of {@link org.apache.hadoop.hbase.KeyValue}s. It has facility for packaging the return
+in different formats.
+ Use {@link org.apache.hadoop.hbase.client.Delete} to remove content.
+ You can remove individual cells or entire families, etc. Pass it to
+ {@link org.apache.hadoop.hbase.client.HTable#delete(Delete)} to execute.
+ </p>
+ <p>Client code accessing a cluster finds the cluster by querying ZooKeeper.
+ This means that the ZooKeeper quorum to use must be on the client CLASSPATH.
+ Usually this means make sure the client can find your <code>hbase-site.xml</code>.
</p>
<h2><a name="client_example">Example API Usage</a></h2>
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/FilterList.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/FilterList.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/FilterList.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/FilterList.java Fri Jul 24 05:20:59 2009
@@ -29,6 +29,7 @@
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.io.Writable;
/**
* Implementation of {@link Filter} that represents an ordered List of Filters
@@ -211,7 +212,7 @@
out.writeByte(operator.ordinal());
out.writeInt(filters.size());
for (Filter filter : filters) {
- HbaseObjectWritable.writeObject(out, filter, filter.getClass(), conf);
+ HbaseObjectWritable.writeObject(out, filter, Writable.class, conf);
}
}
}
\ No newline at end of file
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/HMaster.java Fri Jul 24 05:20:59 2009
@@ -719,7 +719,7 @@
public HMsg[] regionServerReport(HServerInfo serverInfo, HMsg msgs[],
HRegionInfo[] mostLoadedRegions)
throws IOException {
- return serverManager.regionServerReport(serverInfo, msgs,
+ return serverManager.regionServerReport(serverInfo, msgs,
mostLoadedRegions);
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java Fri Jul 24 05:20:59 2009
@@ -1369,8 +1369,8 @@
int numSrvRegs = srvLoad.getNumberOfRegions();
if (numSrvRegs > avgLoadPlusSlop) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Server is overloaded. Server load: " + numSrvRegs +
- " avg: " + avgLoad + ", slop: " + this.slop);
+ LOG.debug("Server is overloaded: load=" + numSrvRegs +
+ ", avg=" + avgLoad + ", slop=" + this.slop);
}
return numSrvRegs - (int)Math.ceil(avgLoad);
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java Fri Jul 24 05:20:59 2009
@@ -183,10 +183,9 @@
if (storedInfo != null && !master.closed.get()) {
// The startup message was from a known server with the same name.
// Timeout the old one right away.
- HServerAddress root = master.getRootRegionLocation();
+ master.getRootRegionLocation();
try {
- master.toDoQueue.put(
- new ProcessServerShutdown(master, storedInfo));
+ master.toDoQueue.put(new ProcessServerShutdown(master, storedInfo));
} catch (InterruptedException e) {
LOG.error("Insertion into toDoQueue was interrupted", e);
}
@@ -261,7 +260,7 @@
}
if (master.shutdownRequested.get()) {
- if(quiescedServers.get() >= serversToServerInfo.size()) {
+ if (quiescedServers.get() >= serversToServerInfo.size()) {
// If the only servers we know about are meta servers, then we can
// proceed with shutdown
LOG.info("All user tables quiesced. Proceeding with shutdown");
@@ -290,13 +289,13 @@
HServerInfo storedInfo = serversToServerInfo.get(info.getServerName());
if (storedInfo == null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("received server report from unknown server: " +
- info.getServerName());
+ LOG.debug("Received report from unknown server -- telling it " +
+ "to " + CALL_SERVER_STARTUP + ": " + info.getServerName());
}
// The HBaseMaster may have been restarted.
// Tell the RegionServer to start over and call regionServerStartup()
- return new HMsg[]{CALL_SERVER_STARTUP};
+ return new HMsg[] {CALL_SERVER_STARTUP};
} else if (storedInfo.getStartCode() != info.getStartCode()) {
// This state is reachable if:
//
@@ -317,7 +316,7 @@
serversToServerInfo.notifyAll();
}
- return new HMsg[]{REGIONSERVER_STOP};
+ return new HMsg[] {REGIONSERVER_STOP};
} else {
return processRegionServerAllsWell(info, mostLoadedRegions, msgs);
}
@@ -372,13 +371,12 @@
* @return
* @throws IOException
*/
- private HMsg[] processRegionServerAllsWell(HServerInfo serverInfo, HRegionInfo[] mostLoadedRegions, HMsg[] msgs)
+ private HMsg[] processRegionServerAllsWell(HServerInfo serverInfo,
+ final HRegionInfo[] mostLoadedRegions, HMsg[] msgs)
throws IOException {
-
// Refresh the info object and the load information
serverAddressToServerInfo.put(serverInfo.getServerAddress(), serverInfo);
serversToServerInfo.put(serverInfo.getServerName(), serverInfo);
-
HServerLoad load = serversToLoad.get(serverInfo.getServerName());
if (load != null) {
this.master.getMetrics().incrementRequests(load.getNumberOfRequests());
@@ -436,8 +434,9 @@
int openingCount = 0;
for (int i = 0; i < incomingMsgs.length; i++) {
HRegionInfo region = incomingMsgs[i].getRegionInfo();
- LOG.info("Received " + incomingMsgs[i] + " from " +
- serverInfo.getServerName() + "; " + (i + 1) + " of " + incomingMsgs.length);
+ LOG.info("Processing " + incomingMsgs[i] + " from " +
+ serverInfo.getServerName() + "; " + (i + 1) + " of " +
+ incomingMsgs.length);
switch (incomingMsgs[i].getType()) {
case MSG_REPORT_PROCESS_OPEN:
openingCount++;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/io/HBaseMapFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/io/HBaseMapFile.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/io/HBaseMapFile.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/io/HBaseMapFile.java Fri Jul 24 05:20:59 2009
@@ -24,8 +24,8 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HStoreKey;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.migration.nineteen.HStoreKey;
import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
@@ -73,10 +73,10 @@
public HBaseReader(FileSystem fs, String dirName, Configuration conf,
boolean blockCacheEnabled, HRegionInfo hri)
throws IOException {
- super(fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri),
+ super(fs, dirName, new org.apache.hadoop.hbase.migration.nineteen.HStoreKey.HStoreKeyWritableComparator(hri),
conf, false); // defer opening streams
this.blockCacheEnabled = blockCacheEnabled;
- open(fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri), conf);
+ open(fs, dirName, new org.apache.hadoop.hbase.migration.nineteen.HStoreKey.HStoreKeyWritableComparator(hri), conf);
// Force reading of the mapfile index by calling midKey. Reading the
// index will bring the index into memory over here on the client and
@@ -85,7 +85,9 @@
// load the index force the issue in HStoreFile MapFiles because an
// access may not happen for some time; meantime we're using up datanode
// resources (See HADOOP-2341). midKey() goes to index. Does not seek.
- midKey();
+
+
+ // Disable for migration !!! midKey();
}
}
@@ -101,7 +103,7 @@
public HBaseWriter(Configuration conf, FileSystem fs, String dirName,
SequenceFile.CompressionType compression, final HRegionInfo hri)
throws IOException {
- super(conf, fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri),
+ super(conf, fs, dirName, new org.apache.hadoop.hbase.migration.nineteen.HStoreKey.HStoreKeyWritableComparator(hri),
VALUE_CLASS, compression);
// Default for mapfiles is 128. Makes random reads faster if we
// have more keys indexed and we're not 'next'-ing around in the
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/BloomFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/BloomFilter.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/BloomFilter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/BloomFilter.java Fri Jul 24 05:20:59 2009
@@ -70,10 +70,6 @@
* contract <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
*
* @version 1.0 - 2 Feb. 07
- *
- * @see org.onelab.filter.Filter The general behavior of a filter
- *
- * @see <a href="http://portal.acm.org/citation.cfm?id=362692&dl=ACM&coll=portal">Space/Time Trade-Offs in Hash Coding with Allowable Errors</a>
*/
public class BloomFilter extends Filter {
private static final byte[] bitvalues = new byte[] {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/CountingBloomFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/CountingBloomFilter.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/CountingBloomFilter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/CountingBloomFilter.java Fri Jul 24 05:20:59 2009
@@ -66,9 +66,6 @@
*
* @version 1.1 - 19 Jan. 08
*
- * @see org.onelab.filter.Filter The general behavior of a filter
- *
- * @see <a href="http://portal.acm.org/citation.cfm?id=343571.343572">Summary cache: a scalable wide-area web cache sharing protocol</a>
*/
public final class CountingBloomFilter extends Filter {
/** Storage for the counting buckets */
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/DynamicBloomFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/DynamicBloomFilter.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/DynamicBloomFilter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/DynamicBloomFilter.java Fri Jul 24 05:20:59 2009
@@ -81,10 +81,6 @@
*
* @version 1.0 - 6 Feb. 07
*
- * @see org.onelab.filter.Filter The general behavior of a filter
- * @see org.onelab.filter.BloomFilter A Bloom filter
- *
- * @see <a href="http://www.cse.fau.edu/~jie/research/publications/Publication_files/infocom2006.pdf">Theory and Network Applications of Dynamic Bloom Filters</a>
*/
public class DynamicBloomFilter extends Filter {
/**
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Filter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Filter.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Filter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Filter.java Fri Jul 24 05:20:59 2009
@@ -69,12 +69,7 @@
* <p>
* It must be extended in order to define the real behavior.
*
- * @see org.onelab.filter.Filter The general behavior of a filter
- *
* @version 1.0 - 2 Feb. 07
- *
- * @see org.onelab.filter.Key The general behavior of a key
- * @see org.onelab.filter.HashFunction A hash function
*/
public abstract class Filter implements Writable {
private static final int VERSION = -1; // negative to accommodate for old format
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/HashFunction.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/HashFunction.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/HashFunction.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/HashFunction.java Fri Jul 24 05:20:59 2009
@@ -56,14 +56,7 @@
* <p>
* It is based on the SHA-1 algorithm.
*
- * @see org.onelab.filter.Filter The general behavior of a filter
- *
* @version 1.0 - 2 Feb. 07
- *
- * @see org.onelab.filter.Key The general behavior of a key being stored in a filter
- * @see org.onelab.filter.Filter The general behavior of a filter
- *
- * @see <a href="http://www.itl.nist.gov/fipspubs/fip180-1.htm">SHA-1 algorithm</a>
*/
public final class HashFunction {
/** The number of hashed values. */
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Key.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Key.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Key.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/Key.java Fri Jul 24 05:20:59 2009
@@ -55,8 +55,6 @@
/**
* The general behavior of a key that must be stored in a filter.
- *
- * @see org.onelab.filter.Filter The general behavior of a filter
*/
public class Key implements WritableComparable {
/** Byte value of key */
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/RetouchedBloomFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/RetouchedBloomFilter.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/RetouchedBloomFilter.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/onelab/filter/RetouchedBloomFilter.java Fri Jul 24 05:20:59 2009
@@ -69,11 +69,6 @@
*
* @version 1.0 - 7 Feb. 07
*
- * @see org.onelab.filter.Filter The general behavior of a filter
- * @see org.onelab.filter.BloomFilter A Bloom filter
- * @see org.onelab.filter.RemoveScheme The different selective clearing algorithms
- *
- * @see <a href="http://www-rp.lip6.fr/site_npa/site_rp/_publications/740-rbf_cameraready.pdf">Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives</a>
*/
public final class RetouchedBloomFilter extends BloomFilter
implements RemoveScheme {
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/regionserver/HStoreFile.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/regionserver/HStoreFile.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/regionserver/HStoreFile.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/nineteen/regionserver/HStoreFile.java Fri Jul 24 05:20:59 2009
@@ -23,6 +23,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,6 +45,9 @@
/**
* A HStore data file. HStores usually have one or more of these files. They
* are produced by flushing the memcache to disk.
+ *
+ * <p>This one has been doctored to be used in migrations. Private and
+ * protecteds have been made public, etc.
*
* <p>Each HStore maintains a bunch of different data files. The filename is a
* mix of the parent dir, the region name, the column name, and a file
@@ -110,14 +115,14 @@
* @param colFamily name of the column family
* @param fileId file identifier
* @param ref Reference to another HStoreFile.
- * @param hri The region info for this file (HACK HBASE-868). TODO: Fix.
+ * @param encodedName Encoded name.
* @throws IOException
*/
- HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir,
- final HRegionInfo hri, byte [] colFamily, long fileId,
+ public HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir,
+ final int encodedName, byte [] colFamily, long fileId,
final Reference ref)
throws IOException {
- this(conf, fs, basedir, hri, colFamily, fileId, ref, false);
+ this(conf, fs, basedir, encodedName, colFamily, fileId, ref, false);
}
/**
@@ -127,20 +132,21 @@
* @param colFamily name of the column family
* @param fileId file identifier
* @param ref Reference to another HStoreFile.
- * @param hri The region info for this file (HACK HBASE-868). TODO: Fix.
+ * @param encodedName Encoded name.
* @param mc Try if this file was result of a major compression.
* @throws IOException
*/
HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir,
- final HRegionInfo hri, byte [] colFamily, long fileId,
+ final int encodedName, byte [] colFamily, long fileId,
final Reference ref, final boolean mc)
throws IOException {
this.conf = conf;
this.fs = fs;
this.basedir = basedir;
- this.encodedRegionName = hri.getEncodedName();
+ this.encodedRegionName = encodedName;
this.colFamily = colFamily;
- this.hri = hri;
+ // NOT PASSED IN MIGRATIONS
+ this.hri = null;
long id = fileId;
if (id == -1) {
@@ -164,6 +170,27 @@
return reference != null;
}
+
+ private static final Pattern REF_NAME_PARSER =
+ Pattern.compile("^(\\d+)(?:\\.(.+))?$");
+
+ /**
+ * @param p Path to check.
+ * @return True if the path has format of a HStoreFile reference.
+ */
+ public static boolean isReference(final Path p) {
+ return isReference(p, REF_NAME_PARSER.matcher(p.getName()));
+ }
+
+ private static boolean isReference(final Path p, final Matcher m) {
+ if (m == null || !m.matches()) {
+ LOG.warn("Failed match of store file name " + p.toString());
+ throw new RuntimeException("Failed match of store file name " +
+ p.toString());
+ }
+ return m.groupCount() > 1 && m.group(2) != null;
+ }
+
Reference getReference() {
return reference;
}
@@ -316,7 +343,7 @@
* @return The sequence id contained in the info file
* @throws IOException
*/
- long loadInfo(final FileSystem filesystem) throws IOException {
+ public long loadInfo(final FileSystem filesystem) throws IOException {
Path p = null;
if (isReference()) {
p = getInfoFilePath(reference.getEncodedRegionName(),
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/package-info.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/package-info.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/migration/package-info.java Fri Jul 24 05:20:59 2009
@@ -1,5 +1,5 @@
/*
- * Copyright 2009The Apache Software Foundation
+ * Copyright 2009 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -19,6 +19,8 @@
*/
/**
Provides classes from old hbase versions used migrating data.
-Nineteen package has classes from hbase 0.19.
+Nineteen package has classes from hbase 0.19. See
+<a href="http://wiki.apache.org/hadoop/Hbase/HowToMigrate">How to Migrate</a>
+for more on migrations.
*/
package org.apache.hadoop.hbase.migration;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=797323&r1=797322&r2=797323&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Fri Jul 24 05:20:59 2009
@@ -1004,6 +1004,7 @@
splits = new ArrayList<Path>(logWriters.size());
for (WriterAndPath wap : logWriters.values()) {
wap.w.close();
+ LOG.debug("Closed " + wap.p);
splits.add(wap.p);
}
}