You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/09/09 19:14:24 UTC
svn commit: r813052 [1/4] - in /hadoop/hbase/branches/0.20_on_hadoop-0.18.3:
./ bin/ conf/ src/java/org/apache/hadoop/hbase/
src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/filter/
src/java/org/apache/hadoop/hbase/io/ src/java...
Author: apurtell
Date: Wed Sep 9 17:14:22 2009
New Revision: 813052
URL: http://svn.apache.org/viewvc?rev=813052&view=rev
Log:
pull up to release
Added:
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/QualifierFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RowFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SkipFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestSingleColumnValueFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
Removed:
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestValueFilter.java
Modified:
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PageFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/ValueFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/RegionManager.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/master/ServerManager.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStore.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/rest/package.html
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/Bytes.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/util/ClassSize.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/TestZooKeeper.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestBatchUpdate.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestFilterList.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestInclusiveStopFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/filter/TestPageFilter.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/io/TestHeapSize.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestCompaction.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHLog.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestMemStore.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestScanWildcardColumnTracker.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java
hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/CHANGES.txt Wed Sep 9 17:14:22 2009
@@ -1,5 +1,7 @@
HBase Change Log
-Release 0.20.0 - Unreleased
+
+Release 0.20.0 - Tue Sep 8 12:48:41 PDT 2009
+
INCOMPATIBLE CHANGES
HBASE-1147 Modify the scripts to use Zookeeper
HBASE-1144 Store the ROOT region location in Zookeeper
@@ -26,6 +28,7 @@
HBASE-1582 Translate ColumnValueFilter and RowFilterSet to the new
Filter interface (Clint Morgan and Stack)
HBASE-1599 Fix TestFilterSet, broken up on hudson (Jon Gray via Stack)
+ HBASE-1799 deprecate o.a.h.h.rest in favor of stargate
BUG FIXES
HBASE-1140 "ant clean test" fails (Nitay Joffe via Stack)
@@ -316,6 +319,34 @@
HBASE-1737 Regions unbalanced when adding new node
HBASE-1745 [tools] Tool to kick region out of inTransistion
HBASE-1757 REST server runs out of fds
+ HBASE-1768 REST server has upper limit of 5k PUT
+ HBASE-1766 Add advanced features to HFile.main() to be able to analyze
+ storefile problems
+ HBASE-1761 getclosest doesn't understand delete family; manifests as
+ "HRegionInfo was null or empty in .META" A.K.A the BS problem
+ HBASE-1738 Scanner doesnt reset when a snapshot is created, could miss
+ new updates into the 'kvset' (active part)
+ HBASE-1767 test zookeeper broken in trunk and 0.20 branch; broken on
+ hudson too
+ HBASE-1791 Timeout in IndexRecordWriter (Bradford Stephens via Andrew
+ Purtell)
+ HBASE-1737 Regions unbalanced when adding new node (recommit)
+ HBASE-1792 [Regression] Cannot save timestamp in the future
+ HBASE-1793 [Regression] HTable.get/getRow with a ts is broken
+ HBASE-1698 Review documentation for o.a.h.h.mapreduce
+ HBASE-1798 [Regression] Unable to delete a row in the future
+ HBASE-1780 HTable.flushCommits clears write buffer in finally clause
+ HBASE-1784 Missing rows after medium intensity insert
+ HBASE-1809 NPE thrown in BoundedRangeFileInputStream
+ HBASE-1810 ConcurrentModificationException in region assignment
+ (Mathias Herberts via Stack)
+ HBASE-1804 Puts are permitted (and stored) when including an appended colon
+ HBASE-1715 Compaction failure in ScanWildcardColumnTracker.checkColumn
+ HBASE-1790 filters are not working correctly (HBASE-1710 HBASE-1807 too)
+ HBASE-1779 ThriftServer logged error if getVer() result is empty
+ HBASE-1778 Improve PerformanceEvaluation (Schubert Zhang via Stack)
+ HBASE-1812 Document fact that Scanners do not respect row locks
+ HBASE-1751 Fix KeyValue javadoc on getValue for client-side
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -549,6 +580,19 @@
region info
HBASE-1743 [debug tool] Add regionsInTransition list to ClusterStatus
detailed output
+ HBASE-1760 Cleanup TODOs in HTable
+ HBASE-1759 Ability to specify scanner caching on a per-scan basis
+ (Ken Weiner via jgray)
+ HBASE-1763 Put writeToWAL methods do not have proper getter/setter names
+ (second commit to fix compile error in hregion)
+ HBASE-1770 HTable.setWriteBufferSize does not flush the writeBuffer when
+ its size is set to a value lower than its current size.
+ (Mathias via jgray)
+ HBASE-1771 PE sequentialWrite is 7x slower because of
+ MemStoreFlusher#checkStoreFileCount
+ HBASE-1772 Up the default ZK session timeout from 30seconds to 60seconds
+ HBASE-1754 Use TCP keepalives
+ HBASE-1800 Too many ZK connections
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/bin/HBase.rb Wed Sep 9 17:14:22 2009
@@ -167,6 +167,7 @@
raise IOError.new("Table " + tableName + " is enabled. Disable it first")
else
@admin.deleteTable(tableName)
+ flush(HConstants::META_TABLE_NAME);
major_compact(HConstants::META_TABLE_NAME);
end
@formatter.footer(now)
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/conf/hbase-default.xml Wed Sep 9 17:14:22 2009
@@ -328,6 +328,26 @@
</description>
</property>
<property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value>7</value>
+ <description>
+ If more than this number of StoreFiles in any one Store
+ (one StoreFile is written per flush of MemStore) then updates are
+ blocked for this HRegion until a compaction is completed, or
+ until hbase.hstore.blockingWaitTime has been exceeded.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hstore.blockingWaitTime</name>
+ <value>90000</value>
+ <description>
+ The time an HRegion will block updates for after hitting the StoreFile
+ limit defined by hbase.hstore.blockingStoreFiles.
+ After this time has elapsed, the HRegion will stop blocking updates even
+ if a compaction has not been completed. Default: 90 seconds.
+ </description>
+ </property>
+ <property>
<name>hbase.hstore.compaction.max</name>
<value>10</value>
<description>Max number of HStoreFiles to compact per 'minor' compaction.
@@ -382,7 +402,7 @@
</property>
<property>
<name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <value>60000</value>
<description>ZooKeeper session timeout. This option is not used by HBase
directly, it is for the internals of ZooKeeper. HBase merely passes it in
whenever a connection is established to ZooKeeper. It is used by ZooKeeper
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HTableDescriptor.java Wed Sep 9 17:14:22 2009
@@ -101,7 +101,7 @@
// Key is hash of the family name.
public final Map<byte [], HColumnDescriptor> families =
- new TreeMap<byte [], HColumnDescriptor>(KeyValue.FAMILY_COMPARATOR);
+ new TreeMap<byte [], HColumnDescriptor>(Bytes.BYTES_RAWCOMPARATOR);
/**
* Private constructor used internally creating table descriptors for
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/KeyValue.java Wed Sep 9 17:14:22 2009
@@ -22,6 +22,7 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Comparator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -68,7 +69,8 @@
*/
public static final char COLUMN_FAMILY_DELIMITER = ':';
- public static final byte[] COLUMN_FAMILY_DELIM_ARRAY = new byte[]{COLUMN_FAMILY_DELIMITER};
+ public static final byte[] COLUMN_FAMILY_DELIM_ARRAY =
+ new byte[]{COLUMN_FAMILY_DELIMITER};
/**
* Comparator for plain key/values; i.e. non-catalog table key/values.
@@ -106,30 +108,6 @@
public static KeyComparator ROOT_KEY_COMPARATOR = new RootKeyComparator();
/**
- * Comparator that compares the family portion of columns only.
- * Use this making NavigableMaps of Stores or when you need to compare
- * column family portion only of two column names.
- */
- public static final RawComparator<byte []> FAMILY_COMPARATOR =
- new RawComparator<byte []> () {
- public int compare(byte [] a, int ao, int al, byte [] b, int bo, int bl) {
- int indexa = KeyValue.getDelimiter(a, ao, al, COLUMN_FAMILY_DELIMITER);
- if (indexa < 0) {
- indexa = al;
- }
- int indexb = KeyValue.getDelimiter(b, bo, bl, COLUMN_FAMILY_DELIMITER);
- if (indexb < 0) {
- indexb = bl;
- }
- return Bytes.compareTo(a, ao, indexa, b, bo, indexb);
- }
-
- public int compare(byte[] a, byte[] b) {
- return compare(a, 0, a.length, b, 0, b.length);
- }
- };
-
- /**
* Get the appropriate row comparator for the specified table.
*
* Hopefully we can get rid of this, I added this here because it's replacing
@@ -291,11 +269,12 @@
/**
* Constructs KeyValue structure filled with null value.
+ * Sets type to {@link KeyValue.Type#Maximum}
* @param row - row key (arbitrary byte array)
* @param timestamp
*/
public KeyValue(final byte [] row, final long timestamp) {
- this(row, timestamp, Type.Put);
+ this(row, timestamp, Type.Maximum);
}
/**
@@ -309,13 +288,14 @@
/**
* Constructs KeyValue structure filled with null value.
+ * Sets type to {@link KeyValue.Type#Maximum}
* @param row - row key (arbitrary byte array)
* @param family family name
* @param qualifier column qualifier
*/
public KeyValue(final byte [] row, final byte [] family,
final byte [] qualifier) {
- this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put);
+ this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
}
/**
@@ -569,48 +549,9 @@
* @return Fully copied clone of this KeyValue
*/
public KeyValue clone() {
- byte [] bytes = new byte[this.length];
- System.arraycopy(this.bytes, this.offset, bytes, 0, this.length);
- return new KeyValue(bytes, 0, bytes.length);
- }
-
- /**
- * Clones a row.
- *
- * @param timestamp The new time stamp for the row.
- * @return Clone of bb's key portion with only the row and timestamp filled in.
- */
- public KeyValue cloneRow(final long timestamp) {
- return new KeyValue(getBuffer(), getRowOffset(), getRowLength(),
- null, 0, 0, null, 0, 0,
- timestamp, Type.codeToType(getType()), null, 0, 0);
- }
-
- /**
- * @return Clone of bb's key portion with type set to Type.Maximum. Use this
- * doing lookups where you are doing getClosest. Using Maximum, you'll be
- * sure to trip over all of the other key types since Maximum sorts first.
- */
- public KeyValue cloneMaximum() {
- return createKey(Type.Maximum);
- }
-
- /**
- * Make a clone with the new type. Does not copy value.
- *
- * @param newtype New type to set on clone of this key.
- * @return Clone of this key with type set to <code>newtype</code>
- */
- private KeyValue createKey(final Type newtype) {
- int keylength = getKeyLength();
- int l = keylength + ROW_OFFSET;
- byte [] other = new byte[l];
- System.arraycopy(getBuffer(), getOffset(), other, 0, l);
- // Set value length to zero.
- Bytes.putInt(other, Bytes.SIZEOF_INT, 0);
- // Set last byte, the type, to new type
- other[l - 1] = newtype.getCode();
- return new KeyValue(other, 0, other.length);
+ byte [] b = new byte[this.length];
+ System.arraycopy(this.bytes, this.offset, b, 0, this.length);
+ return new KeyValue(b, 0, b.length);
}
//---------------------------------------------------------------------------
@@ -855,9 +796,8 @@
}
public boolean updateLatestStamp(final byte [] now) {
- int tsOffset = getTimestampOffset();
- if(Bytes.compareTo(now, 0, Bytes.SIZEOF_LONG,
- this.bytes, tsOffset, Bytes.SIZEOF_LONG) < 0) {
+ if(this.isLatestTimestamp()) {
+ int tsOffset = getTimestampOffset();
System.arraycopy(now, 0, this.bytes, tsOffset, Bytes.SIZEOF_LONG);
return true;
}
@@ -885,8 +825,10 @@
}
/**
- * Do not use unless you have to. Use {@link #getBuffer()} with appropriate
- * offset and lengths instead.
+ * Returns value in a new byte array.
+ * Primarily for use client-side. If server-side, use
+ * {@link #getBuffer()} with appropriate offsets and lengths instead to
+ * save on allocations.
* @return Value in a new byte array.
*/
public byte [] getValue() {
@@ -946,17 +888,27 @@
}
/**
- * @return True if Delete KeyValue type.
+ * @return True if a delete type, a {@link KeyValue.Type#Delete} or
+ * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn}
+ * KeyValue type.
+ */
+ public boolean isDelete() {
+ int t = getType();
+ return Type.Delete.getCode() <= t && t <= Type.DeleteFamily.getCode();
+ }
+
+ /**
+ * @return True if this KV is a {@link KeyValue.Type#Delete} type.
*/
public boolean isDeleteType() {
- return getType() == Type.Delete.code;
+ return getType() == Type.Delete.getCode();
}
/**
- * @return True if DeleteColumn KeyValue type.
+ * @return True if this KV is a delete family type.
*/
- public boolean isDeleteColumnType() {
- return getType() == Type.DeleteColumn.code;
+ public boolean isDeleteFamily() {
+ return getType() == Type.DeleteFamily.getCode();
}
/**
@@ -1258,13 +1210,13 @@
return index;
}
- /*
+ /**
* @param b
* @param delimiter
- * @return Index of delimiter having started from end of <code>b</code> moving
- * leftward.
+ * @return Index of delimiter having started from start of <code>b</code>
+ * moving rightward.
*/
- static int getDelimiter(final byte [] b, int offset, final int length,
+ public static int getDelimiter(final byte [] b, int offset, final int length,
final int delimiter) {
if (b == null) {
throw new NullPointerException();
@@ -1279,12 +1231,13 @@
return result;
}
- /*
+ /**
+ * Find index of passed delimiter walking from end of buffer backwards.
* @param b
* @param delimiter
* @return Index of delimiter
*/
- static int getDelimiterInReverse(final byte [] b, final int offset,
+ public static int getDelimiterInReverse(final byte [] b, final int offset,
final int length, final int delimiter) {
if (b == null) {
throw new NullPointerException();
@@ -1659,6 +1612,21 @@
}
/**
+ * Comparator that compares row component only of a KeyValue.
+ */
+ public static class RowComparator implements Comparator<KeyValue> {
+ final KVComparator comparator;
+
+ public RowComparator(final KVComparator c) {
+ this.comparator = c;
+ }
+
+ public int compare(KeyValue left, KeyValue right) {
+ return comparator.compareRows(left, right);
+ }
+ }
+
+ /**
* Compare key portion of a {@link KeyValue} for keys in <code>.META.</code>
* table.
*/
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnection.java Wed Sep 9 17:14:22 2009
@@ -189,6 +189,6 @@
* @param tableName The name of the table
* @throws IOException
*/
- public void processBatchOfRows(ArrayList<Put> list, byte[] tableName)
+ public int processBatchOfRows(ArrayList<Put> list, byte[] tableName)
throws IOException;
}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Wed Sep 9 17:14:22 2009
@@ -79,6 +79,9 @@
final Map<HBaseConfiguration, TableServers> HBASE_INSTANCES =
new WeakHashMap<HBaseConfiguration, TableServers>();
+ private static final Map<String, ClientZKWatcher> ZK_WRAPPERS =
+ new HashMap<String, ClientZKWatcher>();
+
/**
* Get the connection object for the instance specified by the configuration
* If no current connection exists, create a new connection for that instance
@@ -124,15 +127,99 @@
}
}
}
+ synchronized (ZK_WRAPPERS) {
+ for (ClientZKWatcher watch : ZK_WRAPPERS.values()) {
+ watch.resetZooKeeper();
+ }
+ }
+ }
+
+ /**
+ * Get a watcher of a zookeeper connection for a given quorum address.
+ * If the connection isn't established, a new one is created.
+ * This acts like a multiton.
+ * @param conf
+ * @return ZKW watcher
+ * @throws IOException
+ */
+ public static synchronized ClientZKWatcher getClientZooKeeperWatcher(
+ HBaseConfiguration conf) throws IOException {
+ if (!ZK_WRAPPERS.containsKey(conf.get(HConstants.ZOOKEEPER_QUORUM))) {
+ ZK_WRAPPERS.put(conf.get(HConstants.ZOOKEEPER_QUORUM),
+ new ClientZKWatcher(conf));
+ }
+ return ZK_WRAPPERS.get(conf.get(HConstants.ZOOKEEPER_QUORUM));
+ }
+
+ /**
+ * This class is responsible to handle connection and reconnection
+ * to a zookeeper quorum.
+ *
+ */
+ public static class ClientZKWatcher implements Watcher {
+
+ static final Log LOG = LogFactory.getLog(ClientZKWatcher.class);
+ private ZooKeeperWrapper zooKeeperWrapper;
+ private HBaseConfiguration conf;
+
+ /**
+ * Takes a configuration to pass it to ZKW but won't instanciate it
+ * @param conf
+ * @throws IOException
+ */
+ public ClientZKWatcher(HBaseConfiguration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Called by ZooKeeper when an event occurs on our connection. We use this to
+ * detect our session expiring. When our session expires, we have lost our
+ * connection to ZooKeeper. Our handle is dead, and we need to recreate it.
+ *
+ * See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ * for more information.
+ *
+ * @param event WatchedEvent witnessed by ZooKeeper.
+ */
+ public void process(WatchedEvent event) {
+ KeeperState state = event.getState();
+ LOG.debug("Got ZooKeeper event, state: " + state + ", type: "
+ + event.getType() + ", path: " + event.getPath());
+ if (state == KeeperState.Expired) {
+ resetZooKeeper();
+ }
+ }
+
+ /**
+ * Get this watcher's ZKW, instanciate it if necessary.
+ * @return ZKW
+ */
+ public ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
+ if(zooKeeperWrapper == null) {
+ zooKeeperWrapper = new ZooKeeperWrapper(conf, this);
+ }
+ return zooKeeperWrapper;
+ }
+
+ /**
+ * Clear this connection to zookeeper.
+ */
+ private synchronized void resetZooKeeper() {
+ if (zooKeeperWrapper != null) {
+ zooKeeperWrapper.close();
+ zooKeeperWrapper = null;
+ }
+ }
}
/* Encapsulates finding the servers for an HBase instance */
- private static class TableServers implements ServerConnection, HConstants, Watcher {
+ private static class TableServers implements ServerConnection, HConstants {
static final Log LOG = LogFactory.getLog(TableServers.class);
private final Class<? extends HRegionInterface> serverInterfaceClass;
private final long pause;
private final int numRetries;
private final int maxRPCAttempts;
+ private final long rpcTimeout;
private final Object masterLock = new Object();
private volatile boolean closed;
@@ -156,8 +243,6 @@
cachedRegionLocations =
new HashMap<Integer, SoftValueSortedMap<byte [], HRegionLocation>>();
- private ZooKeeperWrapper zooKeeperWrapper;
-
/**
* constructor
* @param conf Configuration object
@@ -183,7 +268,8 @@
this.pause = conf.getLong("hbase.client.pause", 2 * 1000);
this.numRetries = conf.getInt("hbase.client.retries.number", 10);
this.maxRPCAttempts = conf.getInt("hbase.client.rpc.maxattempts", 1);
-
+ this.rpcTimeout = conf.getLong("hbase.regionserver.lease.period", 60000);
+
this.master = null;
this.masterChecked = false;
}
@@ -195,32 +281,6 @@
return this.pause * HConstants.RETRY_BACKOFF[ntries];
}
- /**
- * Called by ZooKeeper when an event occurs on our connection. We use this to
- * detect our session expiring. When our session expires, we have lost our
- * connection to ZooKeeper. Our handle is dead, and we need to recreate it.
- *
- * See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
- * for more information.
- *
- * @param event WatchedEvent witnessed by ZooKeeper.
- */
- public void process(WatchedEvent event) {
- KeeperState state = event.getState();
- LOG.debug("Got ZooKeeper event, state: " + state + ", type: " +
- event.getType() + ", path: " + event.getPath());
- if (state == KeeperState.Expired) {
- resetZooKeeper();
- }
- }
-
- private synchronized void resetZooKeeper() {
- if (zooKeeperWrapper != null) {
- zooKeeperWrapper.close();
- zooKeeperWrapper = null;
- }
- }
-
// Used by master and region servers during safe mode only
public void unsetRootRegionLocation() {
this.rootRegionLocation = null;
@@ -813,11 +873,10 @@
return getHRegionConnection(regionServer, false);
}
- public synchronized ZooKeeperWrapper getZooKeeperWrapper() throws IOException {
- if (zooKeeperWrapper == null) {
- zooKeeperWrapper = new ZooKeeperWrapper(conf, this);
- }
- return zooKeeperWrapper;
+ public synchronized ZooKeeperWrapper getZooKeeperWrapper()
+ throws IOException {
+ return HConnectionManager.getClientZooKeeperWatcher(conf)
+ .getZooKeeperWrapper();
}
/*
@@ -996,10 +1055,10 @@
return location;
}
- public void processBatchOfRows(ArrayList<Put> list, byte[] tableName)
+ public int processBatchOfRows(ArrayList<Put> list, byte[] tableName)
throws IOException {
if (list.isEmpty()) {
- return;
+ return 0;
}
boolean retryOnlyOne = false;
if (list.size() > 1) {
@@ -1013,7 +1072,8 @@
byte [] region = currentRegion;
boolean isLastRow = false;
Put [] putarray = new Put[0];
- for (int i = 0, tries = 0; i < list.size() && tries < this.numRetries; i++) {
+ int i, tries;
+ for (i = 0, tries = 0; i < list.size() && tries < this.numRetries; i++) {
Put put = list.get(i);
currentPuts.add(put);
// If the next Put goes to a new region, then we are to clear
@@ -1071,6 +1131,7 @@
currentPuts.clear();
}
}
+ return i;
}
void close(boolean stopProxy) {
@@ -1081,7 +1142,6 @@
master = null;
masterChecked = false;
}
- resetZooKeeper();
if (stopProxy) {
synchronized (servers) {
for (HRegionInterface i: servers.values()) {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HTable.java Wed Sep 9 17:14:22 2009
@@ -54,16 +54,18 @@
/**
- * Used to communicate with a single HBase table
- * TODO: checkAndSave in oldAPI
- * TODO: Regex deletes.
+ * Used to communicate with a single HBase table.
+ * This class is not thread safe for writes.
+ * Gets, puts, and deletes take out a row lock for the duration
+ * of their operation. Scans (currently) do not respect
+ * row locking.
*/
public class HTable {
private final HConnection connection;
private final byte [] tableName;
protected final int scannerTimeout;
private volatile HBaseConfiguration configuration;
- private ArrayList<Put> writeBuffer;
+ private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
private long writeBufferSize;
private boolean autoFlush;
private long currentWriteBufferSize;
@@ -123,7 +125,6 @@
conf.getInt("hbase.regionserver.lease.period", 60 * 1000);
this.configuration = conf;
this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
- this.writeBuffer = new ArrayList<Put>();
this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
this.autoFlush = true;
this.currentWriteBufferSize = 0;
@@ -352,7 +353,9 @@
*/
public RowResult getClosestRowBefore(final byte[] row, final byte[] family)
throws IOException {
- Result r = getRowOrBefore(row, family);
+ // Do parse in case we are passed a family with a ':' on it.
+ final byte [] f = KeyValue.parseColumn(family)[0];
+ Result r = getRowOrBefore(row, f);
return r == null || r.isEmpty()? null: r.getRowResult();
}
@@ -580,14 +583,18 @@
* @throws IOException
*/
public void flushCommits() throws IOException {
+ int last = 0;
try {
- connection.processBatchOfRows(writeBuffer, tableName);
+ last = connection.processBatchOfRows(writeBuffer, tableName);
} finally {
+ writeBuffer.subList(0, last).clear();
currentWriteBufferSize = 0;
- writeBuffer.clear();
+ for (int i = 0; i < writeBuffer.size(); i++) {
+ currentWriteBufferSize += writeBuffer.get(i).heapSize();
+ }
}
}
-
+
/**
* Release held resources
*
@@ -672,19 +679,17 @@
}
/**
- * Set the size of the buffer in bytes
+ * Set the size of the buffer in bytes.
+ * If the new size is lower than the current size of data in the
+ * write buffer, the buffer is flushed.
* @param writeBufferSize
+ * @throws IOException
*/
- public void setWriteBufferSize(long writeBufferSize) {
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
this.writeBufferSize = writeBufferSize;
- }
-
- /**
- * Get the write buffer
- * @return the current write buffer
- */
- public ArrayList<Put> getWriteBuffer() {
- return writeBuffer;
+ if(currentWriteBufferSize > writeBufferSize) {
+ flushCommits();
+ }
}
// Old API. Pre-hbase-880, hbase-1304.
@@ -792,9 +797,8 @@
g.addColumn(fq[0], fq[1]);
}
g.setMaxVersions(numVersions);
- if (timestamp != HConstants.LATEST_TIMESTAMP) {
- g.setTimeStamp(timestamp);
- }
+ g.setTimeRange(0,
+ timestamp == HConstants.LATEST_TIMESTAMP ? timestamp : timestamp+1);
Result r = get(g);
return r == null || r.size() <= 0? null: r.getCellValues();
}
@@ -1050,9 +1054,8 @@
}
}
g.setMaxVersions(numVersions);
- if (ts != HConstants.LATEST_TIMESTAMP) {
- g.setTimeStamp(ts);
- }
+ g.setTimeRange(0,
+ ts == HConstants.LATEST_TIMESTAMP ? ts : ts+1);
Result r = get(g);
return r == null || r.size() <= 0? null: r.getRowResult();
}
@@ -1306,6 +1309,8 @@
scan.addColumn(splits[0], splits[1]);
}
}
+ scan.setTimeRange(0,
+ timestamp == HConstants.LATEST_TIMESTAMP ? timestamp : timestamp+1);
OldClientScanner s = new OldClientScanner(new ClientScanner(scan));
s.initialize();
return s;
@@ -1702,7 +1707,8 @@
final long timestamp, final RowLock rl) throws IOException {
final Get g = new Get(row, rl);
g.addColumn(column);
- g.setTimeStamp(timestamp);
+ g.setTimeRange(0,
+ timestamp == HConstants.LATEST_TIMESTAMP ? timestamp : timestamp+1);
return exists(g);
}
@@ -1787,7 +1793,7 @@
private HRegionInfo currentRegion = null;
private ScannerCallable callable = null;
private final LinkedList<Result> cache = new LinkedList<Result>();
- private final int caching = HTable.this.scannerCaching;
+ private final int caching;
private long lastNext;
// Keep lastResult returned successfully in case we have to reset scanner.
private Result lastResult = null;
@@ -1800,7 +1806,14 @@
}
this.scan = scan;
this.lastNext = System.currentTimeMillis();
-
+
+ // Use the caching from the Scan. If not set, use the default cache setting for this table.
+ if (this.scan.getCaching() > 0) {
+ this.caching = this.scan.getCaching();
+ } else {
+ this.caching = HTable.this.scannerCaching;
+ }
+
// Removed filter validation. We have a new format now, only one of all
// the current filters has a validate() method. We can add it back,
// need to decide on what we're going to do re: filter redesign.
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java Wed Sep 9 17:14:22 2009
@@ -68,7 +68,7 @@
public Put(byte [] row) {
this(row, null);
}
-
+
/**
* Create a Put operation for the specified row, using an existing row lock.
* @param row row key
@@ -83,7 +83,7 @@
this.lockId = rowLock.getLockId();
}
}
-
+
/**
* Copy constructor. Creates a Put operation cloned from the specified Put.
* @param putToCopy put to copy
@@ -245,7 +245,7 @@
/**
* @return true if edits should be applied to WAL, false if not
*/
- public boolean writeToWAL() {
+ public boolean getWriteToWAL() {
return this.writeToWAL;
}
@@ -254,7 +254,7 @@
* Not writing the WAL means you may lose edits on server crash.
* @param write true if edits should be written to WAL, false if not
*/
- public void writeToWAL(boolean write) {
+ public void setWriteToWAL(boolean write) {
this.writeToWAL = write;
}
@@ -333,8 +333,7 @@
this.lockId = in.readLong();
this.writeToWAL = in.readBoolean();
int numFamilies = in.readInt();
- this.familyMap =
- new TreeMap<byte [],List<KeyValue>>(Bytes.BYTES_COMPARATOR);
+ if (!this.familyMap.isEmpty()) this.familyMap.clear();
for(int i=0;i<numFamilies;i++) {
byte [] family = Bytes.readByteArray(in);
int numKeys = in.readInt();
@@ -359,7 +358,7 @@
out.writeLong(this.lockId);
out.writeBoolean(this.writeToWAL);
out.writeInt(familyMap.size());
- for(Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
+ for (Map.Entry<byte [], List<KeyValue>> entry : familyMap.entrySet()) {
Bytes.writeByteArray(out, entry.getKey());
List<KeyValue> keys = entry.getValue();
out.writeInt(keys.size());
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Result.java Wed Sep 9 17:14:22 2009
@@ -141,7 +141,7 @@
* Map of families to all versions of its qualifiers and values.
* <p>
* Returns a three level Map of the form:
- * <code>Map<family,Map<qualifier,Map<timestamp,value>>></code>
+ * <code>Map<family,Map<qualifier,Map<timestamp,value>>></code>
* <p>
* Note: All other map returning methods make use of this map internally.
* @return map from families to qualifiers to versions
@@ -186,7 +186,7 @@
/**
* Map of families to their most recent qualifiers and values.
* <p>
- * Returns a two level Map of the form: <code>Map<family,Map<qualifier,value>></code>
+ * Returns a two level Map of the form: <code>Map<family,Map<qualifier,value>></code>
* <p>
* The most recent version of each qualifier will be used.
* @return map from families to qualifiers and value
@@ -218,7 +218,7 @@
/**
* Map of qualifiers to values.
* <p>
- * Returns a Map of the form: <code>Map<qualifier,value></code>
+ * Returns a Map of the form: <code>Map<qualifier,value></code>
* @return map of qualifiers to values
*/
public NavigableMap<byte[], byte[]> getFamilyMap(byte [] family) {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Scan.java Wed Sep 9 17:14:22 2009
@@ -20,6 +20,14 @@
package org.apache.hadoop.hbase.client;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
@@ -30,14 +38,6 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
/**
* Used to perform Scan operations.
* <p>
@@ -47,6 +47,9 @@
* iterate over all rows.
* <p>
* To scan everything for each row, instantiate a Scan object.
+ * <p>
+ * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
+ * <p>
* To further define the scope of what to get when scanning, perform additional
* methods as outlined below.
* <p>
@@ -71,6 +74,7 @@
private byte [] startRow = HConstants.EMPTY_START_ROW;
private byte [] stopRow = HConstants.EMPTY_END_ROW;
private int maxVersions = 1;
+ private int caching = -1;
private Filter filter = null;
private RowFilterInterface oldFilter = null;
private TimeRange tr = new TimeRange();
@@ -118,6 +122,7 @@
startRow = scan.getStartRow();
stopRow = scan.getStopRow();
maxVersions = scan.getMaxVersions();
+ caching = scan.getCaching();
filter = scan.getFilter(); // clone?
oldFilter = scan.getOldFilter(); // clone?
TimeRange ctr = scan.getTimeRange();
@@ -308,7 +313,17 @@
this.maxVersions = maxVersions;
return this;
}
-
+
+ /**
+ * Set the number of rows for caching that will be passed to scanners.
+ * If not set, the default setting from {@link HTable#getScannerCaching()} will apply.
+ * Higher caching values will enable faster scanners but will use more memory.
+ * @param caching the number of rows for caching
+ */
+ public void setCaching(int caching) {
+ this.caching = caching;
+ }
+
/**
* Apply the specified server-side filter when performing the Scan.
* @param filter filter to run on the server
@@ -397,6 +412,13 @@
}
/**
+ * @return caching the number of rows fetched when calling next on a scanner
+ */
+ public int getCaching() {
+ return this.caching;
+ }
+
+ /**
* @return TimeRange
*/
public TimeRange getTimeRange() {
@@ -438,6 +460,8 @@
sb.append(Bytes.toString(this.stopRow));
sb.append(", maxVersions=");
sb.append("" + this.maxVersions);
+ sb.append(", caching=");
+ sb.append("" + this.caching);
sb.append(", timeRange=");
sb.append("[" + this.tr.getMin() + "," + this.tr.getMax() + ")");
sb.append(", families=");
@@ -493,6 +517,7 @@
this.startRow = Bytes.readByteArray(in);
this.stopRow = Bytes.readByteArray(in);
this.maxVersions = in.readInt();
+ this.caching = in.readInt();
if(in.readBoolean()) {
this.filter = (Filter)createForName(Bytes.toString(Bytes.readByteArray(in)));
this.filter.readFields(in);
@@ -524,6 +549,7 @@
Bytes.writeByteArray(out, this.startRow);
Bytes.writeByteArray(out, this.stopRow);
out.writeInt(this.maxVersions);
+ out.writeInt(this.caching);
if(this.filter == null) {
out.writeBoolean(false);
} else {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/ScannerCallable.java Wed Sep 9 17:14:22 2009
@@ -29,7 +29,6 @@
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.ipc.RemoteException;
-
/**
* Retries scanner operations such as create, next, etc.
* Used by {@link ResultScanner}s made by {@link HTable}.
@@ -99,6 +98,7 @@
try {
this.server.close(this.scannerId);
} catch (IOException e) {
+ // ignore
}
this.scannerId = -1L;
}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/package-info.java Wed Sep 9 17:14:22 2009
@@ -30,14 +30,13 @@
<p>To administer HBase, create and drop tables, list and alter tables,
use {@link org.apache.hadoop.hbase.client.HBaseAdmin}. Once created, table access is via an instance
of {@link org.apache.hadoop.hbase.client.HTable}. You add content to a table a row at a time. To insert,
- create an instance of a {@link org.apache.hadoop.hbase.client.Put} object and after setting it appropriately,
- commit your update using {@link org.apache.hadoop.hbase.client.HTable#put(Put)}.
- To fetch your inserted
- value, use {@link org.apache.hadoop.hbase.client.Get}. The Get can be specified to be broad -- get all
- on a particular row -- or narrow; return only a single cell value.
- When finished with your Get settings, invoke {@link org.apache.hadoop.hbase.client.HTable#get(Get)}. Use
- {@link org.apache.hadoop.hbase.client.Scan} to set up a scanner -- a Cursor-like access. After
- configuring your Scan instance, call {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} and then
+ create an instance of a {@link org.apache.hadoop.hbase.client.Put} object. Specify value, target column
+ and optionally a timestamp. Commit your update using {@link org.apache.hadoop.hbase.client.HTable#put(Put)}.
+ To fetch your inserted value, use {@link org.apache.hadoop.hbase.client.Get}. The Get can be specified to be broad -- get all
+ on a particular row -- or narrow; i.e. return only a single cell value. After creating an instance of
+ Get, invoke {@link org.apache.hadoop.hbase.client.HTable#get(Get)}. Use
+ {@link org.apache.hadoop.hbase.client.Scan} to set up a scanner -- a Cursor- like access. After
+ creating and configuring your Scan instance, call {@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} and then
invoke next on the returned object. Both {@link org.apache.hadoop.hbase.client.HTable#get(Get)} and
{@link org.apache.hadoop.hbase.client.HTable#getScanner(Scan)} return a
{@link org.apache.hadoop.hbase.client.Result}.
@@ -47,6 +46,9 @@
You can remove individual cells or entire families, etc. Pass it to
{@link org.apache.hadoop.hbase.client.HTable#delete(Delete)} to execute.
</p>
+ <p>Puts, Gets and Deletes take out a lock on the target row for the duration of their operation.
+ Scans (currently) operate without respect for row locks.
+ </p>
<p>Client code accessing a cluster finds the cluster by querying ZooKeeper.
This means that the ZooKeeper quorum to use must be on the client CLASSPATH.
Usually this means make sure the client can find your <code>hbase-site.xml</code>.
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/BinaryComparator.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * A binary comparator which lexicographically compares against the specified
+ * byte array using {@link Bytes#compareTo(byte[], byte[])}.
+ */
+public class BinaryComparator implements WritableByteArrayComparable {
+
+ private byte [] value;
+
+ /**
+ * Writable constructor, do not use.
+ */
+ public BinaryComparator() {
+ }
+
+ /**
+ * Constructor.
+ * @param value the value to compare against
+ */
+ public BinaryComparator(byte [] value) {
+ this.value = value;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ value = Bytes.readByteArray(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Bytes.writeByteArray(out, value);
+ }
+
+ @Override
+ public int compareTo(byte [] value) {
+ return Bytes.compareTo(this.value, value);
+ }
+}
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,141 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.io.ObjectWritable;
+
+/**
+ * This is a generic filter to be used to filter by comparison. It takes an
+ * operator (equal, greater, not equal, etc) and a byte [] comparator.
+ * <p>
+ * To filter by row key, use {@link RowFilter}.
+ * <p>
+ * To filter by column qualifier, use {@link QualifierFilter}.
+ * <p>
+ * To filter by value, use {@link SingleColumnValueFilter}.
+ * <p>
+ * These filters can be wrapped with {@link SkipFilter} and {@link WhileMatchFilter}
+ * to add more control.
+ * <p>
+ * Multiple filters can be combined using {@link FilterList}.
+ */
+public abstract class CompareFilter implements Filter {
+
+ /** Comparison operators. */
+ public enum CompareOp {
+ /** less than */
+ LESS,
+ /** less than or equal to */
+ LESS_OR_EQUAL,
+ /** equals */
+ EQUAL,
+ /** not equal */
+ NOT_EQUAL,
+ /** greater than or equal to */
+ GREATER_OR_EQUAL,
+ /** greater than */
+ GREATER;
+ }
+
+ protected CompareOp compareOp;
+ protected WritableByteArrayComparable comparator;
+
+ /**
+ * Writable constructor, do not use.
+ */
+ public CompareFilter() {
+ }
+
+ /**
+ * Constructor.
+ * @param compareOp the compare op for row matching
+ * @param comparator the comparator for row matching
+ */
+ public CompareFilter(final CompareOp compareOp,
+ final WritableByteArrayComparable comparator) {
+ this.compareOp = compareOp;
+ this.comparator = comparator;
+ }
+
+ public void reset() {
+ }
+
+ public ReturnCode filterKeyValue(KeyValue v) {
+ return ReturnCode.INCLUDE;
+ }
+
+ public boolean filterRowKey(byte[] data, int offset, int length) {
+ return false;
+ }
+
+ public boolean filterRow() {
+ return false;
+ }
+
+ public boolean filterAllRemaining() {
+ return false;
+ }
+
+ protected boolean doCompare(final CompareOp compareOp,
+ final WritableByteArrayComparable comparator, final byte [] data,
+ final int offset, final int length) {
+ int compareResult =
+ comparator.compareTo(Arrays.copyOfRange(data, offset,
+ offset + length));
+ switch (compareOp) {
+ case LESS:
+ return compareResult <= 0;
+ case LESS_OR_EQUAL:
+ return compareResult < 0;
+ case EQUAL:
+ return compareResult != 0;
+ case NOT_EQUAL:
+ return compareResult == 0;
+ case GREATER_OR_EQUAL:
+ return compareResult > 0;
+ case GREATER:
+ return compareResult >= 0;
+ default:
+ throw new RuntimeException("Unknown Compare op " +
+ compareOp.name());
+ }
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ compareOp = CompareOp.valueOf(in.readUTF());
+ comparator = (WritableByteArrayComparable)
+ HbaseObjectWritable.readObject(in, null);
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(compareOp.name());
+ ObjectWritable.writeObject(out, comparator,
+ WritableByteArrayComparable.class, null);
+ }
+}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java Wed Sep 9 17:14:22 2009
@@ -35,6 +35,7 @@
*/
public class InclusiveStopFilter implements Filter {
private byte [] stopRowKey;
+ private boolean done = false;
public InclusiveStopFilter() {
super();
@@ -56,12 +57,17 @@
return false;
}
// if stopRowKey is <= buffer, then true, filter row.
- return Bytes.compareTo(stopRowKey, 0, stopRowKey.length,
- buffer, offset, length) < 0;
+ int cmp = Bytes.compareTo(stopRowKey, 0, stopRowKey.length,
+ buffer, offset, length);
+
+ if(cmp < 0) {
+ done = true;
+ }
+ return done;
}
public boolean filterAllRemaining() {
- return false;
+ return done;
}
public ReturnCode filterKeyValue(KeyValue v) {
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PageFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PageFilter.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PageFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PageFilter.java Wed Sep 9 17:14:22 2009
@@ -29,14 +29,12 @@
* Implementation of Filter interface that limits results to a specific page
* size. It terminates scanning once the number of filter-passed rows is >
* the given page size.
- *
* <p>
* Note that this filter cannot guarantee that the number of results returned
* to a client are <= page size. This is because the filter is applied
* separately on different region servers. It does however optimize the scan of
* individual HRegions by making sure that the page size is never exceeded
* locally.
- * </p>
*/
public class PageFilter implements Filter {
private long pageSize = Long.MAX_VALUE;
@@ -60,16 +58,15 @@
}
public void reset() {
- rowsAccepted = 0;
+ // noop
}
public boolean filterAllRemaining() {
- return this.rowsAccepted > this.pageSize;
+ return this.rowsAccepted >= this.pageSize;
}
public boolean filterRowKey(byte[] rowKey, int offset, int length) {
- this.rowsAccepted++;
- return filterAllRemaining();
+ return false;
}
public void readFields(final DataInput in) throws IOException {
@@ -81,10 +78,11 @@
}
public ReturnCode filterKeyValue(KeyValue v) {
- return filterAllRemaining() ? ReturnCode.NEXT_ROW : ReturnCode.INCLUDE;
+ return ReturnCode.INCLUDE;
}
public boolean filterRow() {
- return filterAllRemaining();
+ this.rowsAccepted++;
+ return this.rowsAccepted > this.pageSize;
}
}
\ No newline at end of file
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PrefixFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PrefixFilter.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PrefixFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/PrefixFilter.java Wed Sep 9 17:14:22 2009
@@ -32,6 +32,7 @@
*/
public class PrefixFilter implements Filter {
protected byte [] prefix = null;
+ protected boolean passedPrefix = false;
public PrefixFilter(final byte [] prefix) {
this.prefix = prefix;
@@ -52,12 +53,17 @@
return true;
// if they are equal, return false => pass row
// else return true, filter row
- return Bytes.compareTo(buffer, offset, this.prefix.length, this.prefix, 0,
- this.prefix.length) != 0;
+ // if we are passed the prefix, set flag
+ int cmp = Bytes.compareTo(buffer, offset, this.prefix.length, this.prefix, 0,
+ this.prefix.length);
+ if(cmp > 0) {
+ passedPrefix = true;
+ }
+ return cmp != 0;
}
public boolean filterAllRemaining() {
- return false;
+ return passedPrefix;
}
public ReturnCode filterKeyValue(KeyValue v) {
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/QualifierFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/QualifierFilter.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/QualifierFilter.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/QualifierFilter.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,68 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+
+/**
+ * This filter is used to filter based on the column qualifier. It takes an
+ * operator (equal, greater, not equal, etc) and a byte [] comparator for the
+ * column qualifier portion of a key.
+ * <p>
+ * This filter can be wrapped with {@link WhileMatchFilter} and {@link SkipFilter}
+ * to add more control.
+ * <p>
+ * Multiple filters can be combined using {@link FilterList}.
+ * <p>
+ * If an already known column qualifier is looked for, use {@link Get#addColumn}
+ * directly rather than a filter.
+ */
+public class QualifierFilter extends CompareFilter {
+
+ /**
+ * Writable constructor, do not use.
+ */
+ public QualifierFilter() {
+ }
+
+ /**
+ * Constructor.
+ * @param qualifierCompareOp the compare op for column qualifier matching
+ * @param qualifierComparator the comparator for column qualifier matching
+ */
+ public QualifierFilter(final CompareOp qualifierCompareOp,
+ final WritableByteArrayComparable qualifierComparator) {
+ super(qualifierCompareOp, qualifierComparator);
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(KeyValue v) {
+ int qualifierLength = v.getQualifierLength();
+ if (qualifierLength > 0) {
+ if (doCompare(this.compareOp, this.comparator, v.getBuffer(),
+ v.getQualifierOffset(), qualifierLength)) {
+ return ReturnCode.SKIP;
+ }
+ }
+ return ReturnCode.INCLUDE;
+ }
+}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RegexStringComparator.java Wed Sep 9 17:14:22 2009
@@ -27,18 +27,18 @@
import org.apache.hadoop.hbase.util.Bytes;
/**
- * This comparator is for use with ColumnValueFilter, for filtering based on
- * the value of a given column. Use it to test if a given regular expression
- * matches a cell value in the column.
+ * This comparator is for use with {@link CompareFilter} implementations, such
+ * as {@link RowFilter}, {@link QualifierFilter}, and {@link ValueFilter}, for
+ * filtering based on the value of a given column. Use it to test if a given
+ * regular expression matches a cell value in the column.
* <p>
- * Only EQUAL or NOT_EQUAL tests are valid with this comparator.
+ * Only EQUAL or NOT_EQUAL {@link org.apache.hadoop.hbase.filter.CompareFilter.CompareOp}
+ * comparisons are valid with this comparator.
* <p>
* For example:
* <p>
* <pre>
- * ColumnValueFilter cvf =
- * new ColumnValueFilter("col",
- * ColumnValueFilter.CompareOp.EQUAL,
+ * ValueFilter vf = new ValueFilter(CompareOp.EQUAL,
* new RegexStringComparator(
* // v4 IP address
* "(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3,3}" +
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RowFilter.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RowFilter.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/RowFilter.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,84 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * This filter is used to filter based on the key. It takes an operator
+ * (equal, greater, not equal, etc) and a byte [] comparator for the row,
+ * and column qualifier portions of a key.
+ * <p>
+ * This filter can be wrapped with {@link WhileMatchFilter} to add more control.
+ * <p>
+ * Multiple filters can be combined using {@link FilterList}.
+ * <p>
+ * If an already known row range needs to be scanned, use {@link Scan} start
+ * and stop rows directly rather than a filter.
+ */
+public class RowFilter extends CompareFilter {
+
+ private boolean filterOutRow = false;
+
+ /**
+ * Writable constructor, do not use.
+ */
+ public RowFilter() {
+ super();
+ }
+
+ /**
+ * Constructor.
+ * @param rowCompareOp the compare op for row matching
+ * @param rowComparator the comparator for row matching
+ */
+ public RowFilter(final CompareOp rowCompareOp,
+ final WritableByteArrayComparable rowComparator) {
+ super(rowCompareOp, rowComparator);
+ }
+
+ @Override
+ public void reset() {
+ this.filterOutRow = false;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(KeyValue v) {
+ if(this.filterOutRow) {
+ return ReturnCode.NEXT_ROW;
+ }
+ return ReturnCode.INCLUDE;
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] data, int offset, int length) {
+ if(doCompare(this.compareOp, this.comparator, data, offset, length)) {
+ this.filterOutRow = true;
+ }
+ return this.filterOutRow;
+ }
+
+ @Override
+ public boolean filterRow() {
+ return this.filterOutRow;
+ }
+}
\ No newline at end of file
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,178 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.io.HbaseObjectWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This filter is used to filter cells based on value. It takes a
+ * {@link org.apache.hadoop.hbase.filter.CompareFilter.CompareOp}
+ * operator (equal, greater, not equal, etc), and either a byte [] value or
+ * a {@link org.apache.hadoop.hbase.filter.WritableByteArrayComparable}.
+ * <p>
+ * If we have a byte [] value then we just do a lexicographic compare. For
+ * example, if passed value is 'b' and cell has 'a' and the compare operator
+ * is LESS, then we will filter out this cell (return true). If this is not
+ * sufficient (eg you want to deserialize a long and then compare it to a fixed
+ * long value), then you can pass in your own comparator instead.
+ * <p>
+ * You must also specify a family and qualifier. Only the value of this column
+ * will be tested. All other
+ * <p>
+ * To prevent the entire row from being emitted if this filter determines the
+ * column does not pass (it should be filtered), wrap this filter with a
+ * {@link SkipFilter}.
+ * <p>
+ * To filter based on the value of all scanned columns, use {@link ValueFilter}.
+ */
+public class SingleColumnValueFilter implements Filter {
+ static final Log LOG = LogFactory.getLog(SingleColumnValueFilter.class);
+
+ private byte [] columnFamily;
+ private byte [] columnQualifier;
+ private CompareOp compareOp;
+ private WritableByteArrayComparable comparator;
+
+ /**
+ * Writable constructor, do not use.
+ */
+ public SingleColumnValueFilter() {
+ }
+
+ /**
+ * Constructor for binary compare of the value of a single column. If the
+ * column is found and the condition passes, all columns of the row will be
+ * emitted. If the column is not found or the condition fails, the row will
+ * not be emitted.
+ *
+ * @param family name of column family
+ * @param qualifier name of column qualifier
+ * @param compareOp operator
+ * @param value value to compare column values against
+ */
+ public SingleColumnValueFilter(final byte [] family, final byte [] qualifier,
+ final CompareOp compareOp, final byte[] value) {
+ this(family, qualifier, compareOp, new BinaryComparator(value));
+ }
+
+ /**
+ * Constructor for binary compare of the value of a single column. If the
+ * column is found and the condition passes, all columns of the row will be
+ * emitted. If the condition fails, the row will not be emitted.
+ * <p>
+ * Use the filterIfColumnMissing flag to set whether the rest of the columns
+ * in a row will be emitted if the specified column to check is not found in
+ * the row.
+ *
+ * @param family name of column family
+ * @param qualifier name of column qualifier
+ * @param compareOp operator
+ * @param comparator Comparator to use.
+ */
+ public SingleColumnValueFilter(final byte [] family, final byte [] qualifier,
+ final CompareOp compareOp, final WritableByteArrayComparable comparator) {
+ this.columnFamily = family;
+ this.columnQualifier = qualifier;
+ this.compareOp = compareOp;
+ this.comparator = comparator;
+ }
+
+ public boolean filterRowKey(byte[] rowKey, int offset, int length) {
+ return false;
+ }
+
+ public ReturnCode filterKeyValue(KeyValue keyValue) {
+ if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) {
+ return ReturnCode.INCLUDE;
+ }
+ if (filterColumnValue(keyValue.getBuffer(),
+ keyValue.getValueOffset(), keyValue.getValueLength())) {
+ return ReturnCode.NEXT_ROW;
+ }
+ return ReturnCode.INCLUDE;
+ }
+
+ private boolean filterColumnValue(final byte[] data, final int offset,
+ final int length) {
+ int compareResult = comparator.compareTo(Arrays.copyOfRange(data, offset,
+ offset + length));
+
+ switch (compareOp) {
+ case LESS:
+ return compareResult <= 0;
+ case LESS_OR_EQUAL:
+ return compareResult < 0;
+ case EQUAL:
+ return compareResult != 0;
+ case NOT_EQUAL:
+ return compareResult == 0;
+ case GREATER_OR_EQUAL:
+ return compareResult > 0;
+ case GREATER:
+ return compareResult >= 0;
+ default:
+ throw new RuntimeException("Unknown Compare op " + compareOp.name());
+ }
+ }
+
+ public boolean filterAllRemaining() {
+ return false;
+ }
+
+ public boolean filterRow() {
+ return false;
+ }
+
+ public void reset() {
+ }
+
+ public void readFields(final DataInput in) throws IOException {
+ this.columnFamily = Bytes.readByteArray(in);
+ if(this.columnFamily.length == 0) {
+ this.columnFamily = null;
+ }
+ this.columnQualifier = Bytes.readByteArray(in);
+ if(this.columnQualifier.length == 0) {
+ this.columnQualifier = null;
+ }
+ compareOp = CompareOp.valueOf(in.readUTF());
+ comparator = (WritableByteArrayComparable) HbaseObjectWritable.readObject(in,
+ null);
+ }
+
+ public void write(final DataOutput out) throws IOException {
+ Bytes.writeByteArray(out, this.columnFamily);
+ Bytes.writeByteArray(out, this.columnQualifier);
+ out.writeUTF(compareOp.name());
+ HbaseObjectWritable.writeObject(out, comparator,
+ WritableByteArrayComparable.class, null);
+ }
+}
Added: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SkipFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SkipFilter.java?rev=813052&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SkipFilter.java (added)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/SkipFilter.java Wed Sep 9 17:14:22 2009
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.DataInput;
+
+/**
+ * A wrapper filter that filters an entire row if any of the KeyValue checks do
+ * not pass.
+ * <p>
+ * For example, if all columns in a row represent weights of different things,
+ * with the values being the actual weights, and we want to filter out the
+ * entire row if any of its weights are zero. In this case, we want to prevent
+ * rows from being emitted if a single key is filtered. Combine this filter
+ * with a {@link ValueFilter}:
+ * <p>
+ * <pre>
+ * scan.setFilter(new SkipFilter(new ValueFilter(CompareOp.EQUAL,
+ * new BinaryComparator(Bytes.toBytes(0))));
+ * </code>
+ * Any row which contained a column whose value was 0 will be filtered out.
+ * Without this filter, the other non-zero valued columns in the row would still
+ * be emitted.
+ */
+public class SkipFilter implements Filter {
+ private boolean filterRow = false;
+ private Filter filter;
+
+ public SkipFilter() {
+ super();
+ }
+
+ public SkipFilter(Filter filter) {
+ this.filter = filter;
+ }
+
+ public void reset() {
+ filter.reset();
+ filterRow = false;
+ }
+
+ private void changeFR(boolean value) {
+ filterRow = filterRow || value;
+ }
+
+ public boolean filterRowKey(byte[] buffer, int offset, int length) {
+ return false;
+ }
+
+ public boolean filterAllRemaining() {
+ return false;
+ }
+
+ public ReturnCode filterKeyValue(KeyValue v) {
+ ReturnCode c = filter.filterKeyValue(v);
+ changeFR(c != ReturnCode.INCLUDE);
+ return c;
+ }
+
+ public boolean filterRow() {
+ return filterRow;
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(this.filter.getClass().getName());
+ this.filter.write(out);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ String className = in.readUTF();
+ try {
+ this.filter = (Filter)(Class.forName(className).newInstance());
+ this.filter.readFields(in);
+ } catch (InstantiationException e) {
+ throw new RuntimeException("Failed deserialize.", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Failed deserialize.", e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Failed deserialize.", e);
+ }
+ }
+}
Modified: hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/ValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/ValueFilter.java?rev=813052&r1=813051&r2=813052&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/ValueFilter.java (original)
+++ hadoop/hbase/branches/0.20_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/filter/ValueFilter.java Wed Sep 9 17:14:22 2009
@@ -1,5 +1,5 @@
/**
- * Copyright 2008 The Apache Software Foundation
+ * Copyright 2009 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,217 +17,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.hbase.filter;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.hbase.client.Get;
/**
- * This filter is used to filter based on the value of a given column. It takes
- * an operator (equal, greater, not equal, etc) and either a byte [] value or a
- * byte [] comparator. If we have a byte [] value then we just do a
- * lexicographic compare. For example, if passed value is 'b' and cell has 'a'
- * and the compare operator is LESS, then we will filter out this cell (return
- * true). If this is not sufficient (eg you want to deserialize
- * a long and then compare it to a fixed long value), then you can pass in your
- * own comparator instead.
- * */
-public class ValueFilter implements Filter {
- static final Log LOG = LogFactory.getLog(ValueFilter.class);
-
- /** Comparison operators. */
- public enum CompareOp {
- /** less than */
- LESS,
- /** less than or equal to */
- LESS_OR_EQUAL,
- /** equals */
- EQUAL,
- /** not equal */
- NOT_EQUAL,
- /** greater than or equal to */
- GREATER_OR_EQUAL,
- /** greater than */
- GREATER;
- }
-
- private byte [] columnFamily;
- private byte [] columnQualifier;
- private CompareOp compareOp;
- private byte [] value;
- private WritableByteArrayComparable comparator;
- private boolean filterIfColumnMissing;
-
- private boolean filterThisRow = false;
- private boolean foundColValue = false;
-
- ValueFilter() {
- // for Writable
- }
-
- /**
- * Constructor.
- *
- * @param family name of column family
- * @param qualifier name of column qualifier
- * @param compareOp operator
- * @param value value to compare column values against
- */
- public ValueFilter(final byte [] family, final byte [] qualifier,
- final CompareOp compareOp, final byte[] value) {
- this(family, qualifier, compareOp, value, true);
- }
-
- /**
- * Constructor.
- *
- * @param family name of column family
- * @param qualifier name of column qualifier
- * @param compareOp operator
- * @param value value to compare column values against
- * @param filterIfColumnMissing if true then we will filter rows that don't
- * have the column.
- */
- public ValueFilter(final byte [] family, final byte [] qualifier,
- final CompareOp compareOp,
- final byte[] value, boolean filterIfColumnMissing) {
- this.columnFamily = family;
- this.columnQualifier = qualifier;
- this.compareOp = compareOp;
- this.value = value;
- this.filterIfColumnMissing = filterIfColumnMissing;
- }
+ * This filter is used to filter based on column value. It takes an
+ * operator (equal, greater, not equal, etc) and a byte [] comparator for the
+ * cell value.
+ * <p>
+ * This filter can be wrapped with {@link WhileMatchFilter} and {@link SkipFilter}
+ * to add more control.
+ * <p>
+ * Multiple filters can be combined using {@link FilterList}.
+ * <p>
+ * To test the value of a single qualifier when scanning multiple qualifiers,
+ * use {@link SingleColumnValueFilter}.
+ */
+public class ValueFilter extends CompareFilter {
/**
- * Constructor.
- *
- * @param family name of column family
- * @param qualifier name of column qualifier
- * @param compareOp operator
- * @param comparator Comparator to use.
+ * Writable constructor, do not use.
*/
- public ValueFilter(final byte [] family, final byte [] qualifier,
- final CompareOp compareOp,
- final WritableByteArrayComparable comparator) {
- this(family, qualifier, compareOp, comparator, true);
+ public ValueFilter() {
}
/**
* Constructor.
- *
- * @param family name of column family
- * @param qualifier name of column qualifier
- * @param compareOp operator
- * @param comparator Comparator to use.
- * @param filterIfColumnMissing if true then we will filter rows that don't
- * have the column.
+ * @param valueCompareOp the compare op for column qualifier matching
+ * @param valueComparator the comparator for column qualifier matching
*/
- public ValueFilter(final byte [] family, final byte [] qualifier,
- final CompareOp compareOp,
- final WritableByteArrayComparable comparator,
- boolean filterIfColumnMissing) {
- this.columnFamily = family;
- this.columnQualifier = qualifier;
- this.compareOp = compareOp;
- this.comparator = comparator;
- this.filterIfColumnMissing = filterIfColumnMissing;
- }
-
- public boolean filterRowKey(byte[] rowKey, int offset, int length) {
- return false;
+ public ValueFilter(final CompareOp valueCompareOp,
+ final WritableByteArrayComparable valueComparator) {
+ super(valueCompareOp, valueComparator);
}
- public ReturnCode filterKeyValue(KeyValue keyValue) {
- if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) {
- return ReturnCode.INCLUDE;
- }
- this.foundColValue = true;
- boolean filtered = filterColumnValue(keyValue.getBuffer(),
- keyValue.getValueOffset(), keyValue.getValueLength());
- if (filtered) {
- this.filterThisRow = true;
- return ReturnCode.NEXT_ROW;
+ @Override
+ public ReturnCode filterKeyValue(KeyValue v) {
+ if (doCompare(this.compareOp, this.comparator, v.getBuffer(),
+ v.getValueOffset(), v.getValueLength())) {
+ return ReturnCode.SKIP;
}
return ReturnCode.INCLUDE;
}
-
- private boolean filterColumnValue(final byte[] data, final int offset,
- final int length) {
- int compareResult;
- if (comparator != null) {
- compareResult = comparator.compareTo(Arrays.copyOfRange(data, offset,
- offset + length));
- } else {
- compareResult = Bytes.compareTo(value, 0, value.length, data, offset,
- length);
- }
-
- switch (compareOp) {
- case LESS:
- return compareResult <= 0;
- case LESS_OR_EQUAL:
- return compareResult < 0;
- case EQUAL:
- return compareResult != 0;
- case NOT_EQUAL:
- return compareResult == 0;
- case GREATER_OR_EQUAL:
- return compareResult > 0;
- case GREATER:
- return compareResult >= 0;
- default:
- throw new RuntimeException("Unknown Compare op " + compareOp.name());
- }
- }
-
- public boolean filterAllRemaining() {
- return false;
- }
-
- public boolean filterRow() {
- return filterThisRow || (filterIfColumnMissing && !foundColValue);
- }
-
- public void reset() {
- filterThisRow = false;
- foundColValue = false;
- }
-
- public void readFields(final DataInput in) throws IOException {
- int valueLen = in.readInt();
- if (valueLen > 0) {
- value = new byte[valueLen];
- in.readFully(value);
- }
- this.columnFamily = Bytes.readByteArray(in);
- this.columnQualifier = Bytes.readByteArray(in);
- compareOp = CompareOp.valueOf(in.readUTF());
- comparator = (WritableByteArrayComparable) ObjectWritable.readObject(in,
- new HBaseConfiguration());
- filterIfColumnMissing = in.readBoolean();
- }
-
- public void write(final DataOutput out) throws IOException {
- if (value == null) {
- out.writeInt(0);
- } else {
- out.writeInt(value.length);
- out.write(value);
- }
- Bytes.writeByteArray(out, this.columnFamily);
- Bytes.writeByteArray(out, this.columnQualifier);
- out.writeUTF(compareOp.name());
- ObjectWritable.writeObject(out, comparator,
- WritableByteArrayComparable.class, new HBaseConfiguration());
- out.writeBoolean(filterIfColumnMissing);
- }
-}
\ No newline at end of file
+}