You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/02/14 20:18:31 UTC
svn commit: r1568492 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/util/
test/java/org/apache/hadoop/hbase/regionserver/
Author: liyin
Date: Fri Feb 14 19:18:30 2014
New Revision: 1568492
URL: http://svn.apache.org/r1568492
Log:
[HBASE-10538] Rewrite code related to GetScan
Author: daviddeng
Summary:
Instead of setting stopRow equals to startRow, stopRow is set to the
next value greater than startRow, i.e. with an extra trailing ZERO.
Test Plan: TestScanner
Reviewers: liyintang, manukranthk, gauravm, fan
Reviewed By: liyintang
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1174409
Task ID: 3732722
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/Scan.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/Scan.java?rev=1568492&r1=1568491&r2=1568492&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/Scan.java Fri Feb 14 19:18:30 2014
@@ -20,16 +20,6 @@
package org.apache.hadoop.hbase.client;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
-import org.apache.hadoop.hbase.io.TimeRange;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -41,6 +31,16 @@ import java.util.NavigableSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+
/**
* Used to perform Scan operations.
* <p>
@@ -184,7 +184,7 @@ public class Scan extends Operation impl
*/
public Scan(Get get) {
this.startRow = get.getRow();
- this.stopRow = get.getRow();
+ this.stopRow = Bytes.nextOf(get.getRow());
this.filter = get.getFilter();
this.maxVersions = get.getMaxVersions();
this.storeLimit = get.getMaxResultsPerColumnFamily();
@@ -195,8 +195,7 @@ public class Scan extends Operation impl
}
public boolean isGetScan() {
- return this.startRow != null && this.startRow.length > 0 &&
- Bytes.equals(this.startRow, this.stopRow);
+ return Bytes.isNext(startRow, stopRow);
}
/**
@@ -225,7 +224,7 @@ public class Scan extends Operation impl
if (set == null) {
set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
}
-
+
if (qualifier == null) {
set.add(HConstants.EMPTY_BYTE_ARRAY);
} else {
@@ -261,7 +260,7 @@ public class Scan extends Operation impl
public boolean isPreloadBlocks() {
return preloadBlocks;
}
-
+
/**
* Set whether this scanner should preload data blocks or not.
* @param value
@@ -380,23 +379,23 @@ public class Scan extends Operation impl
}
/**
- * This is technically not the max available memory setting, more of a hint.
- * We will add KV's till we exceed this setting if partialRow is true,
+ * This is technically not the max available memory setting, more of a hint.
+ * We will add KV's till we exceed this setting if partialRow is true,
* and add entire rows till we exceed this setting if partialRow is false.
* !!!NOTE!!!: this call will overwrite the caching setting and set it as
* int.max_value. If you really want row-based constraint as well, use
* setCaching(int caching), which will reset maxResponseSize to match your
- * configuration and disable partial row.
+ * configuration and disable partial row.
*/
public void setCaching(int responseSize, boolean partialRow) {
this.maxResponseSize = responseSize;
this.partialRow = partialRow;
- this.caching = Integer.MAX_VALUE;
+ this.caching = Integer.MAX_VALUE;
}
/**
- * Set if pre-fetching is enabled on the region server. If enabled, the
- * region server will try to read the next scan result ahead of time. This
+ * Set if pre-fetching is enabled on the region server. If enabled, the
+ * region server will try to read the next scan result ahead of time. This
* improves scan performance if we are doing large scans.
* @param enablePrefetching if pre-fetching is enabled or not
*/
@@ -407,7 +406,7 @@ public class Scan extends Operation impl
public boolean getServerPrefetching() {
return serverPrefetching;
}
-
+
/**
* @return maximum response size that client can handle for a single call to next()
*/
@@ -423,9 +422,9 @@ public class Scan extends Operation impl
}
/**
- * Set currentPartialResponseSize to accumulated response size
+ * Set currentPartialResponseSize to accumulated response size
* for all the KeyValue pairs collected so far. This is only used at
- * server side, and not used as a client API.
+ * server side, and not used as a client API.
* @param responseSize
*/
public void setCurrentPartialResponseSize(int responseSize) {
@@ -433,8 +432,8 @@ public class Scan extends Operation impl
}
/*
- * Get current PartialResponseSize. This is only used at server side,
- * and not used as a client API.
+ * Get current PartialResponseSize. This is only used at server side,
+ * and not used as a client API.
*/
public int getCurrentPartialResponseSize() {
return this.currentPartialResponseSize;
@@ -690,6 +689,7 @@ public class Scan extends Operation impl
}
//Writable
+ @Override
public void readFields(final DataInput in)
throws IOException {
int version = in.readByte();
@@ -742,6 +742,7 @@ public class Scan extends Operation impl
}
}
+ @Override
public void write(final DataOutput out)
throws IOException {
// We try to talk a protocol version as low as possible so that we can be
@@ -823,6 +824,7 @@ public class Scan extends Operation impl
* @throws IllegalArgumentException When the colon is missing.
* @deprecated use {@link #addColumn(byte[], byte[])} instead
*/
+ @Deprecated
public Scan addColumn(byte[] familyAndQualifier) {
byte [][] fq = KeyValue.parseColumn(familyAndQualifier);
if (fq.length > 1 && fq[1] != null && fq[1].length > 0) {
@@ -842,6 +844,7 @@ public class Scan extends Operation impl
* @deprecated issue multiple {@link #addColumn(byte[], byte[])} instead
* @return this
*/
+ @Deprecated
public Scan addColumns(byte [][] columns) {
for (byte[] column : columns) {
addColumn(column);
@@ -859,6 +862,7 @@ public class Scan extends Operation impl
* @return A reference to this instance.
* @deprecated use {@link #addColumn(byte[], byte[])} instead
*/
+ @Deprecated
public Scan addColumns(String columns) {
String[] cols = columns.split(" ");
for (String col : cols) {
@@ -875,6 +879,7 @@ public class Scan extends Operation impl
* @return The columns in an old style string format.
* @deprecated
*/
+ @Deprecated
public String getInputColumns() {
StringBuilder cols = new StringBuilder("");
for (Map.Entry<byte[], NavigableSet<byte[]>> e :
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java?rev=1568492&r1=1568491&r2=1568492&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java Fri Feb 14 19:18:30 2014
@@ -59,7 +59,6 @@ public class RegionScanner implements In
private final byte [] stopRow;
private Filter filter;
private final int batch;
- private int isScan;
private boolean filterClosed = false;
private long readPt;
private Scan originalScan;
@@ -96,9 +95,6 @@ public class RegionScanner implements In
} else {
this.stopRow = scan.getStopRow();
}
- // If we are doing a get, we want to be [startRow,endRow] normally
- // it is [startRow,endRow) and if startRow=endRow we get nothing.
- this.isScan = scan.isGetScan() ? -1 : 0;
// synchronize on scannerReadPoints so that nobody calculates
// getSmallestReadPoint, before scannerReadPoints is updated.
@@ -207,6 +203,7 @@ public class RegionScanner implements In
this.metric = metric;
}
+ @Override
public ScanResult call() {
ScanResult scanResult = null;
List<Result> outResults = new ArrayList<Result>();
@@ -336,7 +333,7 @@ public class RegionScanner implements In
}
return returnResult;
}
-
+
@Override
public boolean next(List<KeyValue> outResults)
throws IOException {
@@ -458,10 +455,14 @@ public class RegionScanner implements In
}
private boolean isStopRow(byte [] currentRow) {
- return currentRow == null ||
- (stopRow != null &&
- comparator.compareRows(stopRow, 0, stopRow.length,
- currentRow, 0, currentRow.length) <= isScan);
+ if (currentRow == null) {
+ return true;
+ }
+ if (stopRow == null) {
+ return false;
+ }
+ return comparator.compareRows(stopRow, 0, stopRow.length, currentRow, 0,
+ currentRow.length) <= 0;
}
@Override
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Bytes.java?rev=1568492&r1=1568491&r2=1568492&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Bytes.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/Bytes.java Fri Feb 14 19:18:30 2014
@@ -104,9 +104,11 @@ public class Bytes {
public ByteArrayComparator() {
super();
}
+ @Override
public int compare(byte [] left, byte [] right) {
return compareTo(left, right);
}
+ @Override
public int compare(byte [] b1, int s1, int l1, byte [] b2, int s2, int l2) {
return compareTo(b1, s1, l1, b2, s2, l2);
}
@@ -1496,4 +1498,35 @@ public class Bytes {
System.arraycopy(arr, 0, tmp, 0, arr.length);
return tmp;
}
+
+ /**
+ * Returns a byte array next to a given one, i.e. it is the smallest byte
+ * array among all byte arrays that is strictly greater than the give array.
+ * Greater and smaller are defined by Bytes.compareTo.
+ *
+ * @param b
+ * the give array
+ */
+ public static byte[] nextOf(byte[] b) {
+ byte[] res = new byte[b.length + 1];
+ System.arraycopy(b, 0, res, 0, b.length);
+ return res;
+ }
+
+ /**
+ * Return whether b equals nextOf(a)
+ */
+ public static boolean isNext(byte[] a, byte[] b) {
+ if (a == null || b == null) {
+ return false;
+ }
+
+ if (b.length != a.length + 1) {
+ return false;
+ }
+ if (b[a.length] != 0) {
+ return false;
+ }
+ return Bytes.compareTo(a, 0, a.length, b, 0, a.length) == 0;
+ }
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=1568492&r1=1568491&r2=1568492&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanner.java Fri Feb 14 19:18:30 2014
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.filter.Fi
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -521,6 +520,7 @@ public class TestScanner extends HBaseTe
if (flushIndex == count) {
LOG.info("Starting flush at flush index " + flushIndex);
Thread t = new Thread() {
+ @Override
public void run() {
try {
hri.flushcache();
@@ -543,4 +543,34 @@ public class TestScanner extends HBaseTe
LOG.info("Found " + count + " items");
return count;
}
+
+ public void testGetScanner() throws Exception {
+ String ROW_STR = "aaa";
+ byte[] ROW = Bytes.toBytes(ROW_STR);
+ try {
+ this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
+ addContent(this.r, HConstants.CATALOG_FAMILY);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ // Do simple test of getting one row only first.
+ Scan scan = new Scan(new Get(ROW).addFamily(HConstants.CATALOG_FAMILY));
+
+ assertTrue("isGetScan", scan.isGetScan());
+
+ InternalScanner s = r.getScanner(scan);
+ int count = 0;
+ while (s.next(results)) {
+ count++;
+ }
+ s.close();
+ assertEquals("Number of rows returned", 1, count + 1);
+ assertTrue("Results should not be empty", results.size() > 0);
+ for (KeyValue kv : results) {
+ assertEquals("Row", ROW_STR, Bytes.toString(kv.getRow()));
+ }
+ } finally {
+ this.r.close();
+ this.r.getLog().closeAndDelete();
+ shutdownDfs(this.cluster);
+ }
+ }
}