You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/11/05 20:18:24 UTC
svn commit: r1539104 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/reg...
Author: liyin
Date: Tue Nov 5 19:18:24 2013
New Revision: 1539104
URL: http://svn.apache.org/r1539104
Log:
[HBASE-8185] Port the morse fix to trunk.
Author: manukranthk
Summary: Disabling the MVCC while doing a client side scan by passing the appropriate boolean. Fixed on the morse installation : https://phabricator.fb.com/D979753.
Test Plan: Tested on Morse. Observed that this prevents the code taking the MVCC code path.
Reviewers: aaiyer, rshroff, liyintang
Reviewed By: liyintang
CC: hbase-eng@, san
Differential Revision: https://phabricator.fb.com/D1014416
Task ID: 2966841
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Nov 5 19:18:24 2013
@@ -898,6 +898,9 @@ public final class HConstants {
public static final int DEFAULT_CLIENT_LOCAL_SCANNER_MAX_WAITTIME_FOR_FLUSH_MS
= 10000; // ms
+ public static final String CLIENT_SIDE_SCAN = "hbase.client.side.scan";
+ public static final boolean DEFAULT_CLIENT_SIDE_SCAN = false;
+
private HConstants() {
// Can't be instantiated with this constructor.
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java Tue Nov 5 19:18:24 2013
@@ -115,6 +115,7 @@ public class ClientLocalScanner extends
this.currentRegion.getTableDesc().setReadOnly(true);
HTable table = this.htable;
final Configuration conf = table.getConfiguration();
+ conf.setBoolean(HConstants.CLIENT_SIDE_SCAN, true);
boolean flushAndWait = conf.getBoolean(
HConstants.CLIENT_LOCAL_SCANNER_FLUSH_AND_WAIT,
HConstants.DEFAULT_CLIENT_LOCAL_SCANNER_FLUSH_AND_WAIT);
@@ -134,6 +135,7 @@ public class ClientLocalScanner extends
HRegionUtilities.parallelStoreOpener(info, conf, families, tableDir,
fs, this.stores, this.areHardlinksCreated);
+ HConnectionManager.deleteAllZookeeperConnections();
}
private Collection<HColumnDescriptor> filterFamilies(
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Nov 5 19:18:24 2013
@@ -966,9 +966,15 @@ public class Store extends SchemaConfigu
// TODO this used to get the store files in descending order,
// but now we get them in ascending order, which I think is
// actually more correct, since memstore get put at the end.
+
+ boolean isClientSideScanOrCompaction =
+ this.conf.getBoolean(HConstants.CLIENT_SIDE_SCAN,
+ HConstants.DEFAULT_CLIENT_SIDE_SCAN) || isCompaction;
+ // useMVCC = !isClientSideScanOrCompaction
+
List<StoreFileScanner> sfScanners =
StoreFileScanner.getScannersForStoreFiles(storeFiles, cacheBlocks,
- isCompaction, preloadBlocks, matcher);
+ isCompaction, preloadBlocks, matcher, isClientSideScanOrCompaction);
List<KeyValueScanner> scanners =
new ArrayList<KeyValueScanner>(sfScanners.size()+1);
scanners.addAll(sfScanners);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Nov 5 19:18:24 2013
@@ -1297,7 +1297,7 @@ public class StoreFile extends SchemaCon
* @return a scanner
*/
public StoreFileScanner getStoreFileScanner(boolean cacheBlocks) {
- return getStoreFileScanner(cacheBlocks, false, false);
+ return getStoreFileScanner(cacheBlocks, false, false, false);
}
/**
@@ -1310,9 +1310,9 @@ public class StoreFile extends SchemaCon
* @return a scanner
*/
public StoreFileScanner getStoreFileScanner(boolean cacheBlocks,
- boolean isCompaction, boolean preloadBlocks) {
+ boolean isCompaction, boolean preloadBlocks, boolean isClientSideScan) {
return new StoreFileScanner(this, getScanner(cacheBlocks, isCompaction,
- preloadBlocks), !isCompaction);
+ preloadBlocks), !isClientSideScan);
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Tue Nov 5 19:18:24 2013
@@ -91,7 +91,7 @@ public class StoreFileScanner implements
Collection<StoreFile> files, boolean cacheBlocks, boolean isCompaction)
throws IOException {
return getScannersForStoreFiles(files, cacheBlocks, isCompaction,
- false, null);
+ false, null, isCompaction);
}
/**
@@ -101,13 +101,15 @@ public class StoreFileScanner implements
*/
public static List<StoreFileScanner> getScannersForStoreFiles(
Collection<StoreFile> files, boolean cacheBlocks, boolean isCompaction,
- boolean preloadBlocks, ScanQueryMatcher matcher) throws IOException {
+ boolean preloadBlocks, ScanQueryMatcher matcher, boolean isClientSideScan)
+ throws IOException {
List<StoreFileScanner> scanners = new ArrayList<StoreFileScanner>(
files.size());
for (StoreFile file : files) {
StoreFile.Reader r = file.createReader();
StoreFileScanner scanner =
- r.getStoreFileScanner(cacheBlocks, isCompaction, preloadBlocks);
+ r.getStoreFileScanner(cacheBlocks, isCompaction, preloadBlocks,
+ isClientSideScan);
scanner.setScanQueryMatcher(matcher);
scanners.add(scanner);
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Tue Nov 5 19:18:24 2013
@@ -268,9 +268,9 @@ public class StoreScanner extends NonLaz
* Get a filtered list of scanners. Assumes we are not in a compaction.
* @return list of scanners to seek
*/
- private List<KeyValueScanner> getScannersNoCompaction() throws IOException {
- final boolean isCompaction = false;
- return selectScannersFrom(store.getScanners(cacheBlocks, isCompaction,
+ private List<KeyValueScanner> getScannersNoCompaction()
+ throws IOException {
+ return selectScannersFrom(store.getScanners(cacheBlocks, false,
scan.isPreloadBlocks(), matcher));
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Tue Nov 5 19:18:24 2013
@@ -1330,7 +1330,7 @@ REGION_LOOP:
}
s.close();
// If I get to here and all rows have a Server, then all have been assigned.
- if (rows == countOfRegions)
+ if (rows >= countOfRegions)
break;
LOG.info("Found " + rows + " open regions, waiting for " +
countOfRegions);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java Tue Nov 5 19:18:24 2013
@@ -589,7 +589,8 @@ public class DataBlockEncodingTool {
StoreFile.Reader reader = hsf.createReader();
reader.loadFileInfo();
- KeyValueScanner scanner = reader.getStoreFileScanner(true, true, false);
+ KeyValueScanner scanner = reader.getStoreFileScanner(true, true, false,
+ false);
// run the utilities
DataBlockEncodingTool comp = new DataBlockEncodingTool(compressionName);
@@ -715,4 +716,4 @@ public class DataBlockEncodingTool {
}
}
-}
\ No newline at end of file
+}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java Tue Nov 5 19:18:24 2013
@@ -66,7 +66,8 @@ public class EncodedSeekPerformanceTest
NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader reader = storeFile.createReader();
- StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false,
+ false);
KeyValue current;
scanner.seek(KeyValue.LOWESTKEY);
@@ -99,7 +100,8 @@ public class EncodedSeekPerformanceTest
long totalSize = 0;
StoreFile.Reader reader = storeFile.createReader();
- StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false,
+ false);
long startReadingTime = System.nanoTime();
KeyValue current;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java Tue Nov 5 19:18:24 2013
@@ -590,7 +590,8 @@ public class HFileReadWriteTest {
public Boolean call() throws Exception {
Thread.currentThread().setName("reader " + readerId);
Random rand = new Random();
- StoreFileScanner scanner = reader.getStoreFileScanner(true, pread, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(true, pread, false,
+ false);
while (System.currentTimeMillis() < endTime) {
byte[] row = createRandomRow(rand, firstRow, lastRow);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Tue Nov 5 19:18:24 2013
@@ -198,7 +198,7 @@ public class TestCompoundBloomFilter {
NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader r = sf.createReader();
final boolean pread = true; // does not really matter
- StoreFileScanner scanner = r.getStoreFileScanner(true, pread, false);
+ StoreFileScanner scanner = r.getStoreFileScanner(true, pread, false, false);
{
// Test for false negatives (not allowed).
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1539104&r1=1539103&r2=1539104&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Tue Nov 5 19:18:24 2013
@@ -376,7 +376,7 @@ public class TestStoreFile extends HBase
DataBlockEncoding.NONE);
reader.loadFileInfo();
reader.loadBloomfilter();
- StoreFileScanner scanner = reader.getStoreFileScanner(false, false, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(false, false, false, false);
// check false positives rate
int falsePos = 0;
@@ -525,7 +525,8 @@ public class TestStoreFile extends HBase
DataBlockEncoding.NONE);
reader.loadFileInfo();
reader.loadBloomfilter();
- StoreFileScanner scanner = reader.getStoreFileScanner(true, true, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(true, true, false,
+ false);
assertEquals(expKeys[x], reader.generalBloomFilter.getKeyCount());
// check false positives rate
@@ -706,7 +707,8 @@ public class TestStoreFile extends HBase
StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader reader = hsf.createReader();
- StoreFileScanner scanner = reader.getStoreFileScanner(false, false, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(false, false, false,
+ false);
TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
columns.add(qualifier);
@@ -758,7 +760,8 @@ public class TestStoreFile extends HBase
// Read this file, we should see 3 misses
StoreFile.Reader reader = hsf.createReader();
reader.loadFileInfo();
- StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false);
+ StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false,
+ false);
scanner.seek(KeyValue.LOWESTKEY);
while (scanner.next() != null);
assertEquals(startHit, cs.getHitCount());
@@ -780,7 +783,7 @@ public class TestStoreFile extends HBase
// Read this file, we should see 3 hits
reader = hsf.createReader();
- scanner = reader.getStoreFileScanner(true, false, false);
+ scanner = reader.getStoreFileScanner(true, false, false, false);
scanner.seek(KeyValue.LOWESTKEY);
while (scanner.next() != null);
assertEquals(startHit + 3, cs.getHitCount());
@@ -796,13 +799,14 @@ public class TestStoreFile extends HBase
StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader readerOne = hsf.createReader();
readerOne.loadFileInfo();
- StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, false, false);
+ StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, false,
+ false, false);
scannerOne.seek(KeyValue.LOWESTKEY);
hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader readerTwo = hsf.createReader();
readerTwo.loadFileInfo();
- StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, false, false);
+ StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, false, false, false);
scannerTwo.seek(KeyValue.LOWESTKEY);
KeyValue kv1 = null;
KeyValue kv2 = null;