You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/06/05 20:18:20 UTC
svn commit: r1489990 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/client/
Author: liyin
Date: Wed Jun 5 18:18:20 2013
New Revision: 1489990
URL: http://svn.apache.org/r1489990
Log:
[HBASE-8185] Adding functionality to create ClientLocalScanner without creating hardlinks.
Author: manukranthk
Summary: This feature targets the scenarios where the user will be responsible for creating a full table snapshot and reads from the snapshot. This helps in scenarios where we need to scan from a full snapshot to keep the view of the data consistent over scanner(mapper) failures on the scan job.
Test Plan: Unit tests.
Reviewers: aaiyer, liyintang, rshroff
Reviewed By: liyintang
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D830097
Task ID: 2430669
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java Wed Jun 5 18:18:20 2013
@@ -72,6 +72,7 @@ public class ClientLocalScanner extends
*/
public static ThreadPoolExecutor scanPrefetchThreadPool;
private static int numHandlers = 20;
+ private final boolean areHardlinksCreated;
// Initializing the numHandlers statically since the thread pool can be
// reused across different scan operations on the same client.
static {
@@ -80,11 +81,13 @@ public class ClientLocalScanner extends
new DaemonThreadFactory("scan-prefetch-"));
}
- protected ClientLocalScanner(final Scan scan, HTable htable) {
+ protected ClientLocalScanner(final Scan scan, final HTable htable,
+ final boolean areHardlinksCreated) {
super(scan, htable);
// The seek + read functionality will be used in this case since
// scanning large files is faster using seek + read.
Store.isPread = false;
+ this.areHardlinksCreated = areHardlinksCreated;
}
/**
@@ -107,7 +110,7 @@ public class ClientLocalScanner extends
families = filterFamilies(families, scan);
HRegionUtilities.parallelStoreOpener(info, conf, families, tableDir,
- fs, this.stores);
+ fs, this.stores, this.areHardlinksCreated);
}
private Collection<HColumnDescriptor> filterFamilies(
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Wed Jun 5 18:18:20 2013
@@ -590,7 +590,24 @@ public class HTable implements HTableInt
}
public ResultScanner getLocalScanner(final Scan scan) throws IOException {
- ClientLocalScanner s = new ClientLocalScanner(scan, this);
+ return getLocalScanner(scan, true);
+ }
+
+ /**
+ * Create a ClientLocalScanner to scan the HTable using the scan provided.
+ *
+ * @param scan The scan object that determines the way scanning is done.
+ * @param createNewHardlinks If true, tells to create a snapshot of the store
+ * by creating new hard links.
+ * Otherwise, we assume that the table hierarchy under the root directory
+ * is not going to change and hence we read directly from it; use with caution
+ * @return
+ * @throws IOException
+ */
+ public ResultScanner getLocalScanner(final Scan scan,
+ boolean createNewHardlinks) throws IOException {
+ ClientLocalScanner s =
+ new ClientLocalScanner(scan, this, createNewHardlinks);
s.initialize();
return s;
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionUtilities.java Wed Jun 5 18:18:20 2013
@@ -75,9 +75,9 @@ public class HRegionUtilities {
// TODO(manukranthk) : Refactor HRegion code to use this common code path.
public static void parallelStoreOpener(final HRegionInfo info,
final Configuration conf, Collection<HColumnDescriptor> families,
- final Path tableDir, final FileSystem fs, Map<byte[], Store> stores)
- throws IOException{
- // initialize the thread pool for opening stores in parallel.
+ final Path tableDir, final FileSystem fs, Map<byte[], Store> stores,
+ final boolean createNewHardlinks) throws IOException{
+ // initialize the thread pool for opening stores in parallel.
ThreadPoolExecutor storeOpenerThreadPool =
StoreThreadUtils.getStoreOpenAndCloseThreadPool("StoreOpenerThread-"
+ info.getRegionNameAsString(), info, conf);
@@ -88,7 +88,8 @@ public class HRegionUtilities {
for (final HColumnDescriptor family : families) {
completionService.submit(new Callable<Store>() {
public Store call() throws IOException {
- return new ReadOnlyStore(tableDir, info, family, fs, conf);
+ return new ReadOnlyStore(tableDir, info, family,
+ fs, conf, createNewHardlinks);
}
});
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/ReadOnlyStore.java Wed Jun 5 18:18:20 2013
@@ -55,10 +55,14 @@ public class ReadOnlyStore extends Store
* @throws IOException
*/
public ReadOnlyStore(Path basedir, HRegionInfo regionInfo, HColumnDescriptor family,
- FileSystem fs, Configuration confParam)
+ FileSystem fs, Configuration confParam, final boolean createNewHardlinks)
throws IOException {
super(basedir, family, fs, confParam, null, regionInfo);
- this.storefiles = sortAndClone(loadStoreFilesWithHardLinks());
+ if (createNewHardlinks) {
+ this.storefiles = sortAndClone(loadStoreFilesWithHardLinks());
+ } else {
+ this.storefiles = sortAndClone(loadStoreFiles(this.fs.listStatus(this.homedir)));
+ }
}
private List<StoreFile> loadStoreFilesWithHardLinks() throws IOException{
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java?rev=1489990&r1=1489989&r2=1489990&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java Wed Jun 5 18:18:20 2013
@@ -36,16 +36,13 @@ import org.apache.hadoop.hbase.HBaseTest
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.InjectionEvent;
import org.apache.hadoop.hbase.util.InjectionHandler;
import org.junit.AfterClass;
@@ -58,8 +55,6 @@ public class TestClientLocalScanner {
new HBaseTestingUtility();
private static byte [] FAMILY = Bytes.toBytes("testFamily");
private static byte [] FAMILY2 = Bytes.toBytes("testFamily2");
- private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
- private static byte [] VALUE = Bytes.toBytes("testValue");
private static int SLAVES = 3;
/**
@@ -79,29 +74,6 @@ public class TestClientLocalScanner {
}
@Test
- public void testInconsistentRegionDirectories() throws IOException {
- byte [] tableName = Bytes.toBytes("testInconsistentRegionDirectories");
- String rootDir = TEST_UTIL.getConfiguration().get("hbase.rootdir");
- String tmpPath = "/tmp/testInconsistentRegionDirectories/";
- FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
- Path p = new Path(tmpPath);
- fs.mkdirs(p);
- assertTrue(fs.listStatus(p).length == 0);
- TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath);
- HTable t = TEST_UTIL.createTable(tableName, FAMILY);
- TEST_UTIL.loadTable(t, FAMILY);
- try {
- t.getLocalScanner(new Scan());
- } catch (IOException e) {
- assertTrue(fs.listStatus(p).length == 0);
- return;
- } finally {
- TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir);
- }
- assertTrue(false);
- }
-
- @Test
public void testCompareLocalScanToRemoteScan() throws IOException {
byte [] name = Bytes.toBytes("testCompareLocalScanToRemoteScan");
HTable t = TEST_UTIL.createTable(name, new byte[][] {FAMILY, FAMILY2});
@@ -114,11 +86,7 @@ public class TestClientLocalScanner {
TEST_UTIL.flush(name);
assertRowCount(t, rowCount);
- Scan scan = new Scan();
- scan.addFamily(FAMILY);
- scan.setServerPrefetching(true);
- scan.setCaching(100);
- scan.setBatch(100);
+ Scan scan = getScan(100, 100, true, FAMILY);
assertTrue(compareScanners(tmpTable.getScanner(scan),
t.getLocalScanner(scan)));
FileSystem fs = FileSystem.get(t.getConfiguration());
@@ -256,6 +224,69 @@ public class TestClientLocalScanner {
assertEquals(rowCount - endKeyCount, countGreater);
}
+ @Test
+ public void testInconsistentRegionDirectories() throws IOException {
+ byte [] tableName = Bytes.toBytes("testInconsistentRegionDirectories");
+ String rootDir = TEST_UTIL.getConfiguration().get("hbase.rootdir");
+ String tmpPath = "/tmp/testInconsistentRegionDirectories/";
+ FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+ Path p = new Path(tmpPath);
+ fs.mkdirs(p);
+ assertTrue(fs.listStatus(p).length == 0);
+ TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath);
+ HTable t = TEST_UTIL.createTable(tableName, FAMILY);
+ TEST_UTIL.loadTable(t, FAMILY);
+ try {
+ t.getLocalScanner(new Scan());
+ } catch (IOException e) {
+ assertTrue(fs.listStatus(p).length == 0);
+ return;
+ } finally {
+ TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir);
+ }
+ assertTrue(false);
+ }
+
+ @Test
+ public void testLocalScannerWithoutHardlinks() throws IOException {
+ byte [] tableName = Bytes.toBytes("testLocalScannerWithoutHardlinks");
+ HTable t = TEST_UTIL.createTable(tableName, FAMILY);
+ HTable tmpTable = new HTable(TEST_UTIL.getConfiguration(), tableName);
+ int rowCount = TEST_UTIL.loadTable(t, FAMILY);
+ t.flushCommits();
+ TEST_UTIL.flush(tableName);
+ assertRowCount(t, rowCount);
+
+ Scan scan = getScan(100, 100, true, FAMILY);
+ FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+ String hardLinkFolder = TEST_UTIL.getConfiguration().get(
+ HConstants.READ_ONLY_HARDLINKS_FOLDER,
+ HConstants.READ_ONLY_HARDLINKS_FOLDER_DEFAULT);
+ Path folder = new Path(hardLinkFolder);
+
+ ResultScanner scanner = t.getLocalScanner(scan);
+ assertTrue(fs.exists(folder));
+ assertTrue(fs.listStatus(folder).length > 0);
+ fs.delete(folder, true);
+
+ scanner = t.getLocalScanner(scan, false);
+ assertTrue(compareScanners(tmpTable.getScanner(scan),
+ scanner));
+ assertTrue(!fs.exists(folder));
+ scanner.close();
+ scanner.close();
+ }
+
+ public Scan getScan(int caching, int batching,
+ boolean serverPrefetching, byte[] family) {
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ scan.setServerPrefetching(serverPrefetching);
+ scan.setCaching(caching);
+ scan.setBatch(batching);
+ return scan;
+ }
+
/**
* Wait on table split. May return because we waited long enough on the split
* and it didn't happen. Caller should check.