You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/03/14 19:09:49 UTC
svn commit: r1577640 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/ipc/thrift/
test/java/org/apache/hadoop/hbase/client/
Author: liyin
Date: Fri Mar 14 18:09:49 2014
New Revision: 1577640
URL: http://svn.apache.org/r1577640
Log:
[HBASE-9930] Fixing conf dependencies of ClientSideScan and fixing TestClientLocalScanner unit tests.
Author: manukranthk
Summary:
This diff achieves to do the following :
* Fix TestClientLocalScanner
* Remove conf dependencies from ClientLocalScanner
Test Plan: Run TestClientLocalScanner
Reviewers: gauravm, fan
Reviewed By: gauravm
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1217856
Task ID: 3667016, 3274463
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1577640&r1=1577639&r2=1577640&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Fri Mar 14 18:09:49 2014
@@ -1058,6 +1058,9 @@ public final class HConstants {
public static final String THRIFT_HEADER_FROM_CLIENT = "clientHeader";
public static final boolean DISABLE_THRIFT_REGION_INFO_QUALIFIER = false;
+ public static final String USE_CONF_FROM_SERVER =
+ "hbase.client.use.conf.from.server";
+ public static final boolean DEFAULT_USE_CONF_FROM_SERVER = true;
private HConstants() {
// Can't be instantiated with this constructor.
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java?rev=1577640&r1=1577639&r2=1577640&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/ClientLocalScanner.java Fri Mar 14 18:09:49 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ipc.thrift.HBaseToThriftAdapter;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionUtilities;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -123,6 +124,15 @@ public class ClientLocalScanner extends
flushRegionAndWaitForFlush(this.currentRegion,
table.getRegionsInfo().get(info));
}
+
+ if (conf.getBoolean(HConstants.USE_CONF_FROM_SERVER,
+ HConstants.DEFAULT_USE_CONF_FROM_SERVER)) {
+ conf.set("fs.default.name",
+ this.htable.getServerConfProperty("fs.default.name"));
+ if (this.areHardlinksCreated) {
+ conf.set("hbase.rootdir", htable.getServerConfProperty("hbase.rootdir"));
+ }
+ }
Path rootDir = FSUtils.getRootDir(conf);
final Path tableDir =
HTableDescriptor.getTableDir(rootDir, info.getTableDesc().getName());
@@ -210,7 +220,8 @@ public class ClientLocalScanner extends
public Result[] nextOnRS(int nbRows) throws IOException {
try {
RegionScanner s = (RegionScanner)this.currentScanner;
- return s.nextRows(nbRows, HRegion.METRIC_NEXTSIZE);
+ Result[] ret = s.nextRows(nbRows, HRegion.METRIC_NEXTSIZE);
+ return HBaseToThriftAdapter.validateResults(ret);
} catch (IOException e) {
close();
throw e;
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java?rev=1577640&r1=1577639&r2=1577640&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/ipc/thrift/HBaseToThriftAdapter.java Fri Mar 14 18:09:49 2014
@@ -839,7 +839,7 @@ public class HBaseToThriftAdapter implem
* @param res
* @return
*/
- private Result[] validateResults(Result[] res) {
+ public static Result[] validateResults(Result[] res) {
if (res == null) return null;
if (res.length == 1) {
return res[0].isSentinelResult() ? null : res;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java?rev=1577640&r1=1577639&r2=1577640&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestClientLocalScanner.java Fri Mar 14 18:09:49 2014
@@ -87,7 +87,7 @@ public class TestClientLocalScanner {
TEST_UTIL.loadTable(t, FAMILY2);
t.flushCommits();
TEST_UTIL.flush(name);
- TEST_UTIL.assertRowCount(t, rowCount);
+ HBaseTestingUtility.assertRowCount(t, rowCount);
Scan scan = getScan(100, 100, true, FAMILY);
assertTrue(compareScanners(tmpTable.getScanner(scan),
t.getLocalScanner(scan)));
@@ -179,10 +179,10 @@ public class TestClientLocalScanner {
t.flushCommits();
TEST_UTIL.flush(name);
- TEST_UTIL.assertRowCount(t, rowCount);
+ HBaseTestingUtility.assertRowCount(t, rowCount);
// Split the table. Should split on a reasonable key; 'lqj'
Map<HRegionInfo, HServerAddress> regions = splitTable(t);
- TEST_UTIL.assertRowCount(t, rowCount);
+ HBaseTestingUtility.assertRowCount(t, rowCount);
// Get end key of first region.
byte [] endKey = regions.keySet().iterator().next().getEndKey();
// Count rows with a filter that stops us before passed 'endKey'.
@@ -199,6 +199,11 @@ public class TestClientLocalScanner {
// New test. Make it so scan goes into next region by one and then two.
// Make sure count comes out right.
byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
+ String defaultName = t.getConfiguration().get("fs.default.name");
+ String rootdir = t.getConfiguration().get("hbase.rootdir");
+
+ t.getConfiguration().set("fs.default.name", "");
+ t.getConfiguration().set("hbase.rootdir", "");
int plusOneCount = countRows(t, t.getLocalScanner(createScanWithRowFilter(key)));
assertEquals(endKeyCount + 1, plusOneCount);
key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
@@ -224,6 +229,10 @@ public class TestClientLocalScanner {
countGreater = countRows(t, t.getLocalScanner(createScanWithRowFilter(endKey, endKey,
CompareFilter.CompareOp.GREATER_OR_EQUAL)));
assertEquals(rowCount - endKeyCount, countGreater);
+
+ t.getConfiguration().set("fs.default.name", defaultName);
+ t.getConfiguration().set("hbase.rootdir", rootdir);
+ t.close();
}
@Test
@@ -237,14 +246,18 @@ public class TestClientLocalScanner {
assertTrue(fs.listStatus(p).length == 0);
TEST_UTIL.getConfiguration().set("hbase.rootdir", tmpPath);
HTable t = TEST_UTIL.createTable(tableName, FAMILY);
+ t.getConfiguration().setBoolean(HConstants.USE_CONF_FROM_SERVER, false);
TEST_UTIL.loadTable(t, FAMILY);
try {
t.getLocalScanner(new Scan());
- } catch (IOException e) {
+ } catch (Exception e) {
+ LOG.debug("Exception", e);
assertTrue(fs.listStatus(p).length == 0);
return;
} finally {
TEST_UTIL.getConfiguration().set("hbase.rootdir", rootDir);
+ t.getConfiguration().setBoolean(HConstants.USE_CONF_FROM_SERVER, false);
+ t.close();
}
assertTrue(false);
}
@@ -257,7 +270,7 @@ public class TestClientLocalScanner {
int rowCount = TEST_UTIL.loadTable(t, FAMILY);
t.flushCommits();
TEST_UTIL.flush(tableName);
- TEST_UTIL.assertRowCount(t, rowCount);
+ HBaseTestingUtility.assertRowCount(t, rowCount);
Scan scan = getScan(100, 100, true, FAMILY);
FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
@@ -289,6 +302,8 @@ public class TestClientLocalScanner {
assertTrue(fs.exists(homedir));
assertTrue(fs.listStatus(homedir).length == files);
+ t.close();
+ fs.close();
}
public Scan getScan(int caching, int batching,