You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2014/05/17 01:37:39 UTC
svn commit: r1595388 [2/2] - in /hbase/branches/hbase-10070:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/
hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated...
Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java?rev=1595388&r1=1595387&r2=1595388&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java (original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java Fri May 16 23:37:38 2014
@@ -154,6 +154,20 @@ public class HConnectionTestingUtility {
}
}
+ public static ClusterConnection getSpiedClusterConnection(final Configuration conf)
+ throws IOException {
+ HConnectionKey connectionKey = new HConnectionKey(conf);
+ synchronized (ConnectionManager.CONNECTION_INSTANCES) {
+ HConnectionImplementation connection =
+ ConnectionManager.CONNECTION_INSTANCES.get(connectionKey);
+ if (connection == null) {
+ connection = Mockito.spy(new HConnectionImplementation(conf, true));
+ ConnectionManager.CONNECTION_INSTANCES.put(connectionKey, connection);
+ }
+ return connection;
+ }
+ }
+
/**
* @return Count of extant connection instances
*/
Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java?rev=1595388&r1=1595387&r2=1595388&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java (original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestReplicasClient.java Fri May 16 23:37:38 2014
@@ -38,7 +38,10 @@ import org.apache.hadoop.hbase.coprocess
import org.apache.hadoop.hbase.protobuf.RequestConverter;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
@@ -50,9 +53,14 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -81,16 +89,44 @@ public class TestReplicasClient {
*/
public static class SlowMeCopro extends BaseRegionObserver {
static final AtomicLong sleepTime = new AtomicLong(0);
+ static final AtomicBoolean slowDownNext = new AtomicBoolean(false);
+ static final AtomicInteger countOfNext = new AtomicInteger(0);
static final AtomicReference<CountDownLatch> cdl =
new AtomicReference<CountDownLatch>(new CountDownLatch(0));
-
+ Random r = new Random();
public SlowMeCopro() {
}
@Override
public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
final Get get, final List<Cell> results) throws IOException {
+ slowdownCode(e);
+ }
+
+ @Override
+ public RegionScanner preScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> e,
+ final Scan scan, final RegionScanner s) throws IOException {
+ slowdownCode(e);
+ return s;
+ }
+
+ @Override
+ public boolean preScannerNext(final ObserverContext<RegionCoprocessorEnvironment> e,
+ final InternalScanner s, final List<Result> results,
+ final int limit, final boolean hasMore) throws IOException {
+ //this will slow down a certain next operation if the conditions are met. The slowness
+ //will allow the call to go to a replica
+ if (slowDownNext.get()) {
+ //have some "next" return successfully from the primary; hence countOfNext checked
+ if (countOfNext.incrementAndGet() == 2) {
+ sleepTime.set(2000);
+ slowdownCode(e);
+ }
+ }
+ return true;
+ }
+ private void slowdownCode(final ObserverContext<RegionCoprocessorEnvironment> e) {
if (e.getEnvironment().getRegion().getRegionInfo().getReplicaId() == 0) {
CountDownLatch latch = cdl.get();
try {
@@ -118,7 +154,7 @@ public class TestReplicasClient {
// enable store file refreshing
HTU.getConfiguration().setInt(
StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);
-
+ HTU.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
HTU.startMiniCluster(NB_SERVERS);
// Create table then get the single region for our new table.
@@ -158,6 +194,14 @@ public class TestReplicasClient {
@Before
public void before() throws IOException {
HTU.getHBaseAdmin().getConnection().clearRegionCache();
+ try {
+ openRegion(hriPrimary);
+ } catch (Exception ignored) {
+ }
+ try {
+ openRegion(hriSecondary);
+ } catch (Exception ignored) {
+ }
}
@After
@@ -166,6 +210,10 @@ public class TestReplicasClient {
closeRegion(hriSecondary);
} catch (Exception ignored) {
}
+ try {
+ closeRegion(hriPrimary);
+ } catch (Exception ignored) {
+ }
ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary);
ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriSecondary);
@@ -177,6 +225,9 @@ public class TestReplicasClient {
}
private void openRegion(HRegionInfo hri) throws Exception {
+ try {
+ if (isRegionOpened(hri)) return;
+ } catch (Exception e){}
ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName());
// first version is '0'
AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(hri, 0, null);
@@ -210,6 +261,10 @@ public class TestReplicasClient {
ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), null));
}
+ private boolean isRegionOpened(HRegionInfo hri) throws Exception {
+ return getRS().getRegionByEncodedName(hri.getEncodedName()).isAvailable();
+ }
+
private void checkRegionIsClosed(String encodedRegionName) throws Exception {
while (!getRS().getRegionsInTransitionInRS().isEmpty()) {
@@ -463,4 +518,106 @@ public class TestReplicasClient {
closeRegion(hriSecondary);
}
}
-}
\ No newline at end of file
+
+ @Test
+ public void testScanWithReplicas() throws Exception {
+ //simple scan
+ runMultipleScansOfOneType(false, false);
+ }
+
+ @Test
+ public void testSmallScanWithReplicas() throws Exception {
+ //small scan
+ runMultipleScansOfOneType(false, true);
+ }
+
+ @Test
+ public void testReverseScanWithReplicas() throws Exception {
+ //reverse scan
+ runMultipleScansOfOneType(true, false);
+ }
+
+ private void runMultipleScansOfOneType(boolean reversed, boolean small) throws Exception {
+ openRegion(hriSecondary);
+ int NUMROWS = 100;
+ try {
+ for (int i = 0; i < NUMROWS; i++) {
+ byte[] b1 = Bytes.toBytes("testUseRegionWithReplica" + i);
+ Put p = new Put(b1);
+ p.add(f, b1, b1);
+ table.put(p);
+ }
+ LOG.debug("PUT done");
+ int caching = 20;
+ byte[] start;
+ if (reversed) start = Bytes.toBytes("testUseRegionWithReplica" + (NUMROWS - 1));
+ else start = Bytes.toBytes("testUseRegionWithReplica" + 0);
+
+ scanWithReplicas(reversed, small, Consistency.TIMELINE, caching, start, NUMROWS, false, false);
+
+ //Even if we were to slow the server down, unless we ask for stale
+ //we won't get it
+ SlowMeCopro.sleepTime.set(5000);
+ scanWithReplicas(reversed, small, Consistency.STRONG, caching, start, NUMROWS, false, false);
+ SlowMeCopro.sleepTime.set(0);
+
+ HTU.getHBaseAdmin().flush(table.getTableName());
+ LOG.info("flush done");
+ Thread.sleep(1000 + REFRESH_PERIOD * 2);
+
+ //Now set the flag to get a response even if stale
+ SlowMeCopro.sleepTime.set(5000);
+ scanWithReplicas(reversed, small, Consistency.TIMELINE, caching, start, NUMROWS, true, false);
+ SlowMeCopro.sleepTime.set(0);
+
+ // now make some 'next' calls slow
+ SlowMeCopro.slowDownNext.set(true);
+ SlowMeCopro.countOfNext.set(0);
+ scanWithReplicas(reversed, small, Consistency.TIMELINE, caching, start, NUMROWS, true, true);
+ SlowMeCopro.slowDownNext.set(false);
+ SlowMeCopro.countOfNext.set(0);
+ } finally {
+ SlowMeCopro.cdl.get().countDown();
+ SlowMeCopro.sleepTime.set(0);
+ SlowMeCopro.slowDownNext.set(false);
+ SlowMeCopro.countOfNext.set(0);
+ for (int i = 0; i < NUMROWS; i++) {
+ byte[] b1 = Bytes.toBytes("testUseRegionWithReplica" + i);
+ Delete d = new Delete(b1);
+ table.delete(d);
+ }
+ closeRegion(hriSecondary);
+ }
+ }
+
+ private void scanWithReplicas(boolean reversed, boolean small, Consistency consistency,
+ int caching, byte[] startRow, int numRows, boolean staleExpected, boolean slowNext)
+ throws Exception {
+ Scan scan = new Scan(startRow);
+ scan.setCaching(caching);
+ scan.setReversed(reversed);
+ scan.setSmall(small);
+ scan.setConsistency(consistency);
+ ResultScanner scanner = table.getScanner(scan);
+ Iterator<Result> iter = scanner.iterator();
+ HashMap<String, Boolean> map = new HashMap<String, Boolean>();
+ int count = 0;
+ int countOfStale = 0;
+ while (iter.hasNext()) {
+ count++;
+ Result r = iter.next();
+ if (map.containsKey(new String(r.getRow()))) {
+ throw new Exception("Unexpected scan result. Repeated row " + Bytes.toString(r.getRow()));
+ }
+ map.put(new String(r.getRow()), true);
+ if (!slowNext) Assert.assertTrue(r.isStale() == staleExpected);
+ if (r.isStale()) countOfStale++;
+ }
+ LOG.debug("Count of rows " + count + " num rows expected " + numRows);
+ Assert.assertTrue(count == numRows);
+ if (slowNext) {
+ LOG.debug("Count of Stale " + countOfStale);
+ Assert.assertTrue(countOfStale > 1 && countOfStale < numRows);
+ }
+ }
+}
Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java?rev=1595388&r1=1595387&r2=1595388&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java (original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java Fri May 16 23:37:38 2014
@@ -196,7 +196,6 @@ public class TestRestoreSnapshotFromClie
assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
Set<String> fsFamilies = getFamiliesFromFS(tableName);
assertEquals(2, fsFamilies.size());
- table.close();
// Take a snapshot
admin.disableTable(tableName);
@@ -217,7 +216,6 @@ public class TestRestoreSnapshotFromClie
assertEquals(1, htd.getFamilies().size());
fsFamilies = getFamiliesFromFS(tableName);
assertEquals(1, fsFamilies.size());
- table.close();
// Restore back the snapshot (with the cf)
admin.disableTable(tableName);
Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java?rev=1595388&r1=1595387&r2=1595388&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java (original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java Fri May 16 23:37:38 2014
@@ -735,7 +735,7 @@ public class TestVisibilityLabels {
table.put(puts);
} finally {
if (table != null) {
- table.close();
+ table.flushCommits();
}
}
return table;