You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2014/03/11 21:31:01 UTC
git commit: ACCUMULO-2455 small work arounds to increase the chance
that zookeeper updates are in place before tservers need to see them
Repository: accumulo
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 46bf772d6 -> 09307322a
ACCUMULO-2455 small work arounds to increase the chance that zookeeper updates are in place before tservers need to see them
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/09307322
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/09307322
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/09307322
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 09307322a3e9f45cd37bbd4c63fa66ccbe2cc0a2
Parents: 46bf772
Author: Eric Newton <er...@gmail.com>
Authored: Tue Mar 11 16:30:45 2014 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Mar 11 16:30:45 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/proxy/SimpleProxyIT.java | 13 ++++-----
.../org/apache/accumulo/test/NamespacesIT.java | 8 +++---
.../accumulo/test/functional/BloomFilterIT.java | 30 ++++++++++++--------
3 files changed, 28 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index efe36b9..f554d63 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@ -772,15 +772,14 @@ public class SimpleProxyIT {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
+ client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- // Should take 5 seconds to read every record
- for (int i = 0; i < 20; i++) {
+ // Should take 10 seconds to read every record
+ for (int i = 0; i < 40; i++) {
client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
}
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
- client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
-
// scan
Thread t = new Thread() {
@Override
@@ -798,7 +797,7 @@ public class SimpleProxyIT {
};
t.start();
- // look for the scan
+ // look for the scan many times
List<ActiveScan> scans = new ArrayList<ActiveScan>();
for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
@@ -811,7 +810,7 @@ public class SimpleProxyIT {
if (!scans.isEmpty())
break;
- UtilWaitThread.sleep(10);
+ UtilWaitThread.sleep(100);
}
}
t.join();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
index a8d5f7f..af9274b 100644
--- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java
@@ -321,17 +321,17 @@ public class NamespacesIT extends SimpleMacIT {
}
IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan);
assertEquals(setting, setting2);
- s = c.createScanner(t1, Authorizations.EMPTY);
- assertFalse(s.iterator().hasNext());
assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertFalse(s.iterator().hasNext());
// verify can see inserted entry again
c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class));
- s = c.createScanner(t1, Authorizations.EMPTY);
- assertTrue(s.iterator().hasNext());
assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName));
assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName));
+ s = c.createScanner(t1, Authorizations.EMPTY);
+ assertTrue(s.iterator().hasNext());
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 27aab91..50ca776 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor;
import org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor;
import org.apache.accumulo.core.file.keyfunctor.RowFunctor;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +57,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
siteConfig.put(Property.TABLE_BLOOM_SIZE.getKey(), "2000000");
siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%");
siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0");
+ siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M");
siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1G");
cfg.setSiteConfig(siteConfig );
}
@@ -69,9 +71,9 @@ public class BloomFilterIT extends ConfigurableMacIT {
c.tableOperations().setProperty(table, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "false");
}
log.info("Writing");
- write(c, "bt1", 1, 0, 2000000000, 1000);
- write(c, "bt2", 2, 0, 2000000000, 1000);
- write(c, "bt3", 3, 0, 2000000000, 1000);
+ write(c, "bt1", 1, 0, 2000000000, 500);
+ write(c, "bt2", 2, 0, 2000000000, 500);
+ write(c, "bt3", 3, 0, 2000000000, 500);
log.info("Writing complete");
// test inserting an empty key
@@ -94,35 +96,39 @@ public class BloomFilterIT extends ConfigurableMacIT {
// these queries should only run quickly if bloom filters are working, so lets get a base
log.info("Base query");
- long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
- long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
- long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+ long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+ long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+ long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
log.info("Base query complete");
log.info("Rewriting with bloom filters");
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
- c.tableOperations().compact("bt1", null, null, false, true);
c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnFamilyFunctor.class.getName());
- c.tableOperations().compact("bt2", null, null, false, true);
c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnQualifierFunctor.class.getName());
- c.tableOperations().compact("bt3", null, null, false, true);
c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_ENABLED.getKey(), "true");
c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName());
+
+ // ensure the updates to zookeeper propogate
+ UtilWaitThread.sleep(500);
+
c.tableOperations().compact("bt4", null, null, false, true);
+ c.tableOperations().compact("bt1", null, null, false, true);
+ c.tableOperations().compact("bt2", null, null, false, true);
+ c.tableOperations().compact("bt3", null, null, false, true);
log.info("Rewriting with bloom filters complete");
// these queries should only run quickly if bloom
// filters are working
log.info("Bloom query");
- long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000);
- long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000);
- long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000);
+ long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500);
+ long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500);
+ long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500);
log.info("Bloom query complete");
timeCheck(t1 + t2 + t3, tb1 + tb2 + tb3);