You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/10/29 19:22:03 UTC
git commit: ACCUMULO-1823 more tweaks to help stabilize the ITs,
get more debugging for post-test analysis
Updated Branches:
refs/heads/master 219560a9a -> 135e67b68
ACCUMULO-1823 more tweaks to help stabilize the ITs, get more debugging for post-test analysis
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/135e67b6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/135e67b6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/135e67b6
Branch: refs/heads/master
Commit: 135e67b68592f0d1c7ca69bac318a7ad3ed55831
Parents: 219560a
Author: Eric Newton <er...@gmail.com>
Authored: Tue Oct 29 13:39:16 2013 -0400
Committer: Eric Newton <er...@gmail.com>
Committed: Tue Oct 29 14:22:16 2013 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 7 ++++++-
.../minicluster/MiniAccumuloCluster.java | 2 ++
.../apache/accumulo/server/master/Master.java | 2 +-
.../test/functional/GarbageCollectorIT.java | 1 +
.../accumulo/test/functional/RestartIT.java | 19 ++++++++++---------
.../test/functional/RestartStressIT.java | 4 ++--
.../test/functional/SplitRecoveryIT.java | 2 +-
test/src/test/resources/log4j.properties | 5 +++--
8 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 134a037..bee4265 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.conf;
import java.io.File;
import java.lang.annotation.Annotation;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -573,6 +574,9 @@ public enum Property {
|| key.equals(Property.TABLE_LOAD_BALANCER.getKey());
}
+ // This is not a cache for loaded classes, just a way to avoid spamming the debug log
+ static Map<String, Class<? extends Object>> loaded = Collections.synchronizedMap(new HashMap<String, Class<? extends Object>>());
+
public static <T> T createInstanceFromPropertyName(AccumuloConfiguration conf, Property property, Class<T> base, T defaultInstance) {
String clazzName = conf.get(property);
T instance = null;
@@ -580,7 +584,8 @@ public enum Property {
try {
Class<? extends T> clazz = AccumuloVFSClassLoader.loadClass(clazzName, base);
instance = clazz.newInstance();
- log.debug("Loaded class : " + clazzName);
+ if (loaded.put(clazzName, clazz) != clazz)
+ log.debug("Loaded class : " + clazzName);
} catch (Exception e) {
log.warn("Failed to load class ", e);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 1c7ad74..a0cc8b1 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -240,6 +240,8 @@ public class MiniAccumuloCluster {
builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
builder.environment().put("ACCUMULO_CONF_DIR", config.getConfDir().getAbsolutePath());
+ // hadoop-2.2 puts error messages in the logs if this is not set
+ builder.environment().put("HADOOP_HOME", config.getDir().getAbsolutePath());
Process process = builder.start();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java
index c029ae5..28da13d 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -183,7 +183,7 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;
final private static int MAX_CLEANUP_WAIT_TIME = ONE_SECOND;
final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = ONE_SECOND;
- final static int MAX_TSERVER_WORK_CHUNK = 5 * ONE_SECOND;
+ final static int MAX_TSERVER_WORK_CHUNK = 5000;
final private static int MAX_BAD_STATUS_COUNT = 3;
final VolumeManager fs;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 7b9282c..9cdd9ab 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -60,6 +60,7 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
cfg.setSiteConfig(settings);
+ cfg.useMiniDFS();
}
@Test(timeout = 4 * 60 * 1000)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index de29d3b..79a895b 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@ -53,7 +53,11 @@ public class RestartIT extends ConfigurableMacIT {
private static final ScannerOpts SOPTS = new ScannerOpts();
private static final VerifyIngest.Opts VOPTS = new VerifyIngest.Opts();
+ private static final TestIngest.Opts OPTS = new TestIngest.Opts();
private static final BatchWriterOpts BWOPTS = new BatchWriterOpts();
+ static {
+ OPTS.rows = VOPTS.rows = 10 * 1000;
+ }
@Test(timeout = 2 * 60 * 1000)
public void restartMaster() throws Exception {
@@ -61,7 +65,7 @@ public class RestartIT extends ConfigurableMacIT {
c.tableOperations().create("test_ingest");
Process ingest = cluster.exec(TestIngest.class,
"-u", "root", "-p", ROOT_PASSWORD,
- "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers());
+ "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "--rows", ""+OPTS.rows);
for (ProcessReference master : cluster.getProcesses().get(ServerType.MASTER)) {
cluster.killProcess(ServerType.MASTER, master);
}
@@ -75,8 +79,7 @@ public class RestartIT extends ConfigurableMacIT {
public void restartMasterRecovery() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
- TestIngest.Opts opts = new TestIngest.Opts();
- TestIngest.ingest(c, opts, BWOPTS);
+ TestIngest.ingest(c, OPTS, BWOPTS);
for (Entry<ServerType,Collection<ProcessReference>> entry : cluster.getProcesses().entrySet()) {
for (ProcessReference proc : entry.getValue()) {
cluster.killProcess(entry.getKey(), proc);
@@ -98,7 +101,7 @@ public class RestartIT extends ConfigurableMacIT {
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
Process ingest = cluster.exec(TestIngest.class,
"-u", "root", "-p", ROOT_PASSWORD,
- "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers());
+ "-i", cluster.getInstanceName(), "-z", cluster.getZooKeepers(), "--rows", ""+VOPTS.rows);
for (ProcessReference master : cluster.getProcesses().get(ServerType.MASTER)) {
cluster.killProcess(ServerType.MASTER, master);
}
@@ -108,12 +111,11 @@ public class RestartIT extends ConfigurableMacIT {
ingest.destroy();
}
- @Test(timeout = 100 * 60 * 1000)
+ @Test(timeout = 10 * 60 * 1000)
public void killedTabletServer() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
- TestIngest.Opts opts = new TestIngest.Opts();
- TestIngest.ingest(c, opts, BWOPTS);
+ TestIngest.ingest(c, OPTS, BWOPTS);
VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
List<ProcessReference> procs = new ArrayList<ProcessReference>(cluster.getProcesses().get(ServerType.TABLET_SERVER));
for (ProcessReference tserver : procs) {
@@ -142,8 +144,7 @@ public class RestartIT extends ConfigurableMacIT {
public void killedTabletServerDuringShutdown() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
- TestIngest.Opts opts = new TestIngest.Opts();
- TestIngest.ingest(c, opts, BWOPTS);
+ TestIngest.ingest(c, OPTS, BWOPTS);
List<ProcessReference> procs = new ArrayList<ProcessReference>(cluster.getProcesses().get(ServerType.TABLET_SERVER));
cluster.killProcess(ServerType.TABLET_SERVER, procs.get(0));
assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index 28a0b19..31402ba 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@ -38,7 +38,7 @@ public class RestartStressIT extends ConfigurableMacIT {
Map<String, String> opts = new HashMap<String, String>();
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
- opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "500K");
+ opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
@@ -50,7 +50,7 @@ public class RestartStressIT extends ConfigurableMacIT {
static {
IOPTS = new TestIngest.Opts();
VOPTS = new VerifyIngest.Opts();
- IOPTS.rows = VOPTS.rows = 100*1000;
+ IOPTS.rows = VOPTS.rows = 10*1000;
}
private static final ScannerOpts SOPTS = new ScannerOpts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index e976493..a55b543 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@ -259,7 +259,7 @@ public class SplitRecoveryIT extends ConfigurableMacIT {
new SplitRecoveryIT().run();
}
- @Test(timeout = 10 * 1000)
+ @Test(timeout = 30 * 1000)
public void test() throws Exception {
assertEquals(0, exec(SplitRecoveryIT.class).waitFor());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/135e67b6/test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/test/src/test/resources/log4j.properties b/test/src/test/resources/log4j.properties
index 277822a..27e4db2 100644
--- a/test/src/test/resources/log4j.properties
+++ b/test/src/test/resources/log4j.properties
@@ -18,6 +18,7 @@ log4j.appender.CA=org.apache.log4j.ConsoleAppender
log4j.appender.CA.layout=org.apache.log4j.PatternLayout
log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{2}] %-5p: %m%n
+log4j.logger.org.apache.accumulo.core=TRACE
log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=off
log4j.logger.org.apache.accumulo.core.util.shell.Shell=FATAL
@@ -27,6 +28,6 @@ log4j.logger.org.apache.hadoop.mapred=ERROR
log4j.logger.org.apache.hadoop.tools.DistCp=WARN
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
log4j.logger.org.apache.hadoop.util.ProcessTree=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
-log4j.logger.org.apache.zookeeper.ZooKeeper=WARN
+#log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+#log4j.logger.org.apache.zookeeper.ZooKeeper=WARN
log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO