You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/04/05 19:41:10 UTC
svn commit: r1465060 - in
/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase:
IntegrationTestDataIngestSlowDeterministic.java
IntegrationTestDataIngestWithChaosMonkey.java util/ChaosMonkey.java
Author: sershe
Date: Fri Apr 5 17:41:09 2013
New Revision: 1465060
URL: http://svn.apache.org/r1465060
Log:
HBASE-8260 create generic integration test for trunk and 94 that is more deterministic, can be run for longer and is less aggressive
Added:
hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java
Modified:
hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
Added: hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java?rev=1465060&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java (added)
+++ hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java Fri Apr 5 17:41:09 2013
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.util.ChaosMonkey;
+import org.apache.hadoop.hbase.util.ChaosMonkey.BatchRestartRs;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartActiveMaster;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRandomRs;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingMeta;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RestartRsHoldingRoot;
+import org.apache.hadoop.hbase.util.ChaosMonkey.RollingBatchRestartRs;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * A system test which does large data ingestion and verify using {@link LoadTestTool}.
+ * It performs a set of actions deterministically using ChaosMonkey, then starts killing
+ * things randomly. You can configure how long should the load test run by using
+ * "hbase.IntegrationTestDataIngestSlowDeterministic.runtime" configuration parameter.
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestDataIngestSlowDeterministic extends IngestIntegrationTestBase {
+ private static final int SERVER_COUNT = 4; // number of slaves for the smallest cluster
+ private static final long DEFAULT_RUN_TIME = 30 * 60 * 1000;
+ private static final long CHAOS_EVERY_MS = 150 * 1000; // Chaos every 2.5 minutes.
+
+ private ChaosMonkey monkey;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp(SERVER_COUNT);
+ ChaosMonkey.Action[] actions = new ChaosMonkey.Action[] {
+ new RestartRandomRs(60000),
+ new BatchRestartRs(5000, 0.5f),
+ new RestartActiveMaster(5000),
+ new RollingBatchRestartRs(5000, 1.0f),
+ new RestartRsHoldingMeta(35000),
+ new RestartRsHoldingRoot(35000)
+ };
+ monkey = new ChaosMonkey(util, new ChaosMonkey.CompositeSequenialPolicy(
+ new ChaosMonkey.DoActionsOncePolicy(CHAOS_EVERY_MS, actions),
+ new ChaosMonkey.PeriodicRandomActionPolicy(CHAOS_EVERY_MS, actions)));
+ monkey.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (monkey != null) {
+ monkey.stop("tearDown");
+ monkey.waitForStop();
+ }
+ super.tearDown();
+ }
+
+ @Test
+ public void testDataIngest() throws Exception {
+ runIngestTest(DEFAULT_RUN_TIME, 2500, 10, 1024, 10);
+ }
+}
Modified: hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java?rev=1465060&r1=1465059&r2=1465060&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java (original)
+++ hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java Fri Apr 5 17:41:09 2013
@@ -18,11 +18,6 @@
package org.apache.hadoop.hbase;
-import java.io.IOException;
-
-import junit.framework.Assert;
-
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChaosMonkey;
import org.junit.After;
import org.junit.Before;
Modified: hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java?rev=1465060&r1=1465059&r2=1465060&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java (original)
+++ hbase/branches/0.95/hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java Fri Apr 5 17:41:09 2013
@@ -453,16 +453,122 @@ public class ChaosMonkey extends Abstrac
}
}
+ /** A policy that runs multiple other policies one after the other */
+ public static class CompositeSequenialPolicy extends Policy {
+ private List<Policy> policies;
+ public CompositeSequenialPolicy(Policy... policies) {
+ this.policies = Arrays.asList(policies);
+ }
+
+ @Override
+ public void stop(String why) {
+ super.stop(why);
+ for (Policy p : policies) {
+ p.stop(why);
+ }
+ }
+
+ @Override
+ public void run() {
+ for (Policy p : policies) {
+ p.run();
+ }
+ }
+
+ @Override
+ public void init(PolicyContext context) throws Exception {
+ super.init(context);
+ for (Policy p : policies) {
+ p.init(context);
+ }
+ }
+ }
+
+ /** A policy which does stuff every time interval. */
+ public static abstract class PeriodicPolicy extends Policy {
+ private long periodMs;
+
+ public PeriodicPolicy(long periodMs) {
+ this.periodMs = periodMs;
+ }
+
+ @Override
+ public void run() {
+ // Add some jitter.
+ int jitter = new Random().nextInt((int)periodMs);
+ LOG.info("Sleeping for " + jitter + " to add jitter");
+ Threads.sleep(jitter);
+
+ while (!isStopped()) {
+ long start = System.currentTimeMillis();
+ runOneIteration();
+
+ if (isStopped()) return;
+ long sleepTime = periodMs - (System.currentTimeMillis() - start);
+ if (sleepTime > 0) {
+ LOG.info("Sleeping for: " + sleepTime);
+ Threads.sleep(sleepTime);
+ }
+ }
+ }
+
+ protected abstract void runOneIteration();
+
+ @Override
+ public void init(PolicyContext context) throws Exception {
+ super.init(context);
+ LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period: " + periodMs);
+ }
+ }
+
+
+ /** A policy which performs a sequence of actions deterministically. */
+ public static class DoActionsOncePolicy extends PeriodicPolicy {
+ private List<Action> actions;
+
+ public DoActionsOncePolicy(long periodMs, List<Action> actions) {
+ super(periodMs);
+ this.actions = new ArrayList<ChaosMonkey.Action>(actions);
+ }
+
+ public DoActionsOncePolicy(long periodMs, Action... actions) {
+ this(periodMs, Arrays.asList(actions));
+ }
+
+ @Override
+ protected void runOneIteration() {
+ if (actions.isEmpty()) {
+ this.stop("done");
+ return;
+ }
+ Action action = actions.remove(0);
+
+ try {
+ action.perform();
+ } catch (Exception ex) {
+ LOG.warn("Exception occured during performing action: "
+ + StringUtils.stringifyException(ex));
+ }
+ }
+
+ @Override
+ public void init(PolicyContext context) throws Exception {
+ super.init(context);
+ for (Action action : actions) {
+ action.init(this.context);
+ }
+ }
+ }
+
/**
* A policy, which picks a random action according to the given weights,
* and performs it every configurable period.
*/
- public static class PeriodicRandomActionPolicy extends Policy {
- private long periodMs;
+ public static class PeriodicRandomActionPolicy extends PeriodicPolicy {
private List<Pair<Action, Integer>> actions;
public PeriodicRandomActionPolicy(long periodMs, List<Pair<Action, Integer>> actions) {
- this.periodMs = periodMs;
+ super(periodMs);
this.actions = actions;
}
@@ -472,7 +578,7 @@ public class ChaosMonkey extends Abstrac
}
public PeriodicRandomActionPolicy(long periodMs, Action... actions) {
- this.periodMs = periodMs;
+ super(periodMs);
this.actions = new ArrayList<Pair<Action, Integer>>(actions.length);
for (Action action : actions) {
this.actions.add(new Pair<Action, Integer>(action, 1));
@@ -480,35 +586,19 @@ public class ChaosMonkey extends Abstrac
}
@Override
- public void run() {
- //add some jitter
- int jitter = new Random().nextInt((int)periodMs);
- LOG.info("Sleeping for " + jitter + " to add jitter");
- Threads.sleep(jitter);
-
- while (!isStopped()) {
- long start = System.currentTimeMillis();
- Action action = selectWeightedRandomItem(actions);
-
- try {
- action.perform();
- } catch (Exception ex) {
- LOG.warn("Exception occured during performing action: "
- + StringUtils.stringifyException(ex));
- }
-
- long sleepTime = periodMs - (System.currentTimeMillis() - start);
- if (sleepTime > 0) {
- LOG.info("Sleeping for:" + sleepTime);
- Threads.sleep(sleepTime);
- }
+ protected void runOneIteration() {
+ Action action = selectWeightedRandomItem(actions);
+ try {
+ action.perform();
+ } catch (Exception ex) {
+ LOG.warn("Exception occured during performing action: "
+ + StringUtils.stringifyException(ex));
}
}
@Override
public void init(PolicyContext context) throws Exception {
super.init(context);
- LOG.info("Using ChaosMonkey Policy: " + this.getClass() + ", period:" + periodMs);
for (Pair<Action, Integer> action : actions) {
action.getFirst().init(this.context);
}
@@ -661,4 +751,4 @@ public class ChaosMonkey extends Abstrac
System.exit(ret);
}
-}
\ No newline at end of file
+}