You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2022/07/05 11:32:31 UTC
[cassandra] branch cassandra-4.1 updated: Fix MixedModeAvailability upgrade dtests
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-4.1
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-4.1 by this push:
new 91d08e496e Fix MixedModeAvailability upgrade dtests
91d08e496e is described below
commit 91d08e496ee439a2179e617f4db8804ab07c301a
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Wed Jun 8 17:12:58 2022 +0100
Fix MixedModeAvailability upgrade dtests
patch by Andrés de la Peña; reviewed by Berenguer Blasi for CASSANDRA-17307
---
.../upgrade/MixedModeAvailabilityTestBase.java | 197 +++++++++++----------
...ava => MixedModeAvailabilityV30AllOneTest.java} | 13 +-
...ava => MixedModeAvailabilityV30OneAllTest.java} | 13 +-
... MixedModeAvailabilityV30QuorumQuorumTest.java} | 13 +-
...ava => MixedModeAvailabilityV3XAllOneTest.java} | 13 +-
...ava => MixedModeAvailabilityV3XOneAllTest.java} | 13 +-
... MixedModeAvailabilityV3XQuorumQuorumTest.java} | 13 +-
7 files changed, 135 insertions(+), 140 deletions(-)
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
index c1ae153525..4e50eb1481 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityTestBase.java
@@ -18,18 +18,22 @@
package org.apache.cassandra.distributed.upgrade;
-import java.util.Arrays;
-import java.util.List;
import java.util.UUID;
+import java.util.concurrent.RejectedExecutionException;
-import com.vdurmont.semver4j.Semver;
+import org.junit.Test;
+import com.vdurmont.semver4j.Semver;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.exceptions.ReadFailureException;
import org.apache.cassandra.exceptions.ReadTimeoutException;
+import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.net.Verb;
+import org.assertj.core.api.Assertions;
+import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
@@ -37,47 +41,67 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.apache.cassandra.net.Verb.READ_REQ;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static java.lang.String.format;
-public class MixedModeAvailabilityTestBase extends UpgradeTestBase
+public abstract class MixedModeAvailabilityTestBase extends UpgradeTestBase
{
private static final int NUM_NODES = 3;
private static final int COORDINATOR = 1;
- private static final List<Tester> TESTERS = Arrays.asList(new Tester(ONE, ALL),
- new Tester(QUORUM, QUORUM),
- new Tester(ALL, ONE));
+ private static final String INSERT = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)");
+ private static final String SELECT = withKeyspace("SELECT * FROM %s.t WHERE k = ?");
+ private final Semver initial;
+ private final ConsistencyLevel writeConsistencyLevel;
+ private final ConsistencyLevel readConsistencyLevel;
+
+ public MixedModeAvailabilityTestBase(Semver initial, ConsistencyLevel writeConsistencyLevel, ConsistencyLevel readConsistencyLevel)
+ {
+ this.initial = initial;
+ this.writeConsistencyLevel = writeConsistencyLevel;
+ this.readConsistencyLevel = readConsistencyLevel;
+ }
- protected static void testAvailability(Semver initial) throws Throwable
+ @Test
+ public void testAvailabilityCoordinatorNotUpgraded() throws Throwable
{
- testAvailability(initial, UpgradeTestBase.CURRENT);
+ testAvailability(false, initial, writeConsistencyLevel, readConsistencyLevel);
}
- protected static void testAvailability(Semver initial, Semver upgrade) throws Throwable
+ @Test
+ public void testAvailabilityCoordinatorUpgraded() throws Throwable
{
- testAvailability(true, initial, upgrade);
- testAvailability(false, initial, upgrade);
+ testAvailability(true, initial, writeConsistencyLevel, readConsistencyLevel);
}
private static void testAvailability(boolean upgradedCoordinator,
Semver initial,
- Semver upgrade) throws Throwable
+ ConsistencyLevel writeConsistencyLevel,
+ ConsistencyLevel readConsistencyLevel) throws Throwable
{
new TestCase()
.nodes(NUM_NODES)
.nodesToUpgrade(upgradedCoordinator ? 1 : 2)
- .upgrades(initial, upgrade)
- .withConfig(config -> config.set("read_request_timeout_in_ms", SECONDS.toMillis(2))
- .set("write_request_timeout_in_ms", SECONDS.toMillis(2)))
- .setup(c -> c.schemaChange(withKeyspace("CREATE TABLE %s.t (k uuid, c int, v int, PRIMARY KEY (k, c))")))
+ .upgrades(initial, UpgradeTestBase.CURRENT)
+ .withConfig(config -> config.set("read_request_timeout_in_ms", SECONDS.toMillis(5))
+ .set("write_request_timeout_in_ms", SECONDS.toMillis(5)))
+ // use retry of 10ms so that each check is consistent
+ // At the start of the world cfs.sampleLatencyNanos == 0, which means speculation acts as if ALWAYS is done,
+ // but after the first refresh this gets set high enough that we don't trigger speculation for the rest of the test!
+ // To be consistent set retry to 10ms so cfs.sampleLatencyNanos stays consistent for the duration of the test.
+ .setup(cluster -> {
+ cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k uuid, c int, v int, PRIMARY KEY (k, c)) WITH speculative_retry = '10ms'"));
+ cluster.setUncaughtExceptionsFilter(throwable -> throwable instanceof RejectedExecutionException);
+ })
.runAfterNodeUpgrade((cluster, n) -> {
+ ICoordinator coordinator = cluster.coordinator(COORDINATOR);
+
// using 0 to 2 down nodes...
- for (int numNodesDown = 0; numNodesDown < NUM_NODES; numNodesDown++)
+ for (int i = 0; i < NUM_NODES; i++)
{
+ final int numNodesDown = i;
+
// disable communications to the down nodes
if (numNodesDown > 0)
{
@@ -85,10 +109,38 @@ public class MixedModeAvailabilityTestBase extends UpgradeTestBase
cluster.filters().outbound().verbs(Verb.MUTATION_REQ.id).to(replica(COORDINATOR, numNodesDown)).drop();
}
- // run the test cases that are compatible with the number of down nodes
- ICoordinator coordinator = cluster.coordinator(COORDINATOR);
- for (Tester tester : TESTERS)
- tester.test(coordinator, numNodesDown, upgradedCoordinator);
+ UUID key = UUID.randomUUID();
+ Object[] row1 = row(key, 1, 10);
+ Object[] row2 = row(key, 2, 20);
+
+ boolean wrote = false;
+ try
+ {
+ // test write
+ maybeFail(false, numNodesDown > maxNodesDown(writeConsistencyLevel), () -> {
+ coordinator.execute(INSERT, writeConsistencyLevel, row1);
+ coordinator.execute(INSERT, writeConsistencyLevel, row2);
+ });
+
+ wrote = true;
+
+ // test read
+ maybeFail(true, numNodesDown > maxNodesDown(readConsistencyLevel), () -> {
+ Object[][] rows = coordinator.execute(SELECT, readConsistencyLevel, key);
+ if (numNodesDown <= maxNodesDown(writeConsistencyLevel))
+ assertRows(rows, row1, row2);
+ });
+ }
+ catch (Throwable t)
+ {
+ throw new AssertionError(format("Unexpected error while %s in case write-read consistency %s-%s with %s coordinator and %d nodes down: %s",
+ wrote ? "reading" : "writing",
+ writeConsistencyLevel,
+ readConsistencyLevel,
+ upgradedCoordinator ? "upgraded" : "not upgraded",
+ numNodesDown,
+ t), t);
+ }
}
}).run();
}
@@ -99,88 +151,49 @@ public class MixedModeAvailabilityTestBase extends UpgradeTestBase
return depth == 0 ? node : replica(node == NUM_NODES ? 1 : node + 1, depth - 1);
}
- private static class Tester
+ private static void maybeFail(boolean isRead, boolean shouldFail, Runnable test)
{
- private static final String INSERT = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)");
- private static final String SELECT = withKeyspace("SELECT * FROM %s.t WHERE k = ?");
-
- private final ConsistencyLevel writeConsistencyLevel;
- private final ConsistencyLevel readConsistencyLevel;
-
- private Tester(ConsistencyLevel writeConsistencyLevel, ConsistencyLevel readConsistencyLevel)
+ try
{
- this.writeConsistencyLevel = writeConsistencyLevel;
- this.readConsistencyLevel = readConsistencyLevel;
+ test.run();
+ assertFalse("Should have failed", shouldFail);
}
-
- public void test(ICoordinator coordinator, int numNodesDown, boolean upgradedCoordinator)
+ catch (Exception e)
{
- UUID key = UUID.randomUUID();
- Object[] row1 = row(key, 1, 10);
- Object[] row2 = row(key, 2, 20);
+ if (!shouldFail)
+ throw e;
- boolean wrote = false;
- try
- {
- // test write
- maybeFail(WriteTimeoutException.class, numNodesDown > maxNodesDown(writeConsistencyLevel), () -> {
- coordinator.execute(INSERT, writeConsistencyLevel, row1);
- coordinator.execute(INSERT, writeConsistencyLevel, row2);
- });
-
- wrote = true;
-
- // test read
- maybeFail(ReadTimeoutException.class, numNodesDown > maxNodesDown(readConsistencyLevel), () -> {
- Object[][] rows = coordinator.execute(SELECT, readConsistencyLevel, key);
- if (numNodesDown <= maxNodesDown(writeConsistencyLevel))
- assertRows(rows, row1, row2);
- });
- }
- catch (Throwable t)
- {
- throw new AssertionError(format("Unexpected error while %s in case write-read consistency %s-%s with %s coordinator and %d nodes down",
- wrote ? "reading" : "writing",
- writeConsistencyLevel,
- readConsistencyLevel,
- upgradedCoordinator ? "upgraded" : "not upgraded",
- numNodesDown), t);
- }
- }
+ // we should use exception class names due to the different classpaths
+ String className = (e instanceof RuntimeException && e.getCause() != null)
+ ? e.getCause().getClass().getCanonicalName()
+ : e.getClass().getCanonicalName();
- private static <E extends Exception> void maybeFail(Class<E> exceptionClass, boolean shouldFail, Runnable test)
- {
- try
+ if (isRead)
{
- test.run();
- assertFalse(shouldFail);
+ Assertions.assertThat(className)
+ .isIn(ReadTimeoutException.class.getCanonicalName(),
+ ReadFailureException.class.getCanonicalName());
}
- catch (Exception e)
+ else
{
- // we should use exception class names due to the different classpaths
- String className = e.getClass().getCanonicalName();
- if (e instanceof RuntimeException && e.getCause() != null)
- className = e.getCause().getClass().getCanonicalName();
-
- if (shouldFail)
- assertEquals(exceptionClass.getCanonicalName(), className);
- else
- throw e;
+ Assertions.assertThat(className)
+ .isIn(WriteTimeoutException.class.getCanonicalName(),
+ WriteFailureException.class.getCanonicalName());
}
}
+ }
- private static int maxNodesDown(ConsistencyLevel cl)
- {
- if (cl == ONE)
- return 2;
+ private static int maxNodesDown(ConsistencyLevel cl)
+ {
+ if (cl == ONE)
+ return 2;
- if (cl == QUORUM)
- return 1;
+ if (cl == QUORUM)
+ return 1;
- if (cl == ALL)
- return 0;
+ if (cl == ALL)
+ return 0;
- throw new IllegalArgumentException("Unsupported consistency level: " + cl);
- }
+ throw new IllegalArgumentException("Unsupported consistency level: " + cl);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30AllOneTest.java
similarity index 71%
copy from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
copy to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30AllOneTest.java
index 984df3ba9f..96dc4547dc 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30AllOneTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v30.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v30 with ALL-ONE write-read consistency.
*/
-public class MixedModeAvailabilityV30Test extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV30AllOneTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV30AllOneTest()
{
- testAvailability(v30);
+ super(v30, ConsistencyLevel.ALL, ConsistencyLevel.ONE);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30OneAllTest.java
similarity index 71%
rename from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XTest.java
rename to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30OneAllTest.java
index 70230f5f0a..be5154979e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30OneAllTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v3X.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v30 with ONE-ALL write-read consistency.
*/
-public class MixedModeAvailabilityV3XTest extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV30OneAllTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV30OneAllTest()
{
- testAvailability(v3X);
+ super(v30, ConsistencyLevel.ONE, ConsistencyLevel.ALL);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30QuorumQuorumTest.java
similarity index 69%
copy from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
copy to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30QuorumQuorumTest.java
index 984df3ba9f..8df53d187e 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30QuorumQuorumTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v30.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v30 with QUORUM-QUORUM write-read consistency.
*/
-public class MixedModeAvailabilityV30Test extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV30QuorumQuorumTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV30QuorumQuorumTest()
{
- testAvailability(v30);
+ super(v30, ConsistencyLevel.QUORUM, ConsistencyLevel.QUORUM);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XAllOneTest.java
similarity index 71%
copy from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
copy to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XAllOneTest.java
index 984df3ba9f..955e0ba082 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XAllOneTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v30.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v3X with ALL-ONE write-read consistency.
*/
-public class MixedModeAvailabilityV30Test extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV3XAllOneTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV3XAllOneTest()
{
- testAvailability(v30);
+ super(v3X, ConsistencyLevel.ALL, ConsistencyLevel.ONE);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
similarity index 70%
copy from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
copy to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
index 984df3ba9f..8ea94ea749 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XOneAllTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v30.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v3X with ONE-ALL write-read consistency.
*/
-public class MixedModeAvailabilityV30Test extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV3XOneAllTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV3XOneAllTest() throws Throwable
{
- testAvailability(v30);
+ super(v3X, ConsistencyLevel.ONE, ConsistencyLevel.ALL);
}
}
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XQuorumQuorumTest.java
similarity index 69%
rename from test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
rename to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XQuorumQuorumTest.java
index 984df3ba9f..e65f4f46d8 100644
--- a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV30Test.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeAvailabilityV3XQuorumQuorumTest.java
@@ -18,18 +18,15 @@
package org.apache.cassandra.distributed.upgrade;
-import org.junit.Test;
-
-import org.apache.cassandra.distributed.shared.Versions;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
/**
- * {@link MixedModeAvailabilityTestBase} for upgrades from v30.
+ * {@link MixedModeAvailabilityTestBase} for upgrades from v3X with QUORUM-QUORUM write-read consistency.
*/
-public class MixedModeAvailabilityV30Test extends MixedModeAvailabilityTestBase
+public class MixedModeAvailabilityV3XQuorumQuorumTest extends MixedModeAvailabilityTestBase
{
- @Test
- public void testAvailability() throws Throwable
+ public MixedModeAvailabilityV3XQuorumQuorumTest()
{
- testAvailability(v30);
+ super(v3X, ConsistencyLevel.QUORUM, ConsistencyLevel.QUORUM);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org