You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jm...@apache.org on 2022/10/28 18:48:07 UTC
[cassandra] branch trunk updated: Disable resumable bootstrap by default
This is an automated email from the ASF dual-hosted git repository.
jmckenzie pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 39a470235a Disable resumable bootstrap by default
39a470235a is described below
commit 39a470235af13837a1a022ab0a1b6f8f062bcf6a
Author: Josh McKenzie <jm...@apache.org>
AuthorDate: Tue Sep 20 15:22:51 2022 -0400
Disable resumable bootstrap by default
Patch by Marcus Eriksson; reviewed by Jordan West, Blake Eggleston, and Josh McKenzie for CASSANDRA-17679
Co-authored-by: Marcus Eriksson <ma...@apache.org>
Co-authored-by: Josh McKenzie <jm...@apache.org>
---
CHANGES.txt | 1 +
NEWS.txt | 9 +-
.../config/CassandraRelevantProperties.java | 13 ++
.../org/apache/cassandra/db/SystemKeyspace.java | 11 +-
.../org/apache/cassandra/dht/RangeStreamer.java | 79 ++++++++----
src/java/org/apache/cassandra/locator/Replica.java | 1 -
.../apache/cassandra/service/StorageService.java | 2 +-
.../cassandra/tools/nodetool/BootstrapResume.java | 10 ++
.../test/BootstrapBinaryDisabledTest.java | 23 ++++
.../distributed/test/ring/BootstrapTest.java | 139 ++++++++++++++++++++-
10 files changed, 256 insertions(+), 32 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index d3f6df4183..1db667c8a4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.2
+ * Disable resumable bootstrap by default (CASSANDRA-17679)
* Include Git SHA in --verbose flag for nodetool version (CASSANDRA-17753)
* Update Byteman to 4.0.20 and Jacoco to 0.8.8 (CASSANDRA-16413)
* Add memtable option among possible tab completions for a table (CASSANDRA-17982)
diff --git a/NEWS.txt b/NEWS.txt
index 1f48643a81..3425d75593 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -91,8 +91,13 @@ New features
- Added new CQL table property 'allow_auto_snapshot' which is by default true. When set to false and 'auto_snapshot: true'
in cassandra.yaml, there will be no snapshot taken when a table is truncated or dropped. When auto_snapshot in
casandra.yaml is set to false, the newly added table property does not have any effect.
- - Added --older-than and --older-than-timestamp options to nodetool clearsnapshot command. It is possible to
- clear snapshots which are older than some period for example, "--older-than 5h" to remove
+ - Changed default on resumable bootstrap to be disabled. Resumable bootstrap has edge cases with potential correctness
+ violations or data loss scenarios if nodes go down during bootstrap, tombstones are written, and operations race with
+ repair. As streaming is considerably faster in the 4.0+ era (as well as with zero copy streaming), the risks of
+ having these edge cases during a failed and resumed bootstrap are no longer deemed acceptable.
+ To re-enable this feature, use the -Dcassandra.reset_bootstrap_progress=false environment flag.
+ - Added --older-than and --older-than-timestamp options to nodetool clearsnapshot command. It is possible to
+ clear snapshots which are older than some period for example, "--older-than 5h" to remove
snapshots older than 5 hours and it is possible to clear all snapshots older than some timestamp, for example
--older-than-timestamp 2022-12-03T10:15:30Z.
- Cassandra logs can be viewed in the virtual table system_views.system_logs.
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 88d70f402b..4a3bfc5a06 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -154,6 +154,11 @@ public enum CassandraRelevantProperties
*/
BOOTSTRAP_SCHEMA_DELAY_MS("cassandra.schema_delay_ms"),
+ /**
+ * Whether we reset any found data from previously run bootstraps.
+ */
+ RESET_BOOTSTRAP_PROGRESS("cassandra.reset_bootstrap_progress"),
+
/**
* When draining, how long to wait for mutating executors to shutdown.
*/
@@ -412,6 +417,14 @@ public enum CassandraRelevantProperties
System.setProperty(key, Boolean.toString(value));
}
+ /**
+ * Clears the value set in the system property.
+ */
+ public void clearValue()
+ {
+ System.clearProperty(key);
+ }
+
/**
* Gets the value of a system property as a int.
* @return system property int value if it exists, defaultValue otherwise.
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 3ed52dc433..5280b3f1da 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -1660,12 +1660,18 @@ public final class SystemKeyspace
}
}
- public static void resetAvailableRanges()
+ public static void resetAvailableStreamedRanges()
{
ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES_V2);
availableRanges.truncateBlockingWithoutSnapshot();
}
+ public static void resetAvailableStreamedRangesForKeyspace(String keyspace)
+ {
+ String cql = "DELETE FROM %s.%s WHERE keyspace_name = ?";
+ executeInternal(format(cql, SchemaConstants.SYSTEM_KEYSPACE_NAME, AVAILABLE_RANGES_V2), keyspace);
+ }
+
public static synchronized void updateTransferredRanges(StreamOperation streamOperation,
InetAddressAndPort peer,
String keyspace,
@@ -1775,7 +1781,8 @@ public final class SystemKeyspace
return rawRanges.stream().map(buf -> byteBufferToRange(buf, partitioner)).collect(Collectors.toSet());
}
- static ByteBuffer rangeToBytes(Range<Token> range)
+ @VisibleForTesting
+ public static ByteBuffer rangeToBytes(Range<Token> range)
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java
index dda6863153..08d834459a 100644
--- a/src/java/org/apache/cassandra/dht/RangeStreamer.java
+++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java
@@ -71,6 +71,7 @@ import static com.google.common.base.Predicates.and;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterables.any;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
import static org.apache.cassandra.locator.Replica.fullReplica;
/**
@@ -667,32 +668,62 @@ public class RangeStreamer
logger.debug("Keyspace {} Sources {}", keyspace, sources);
sources.asMap().forEach((source, fetchReplicas) -> {
- // filter out already streamed ranges
- SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
+ List<FetchReplica> remaining;
- Predicate<FetchReplica> isAvailable = fetch -> {
- boolean isInFull = available.full.contains(fetch.local.range());
- boolean isInTrans = available.trans.contains(fetch.local.range());
-
- if (!isInFull && !isInTrans)
- //Range is unavailable
- return false;
-
- if (fetch.local.isFull())
- //For full, pick only replicas with matching transientness
- return isInFull == fetch.remote.isFull();
-
- // Any transient or full will do
- return true;
- };
-
- List<FetchReplica> remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
-
- if (remaining.size() < available.full.size() + available.trans.size())
+ // If the operator's specified they want to reset bootstrap progress, we don't check previous attempted
+ // bootstraps and just restart with all.
+ if (RESET_BOOTSTRAP_PROGRESS.getBoolean())
+ {
+ // TODO: Also remove the files on disk. See discussion in CASSANDRA-17679
+ SystemKeyspace.resetAvailableStreamedRangesForKeyspace(keyspace);
+ remaining = new ArrayList<>(fetchReplicas);
+ }
+ else
{
- List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
- logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}",
- fetchReplicas, skipped, available.full, available.trans);
+ // Filter out already streamed ranges
+ SystemKeyspace.AvailableRanges available = stateStore.getAvailableRanges(keyspace, metadata.partitioner);
+
+ Predicate<FetchReplica> isAvailable = fetch -> {
+ boolean isInFull = available.full.contains(fetch.local.range());
+ boolean isInTrans = available.trans.contains(fetch.local.range());
+
+ if (!isInFull && !isInTrans)
+ // Range is unavailable
+ return false;
+
+ if (fetch.local.isFull())
+ // For full, pick only replicas with matching transientness
+ return isInFull == fetch.remote.isFull();
+
+ // Any transient or full will do
+ return true;
+ };
+
+ remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList());
+
+ if (remaining.size() < available.full.size() + available.trans.size())
+ {
+ // If the operator hasn't specified what to do when we discover a previous partially successful bootstrap,
+ // we error out and tell them to manually reconcile it. See CASSANDRA-17679.
+ if (!RESET_BOOTSTRAP_PROGRESS.isPresent())
+ {
+ List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
+ String msg = String.format("Discovered existing bootstrap data and %s " +
+ "is not configured; aborting bootstrap. Please clean up local files manually " +
+ "and try again or set cassandra.reset_bootstrap_progress=true to ignore. " +
+ "Found: %s. Fully available: %s. Transiently available: %s",
+ RESET_BOOTSTRAP_PROGRESS.getKey(), skipped, available.full, available.trans);
+ logger.error(msg);
+ throw new IllegalStateException(msg);
+ }
+
+ if (!RESET_BOOTSTRAP_PROGRESS.getBoolean())
+ {
+ List<FetchReplica> skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList());
+ logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}",
+ fetchReplicas, skipped, available.full, available.trans);
+ }
+ }
}
if (logger.isTraceEnabled())
diff --git a/src/java/org/apache/cassandra/locator/Replica.java b/src/java/org/apache/cassandra/locator/Replica.java
index 4c5f7c6f04..4c58e64350 100644
--- a/src/java/org/apache/cassandra/locator/Replica.java
+++ b/src/java/org/apache/cassandra/locator/Replica.java
@@ -191,6 +191,5 @@ public final class Replica implements Comparable<Replica>
{
return transientReplica(endpoint, new Range<>(start, end));
}
-
}
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index b4022ba8a7..d616233104 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -1946,7 +1946,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
if (Boolean.getBoolean("cassandra.reset_bootstrap_progress"))
{
logger.info("Resetting bootstrap progress to start fresh");
- SystemKeyspace.resetAvailableRanges();
+ SystemKeyspace.resetAvailableStreamedRanges();
}
// Force disk boundary invalidation now that local tokens are set
diff --git a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
index b0058185b7..a55cded6ac 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/BootstrapResume.java
@@ -22,17 +22,27 @@ import io.airlift.airline.Command;
import java.io.IOError;
import java.io.IOException;
+import io.airlift.airline.Option;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
+
@Command(name = "resume", description = "Resume bootstrap streaming")
public class BootstrapResume extends NodeToolCmd
{
+ @Option(title = "force",
+ name = { "-f", "--force"},
+ description = "Use --force to resume bootstrap regardless of cassandra.reset_bootstrap_progress environment variable. WARNING: This is potentially dangerous, see CASSANDRA-17679")
+ boolean force = false;
+
@Override
protected void execute(NodeProbe probe)
{
try
{
+ if ((!RESET_BOOTSTRAP_PROGRESS.isPresent() || RESET_BOOTSTRAP_PROGRESS.getBoolean()) && !force)
+ throw new RuntimeException("'nodetool bootstrap resume' is disabled.");
probe.resumeBootstrap(probe.output().out);
}
catch (IOException e)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java b/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
index 3f50c3089c..c7140badd2 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/BootstrapBinaryDisabledTest.java
@@ -24,7 +24,9 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
@@ -38,11 +40,32 @@ import org.apache.cassandra.distributed.shared.Byteman;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.utils.Shared;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
+
/**
* Replaces python dtest bootstrap_test.py::TestBootstrap::test_bootstrap_binary_disabled
*/
public class BootstrapBinaryDisabledTest extends TestBaseImpl
{
+ static String originalResetBootstrapProgress = null;
+
+ @BeforeClass
+ public static void beforeClass() throws Throwable
+ {
+ TestBaseImpl.beforeClass();
+ originalResetBootstrapProgress = RESET_BOOTSTRAP_PROGRESS.getString();
+ RESET_BOOTSTRAP_PROGRESS.setBoolean(false);
+ }
+
+ @AfterClass
+ public static void afterClass()
+ {
+ if (originalResetBootstrapProgress == null)
+ RESET_BOOTSTRAP_PROGRESS.clearValue();
+ else
+ RESET_BOOTSTRAP_PROGRESS.setString(originalResetBootstrapProgress);
+ }
+
@Test
public void test() throws IOException, TimeoutException
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
index 423e78b4ba..b337f8db60 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java
@@ -19,7 +19,10 @@
package org.apache.cassandra.distributed.test.ring;
import java.lang.management.ManagementFactory;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -29,6 +32,11 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
@@ -37,8 +45,10 @@ import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.schema.SchemaConstants;
import static java.util.Arrays.asList;
+import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS;
import static org.apache.cassandra.distributed.action.GossipHelper.bootstrap;
import static org.apache.cassandra.distributed.action.GossipHelper.pullSchemaFrom;
import static org.apache.cassandra.distributed.action.GossipHelper.statusToBootstrap;
@@ -50,6 +60,8 @@ public class BootstrapTest extends TestBaseImpl
{
private long savedMigrationDelay;
+ static String originalResetBootstrapProgress = null;
+
@Before
public void beforeTest()
{
@@ -61,16 +73,139 @@ public class BootstrapTest extends TestBaseImpl
// for each test case, the MIGRATION_DELAY time is adjusted accordingly
savedMigrationDelay = CassandraRelevantProperties.MIGRATION_DELAY.getLong();
CassandraRelevantProperties.MIGRATION_DELAY.setLong(ManagementFactory.getRuntimeMXBean().getUptime() + savedMigrationDelay);
+
+ originalResetBootstrapProgress = RESET_BOOTSTRAP_PROGRESS.getString();
}
@After
public void afterTest()
{
CassandraRelevantProperties.MIGRATION_DELAY.setLong(savedMigrationDelay);
+ if (originalResetBootstrapProgress == null)
+ RESET_BOOTSTRAP_PROGRESS.clearValue();
+ else
+ RESET_BOOTSTRAP_PROGRESS.setString(originalResetBootstrapProgress);
+ }
+
+ @Test
+ public void bootstrapWithResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.setBoolean(false);
+ bootstrapTest();
+ }
+
+ @Test
+ public void bootstrapWithoutResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.setBoolean(true);
+ bootstrapTest();
+ }
+
+ /**
+ * Confirm that a normal, non-resumed bootstrap without the reset_bootstrap_progress param specified works without issue.
+ * @throws Throwable
+ */
+ @Test
+ public void bootstrapUnspecifiedResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.clearValue();
+ bootstrapTest();
}
+ /**
+ * Confirm that, in the absence of the reset_bootstrap_progress param being set and in the face of a found prior
+ * partial bootstrap, we error out and don't complete our bootstrap.
+ *
+ * Test w/out vnodes only; logic is identical for both run env but the token alloc in this test doesn't work for
+ * vnode env and it's not worth the lift to update it to work in both env.
+ *
+ * @throws Throwable
+ */
@Test
- public void bootstrapTest() throws Throwable
+ public void bootstrapUnspecifiedFailsOnResumeTest() throws Throwable
+ {
+ RESET_BOOTSTRAP_PROGRESS.clearValue();
+
+ // Need our partitioner active for rangeToBytes conversion below
+ Config c = DatabaseDescriptor.loadConfig();
+ DatabaseDescriptor.daemonInitialization(() -> c);
+
+ int originalNodeCount = 2;
+ int expandedNodeCount = originalNodeCount + 1;
+
+ boolean sawException = false;
+ try (Cluster cluster = builder().withNodes(originalNodeCount)
+ .withoutVNodes()
+ .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(expandedNodeCount))
+ .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(expandedNodeCount, "dc0", "rack0"))
+ .withConfig(config -> config.with(NETWORK, GOSSIP))
+ .start())
+ {
+ populate(cluster, 0, 100);
+
+ IInstanceConfig config = cluster.newInstanceConfig();
+ IInvokableInstance newInstance = cluster.bootstrap(config);
+ withProperty("cassandra.join_ring", false, () -> newInstance.startup(cluster));
+
+ cluster.forEach(statusToBootstrap(newInstance));
+
+ List<Token> tokens = cluster.tokens();
+ assert tokens.size() >= 3;
+
+ /*
+ Our local tokens:
+ Tokens in cluster tokens: [-3074457345618258603, 3074457345618258601, 9223372036854775805]
+
+ From the bootstrap process:
+ fetchReplicas in our test keyspace:
+ [FetchReplica
+ {local=Full(/127.0.0.3:7012,(-3074457345618258603,3074457345618258601]),
+ remote=Full(/127.0.0.1:7012,(-3074457345618258603,3074457345618258601])},
+ FetchReplica
+ {local=Full(/127.0.0.3:7012,(9223372036854775805,-3074457345618258603]),
+ remote=Full(/127.0.0.1:7012,(9223372036854775805,-3074457345618258603])},
+ FetchReplica
+ {local=Full(/127.0.0.3:7012,(3074457345618258601,9223372036854775805]),
+ remote=Full(/127.0.0.1:7012,(3074457345618258601,9223372036854775805])}]
+ */
+
+ // Insert some bogus ranges in the keyspace to be bootstrapped to trigger the check on available ranges on bootstrap.
+ // Note: these have to precisely overlap with the token ranges hit during streaming or they won't trigger the
+ // availability logic on bootstrap to then except out; we can't just have _any_ range for a keyspace, but rather,
+ // must have a range that overlaps with what we're trying to stream.
+ Set<Range <Token>> fullSet = new HashSet<>();
+ fullSet.add(new Range<>(tokens.get(0), tokens.get(1)));
+ fullSet.add(new Range<>(tokens.get(1), tokens.get(2)));
+ fullSet.add(new Range<>(tokens.get(2), tokens.get(0)));
+
+ // Should be fine to trigger on full ranges only but add a partial for good measure.
+ Set<Range <Token>> partialSet = new HashSet<>();
+ partialSet.add(new Range<>(tokens.get(2), tokens.get(1)));
+
+ String cql = String.format("INSERT INTO %s.%s (keyspace_name, full_ranges, transient_ranges) VALUES (?, ?, ?)",
+ SchemaConstants.SYSTEM_KEYSPACE_NAME,
+ SystemKeyspace.AVAILABLE_RANGES_V2);
+
+ newInstance.executeInternal(cql,
+ KEYSPACE,
+ fullSet.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()),
+ partialSet.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()));
+
+ // We expect bootstrap to throw an exception on node3 w/the seen ranges we've inserted
+ cluster.run(asList(pullSchemaFrom(cluster.get(1)),
+ bootstrap()),
+ newInstance.config().num());
+ }
+ catch (RuntimeException rte)
+ {
+ if (rte.getMessage().contains("Discovered existing bootstrap data"))
+ sawException = true;
+ }
+ Assert.assertTrue("Expected to see a RuntimeException w/'Discovered existing bootstrap data' in the error message; did not.",
+ sawException);
+ }
+
+ private void bootstrapTest() throws Throwable
{
int originalNodeCount = 2;
int expandedNodeCount = originalNodeCount + 1;
@@ -122,7 +257,7 @@ public class BootstrapTest extends TestBaseImpl
populate(cluster, 0, 100);
- Assert.assertEquals(100, newInstance.executeInternal("SELECT *FROM " + KEYSPACE + ".tbl").length);
+ Assert.assertEquals(100, newInstance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl").length);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org