You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2018/04/30 17:53:01 UTC
[1/6] cassandra git commit: Delay hints store excise by write timeout
to avoid race with decommission
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 bc1f84129 -> b2f6ce961
refs/heads/cassandra-3.11 684120deb -> 32f9e6763
refs/heads/trunk 71a27ee2b -> 9fecf9477
Delay hints store excise by write timeout to avoid race with decommission
patch by Jaydeepkumar Chovatia; reviewed by Aleksey Yeschenko for
CASSANDRA-13740
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b2f6ce96
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b2f6ce96
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b2f6ce96
Branch: refs/heads/cassandra-3.0
Commit: b2f6ce961f38a3e4cd744e102026bf7a471056c9
Parents: bc1f841
Author: Jaydeepkumar Chovatia <ch...@gmail.com>
Authored: Thu Aug 3 15:34:26 2017 -0700
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 17:32:05 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 114 ++++++++++++++++++-
6 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cf470d6..857cf96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.17
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
* Deprecate background repair and probablistic read_repair_chance table options
(CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsCatalog.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java
index 6d01629..d1f6fba 100644
--- a/src/java/org/apache/cassandra/hints/HintsCatalog.java
+++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java
@@ -23,6 +23,7 @@ import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
+import javax.annotation.Nullable;
import com.google.common.collect.ImmutableMap;
@@ -94,6 +95,12 @@ final class HintsCatalog
: store;
}
+ @Nullable
+ HintsStore getNullable(UUID hostId)
+ {
+ return stores.get(hostId);
+ }
+
/**
* Delete all hints for all host ids.
*
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index 9cd4ed3..268ee1f 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -302,7 +302,7 @@ public final class HintsService implements HintsServiceMBean
*/
public void excise(UUID hostId)
{
- HintsStore store = catalog.get(hostId);
+ HintsStore store = catalog.getNullable(hostId);
if (store == null)
return;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java
index c066331..bb3aa0f 100644
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +78,12 @@ final class HintsStore
return new HintsStore(hostId, hintsDirectory, writerParams, descriptors);
}
+ @VisibleForTesting
+ int getDispatchQueueSize()
+ {
+ return dispatchDequeue.size();
+ }
+
InetAddress address()
{
return StorageService.instance.getEndpointForHostId(hostId);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 77fcb81..5f76f7d 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2276,7 +2276,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
UUID hostId = tokenMetadata.getHostId(endpoint);
if (hostId != null && tokenMetadata.isMember(endpoint))
- HintsService.instance.excise(hostId);
+ {
+ // enough time for writes to expire and MessagingService timeout reporter callback to fire, which is where
+ // hints are mostly written from - using getMinRpcTimeout() / 2 for the interval.
+ long delay = DatabaseDescriptor.getMinRpcTimeout() + DatabaseDescriptor.getWriteRpcTimeout();
+ ScheduledExecutors.optionalTasks.schedule(() -> HintsService.instance.excise(hostId), delay, TimeUnit.MILLISECONDS);
+ }
removeEndpoint(endpoint);
tokenMetadata.removeEndpoint(endpoint);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index 51b6aa3..928fd31 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@ -19,16 +19,42 @@ package org.apache.cassandra.hints;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.*;
import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.Assert.*;
+import static org.apache.cassandra.Util.dk;
public class HintsCatalogTest
{
+ private static final String KEYSPACE = "hint_test";
+ private static final String TABLE0 = "table_0";
+ private static final String TABLE1 = "table_1";
+ private static final String TABLE2 = "table_2";
+ private static final int WRITE_BUFFER_SIZE = 256 << 10;
+
+ @BeforeClass
+ public static void defineSchema()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE0),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE2));
+ }
+
@Test
public void loadCompletenessAndOrderTest() throws IOException
{
@@ -43,7 +69,21 @@ public class HintsCatalogTest
}
}
- public static void loadCompletenessAndOrderTest(File directory) throws IOException
+ @Test
+ public void exciseHintFiles() throws IOException
+ {
+ File directory = Files.createTempDirectory(null).toFile();
+ try
+ {
+ exciseHintFiles(directory);
+ }
+ finally
+ {
+ directory.deleteOnExit();
+ }
+ }
+
+ private void loadCompletenessAndOrderTest(File directory) throws IOException
{
UUID hostId1 = UUID.randomUUID();
UUID hostId2 = UUID.randomUUID();
@@ -79,6 +119,39 @@ public class HintsCatalogTest
assertNull(store2.poll());
}
+ private static void exciseHintFiles(File directory) throws IOException
+ {
+ UUID hostId = UUID.randomUUID();
+
+ HintsDescriptor descriptor1 = new HintsDescriptor(hostId, System.currentTimeMillis());
+ HintsDescriptor descriptor2 = new HintsDescriptor(hostId, System.currentTimeMillis() + 1);
+ HintsDescriptor descriptor3 = new HintsDescriptor(hostId, System.currentTimeMillis() + 2);
+ HintsDescriptor descriptor4 = new HintsDescriptor(hostId, System.currentTimeMillis() + 3);
+
+ createHintFile(directory, descriptor1);
+ createHintFile(directory, descriptor2);
+ createHintFile(directory, descriptor3);
+ createHintFile(directory, descriptor4);
+
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(1, catalog.stores().count());
+
+ HintsStore store = catalog.get(hostId);
+
+ //should have 4 hint files
+ assertEquals(4, store.getDispatchQueueSize());
+
+ //excise store as a result it should remove all the hint files
+ catalog.exciseStore(hostId);
+
+ catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(0, catalog.stores().count());
+ store = catalog.get(hostId);
+
+ //should have 0 hint files now
+ assertEquals(0, store.getDispatchQueueSize());
+ }
+
@SuppressWarnings("EmptyTryBlock")
private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
{
@@ -86,4 +159,43 @@ public class HintsCatalogTest
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
+ Mutation mutation = new Mutation(KEYSPACE, dk(key));
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
+ .clustering("column0")
+ .add("val", "value0")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
+ .clustering("column1")
+ .add("val", "value1")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
+ .clustering("column2")
+ .add("val", "value2")
+ .build();
+
+ return mutation;
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[4/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by al...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/32f9e676
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/32f9e676
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/32f9e676
Branch: refs/heads/trunk
Commit: 32f9e6763b7a857a015da0b20af1d3ff486da0c9
Parents: 684120d b2f6ce9
Author: Aleksey Yeshchenko <al...@apple.com>
Authored: Mon Apr 30 18:48:08 2018 +0100
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 18:48:08 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 120 ++++++++++++++++++-
6 files changed, 138 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 5450322,857cf96..c392059
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,15 -1,5 +1,16 @@@
-3.0.17
+3.11.3
+ * Allow existing nodes to use all peers in shadow round (CASSANDRA-13851)
+ * Fix cqlsh to read connection.ssl cqlshrc option again (CASSANDRA-14299)
+ * Downgrade log level to trace for CommitLogSegmentManager (CASSANDRA-14370)
+ * CQL fromJson(null) throws NullPointerException (CASSANDRA-13891)
+ * Serialize empty buffer as empty string for json output format (CASSANDRA-14245)
+ * Allow logging implementation to be interchanged for embedded testing (CASSANDRA-13396)
+ * SASI tokenizer for simple delimiter based entries (CASSANDRA-14247)
+ * Fix Loss of digits when doing CAST from varint/bigint to decimal (CASSANDRA-14170)
+ * RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
+ * Fix wildcard GROUP BY queries (CASSANDRA-14209)
+Merged from 3.0:
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
* Deprecate background repair and probablistic read_repair_chance table options
(CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index a255338,928fd31..68acd0c
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@@ -43,7 -69,21 +69,7 @@@ public class HintsCatalogTes
}
}
- public static void loadCompletenessAndOrderTest(File directory) throws IOException
- @Test
- public void exciseHintFiles() throws IOException
- {
- File directory = Files.createTempDirectory(null).toFile();
- try
- {
- exciseHintFiles(directory);
- }
- finally
- {
- directory.deleteOnExit();
- }
- }
-
+ private void loadCompletenessAndOrderTest(File directory) throws IOException
{
UUID hostId1 = UUID.randomUUID();
UUID hostId2 = UUID.randomUUID();
@@@ -79,37 -119,39 +105,84 @@@
assertNull(store2.poll());
}
+ @Test
+ public void deleteHintsTest() throws IOException
+ {
+ File directory = Files.createTempDirectory(null).toFile();
+ UUID hostId1 = UUID.randomUUID();
+ UUID hostId2 = UUID.randomUUID();
+ long now = System.currentTimeMillis();
+ writeDescriptor(directory, new HintsDescriptor(hostId1, now));
- writeDescriptor(directory, new HintsDescriptor(hostId1, now+1));
- writeDescriptor(directory, new HintsDescriptor(hostId2, now+2));
- writeDescriptor(directory, new HintsDescriptor(hostId2, now+3));
++ writeDescriptor(directory, new HintsDescriptor(hostId1, now + 1));
++ writeDescriptor(directory, new HintsDescriptor(hostId2, now + 2));
++ writeDescriptor(directory, new HintsDescriptor(hostId2, now + 3));
+
+ // load catalog containing two stores (one for each host)
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(2, catalog.stores().count());
+ assertTrue(catalog.hasFiles());
+
+ // delete all hints from store 1
+ assertTrue(catalog.get(hostId1).hasFiles());
+ catalog.deleteAllHints(hostId1);
+ assertFalse(catalog.get(hostId1).hasFiles());
+ // stores are still keepts for each host, even after deleting hints
+ assertEquals(2, catalog.stores().count());
+ assertTrue(catalog.hasFiles());
+
+ // delete all hints from all stores
+ catalog.deleteAllHints();
+ assertEquals(2, catalog.stores().count());
+ assertFalse(catalog.hasFiles());
+ }
+
++ @Test
++ public void exciseHintFiles() throws IOException
++ {
++ File directory = Files.createTempDirectory(null).toFile();
++ try
++ {
++ exciseHintFiles(directory);
++ }
++ finally
++ {
++ directory.deleteOnExit();
++ }
++ }
++
+ private static void exciseHintFiles(File directory) throws IOException
+ {
+ UUID hostId = UUID.randomUUID();
+
+ HintsDescriptor descriptor1 = new HintsDescriptor(hostId, System.currentTimeMillis());
+ HintsDescriptor descriptor2 = new HintsDescriptor(hostId, System.currentTimeMillis() + 1);
+ HintsDescriptor descriptor3 = new HintsDescriptor(hostId, System.currentTimeMillis() + 2);
+ HintsDescriptor descriptor4 = new HintsDescriptor(hostId, System.currentTimeMillis() + 3);
+
+ createHintFile(directory, descriptor1);
+ createHintFile(directory, descriptor2);
+ createHintFile(directory, descriptor3);
+ createHintFile(directory, descriptor4);
+
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(1, catalog.stores().count());
+
+ HintsStore store = catalog.get(hostId);
+
+ //should have 4 hint files
+ assertEquals(4, store.getDispatchQueueSize());
+
+ //excise store as a result it should remove all the hint files
+ catalog.exciseStore(hostId);
+
+ catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(0, catalog.stores().count());
+ store = catalog.get(hostId);
+
+ //should have 0 hint files now
+ assertEquals(0, store.getDispatchQueueSize());
+ }
+
@SuppressWarnings("EmptyTryBlock")
private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
{
@@@ -117,4 -159,43 +190,43 @@@
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
- Mutation mutation = new Mutation(KEYSPACE, dk(key));
++ Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key));
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
- .clustering("column0")
- .add("val", "value0")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE0))
++ .timestamp(now)
++ .row("column0")
++ .add("val", "value0");
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
- .clustering("column1")
- .add("val", "value1")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE1))
++ .timestamp(now + 1)
++ .row("column1")
++ .add("val", "value1");
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
- .clustering("column2")
- .add("val", "value2")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE2))
++ .timestamp(now + 2)
++ .row("column2")
++ .add("val", "value2");
+
- return mutation;
++ return builder.build();
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[3/6] cassandra git commit: Delay hints store excise by write timeout
to avoid race with decommission
Posted by al...@apache.org.
Delay hints store excise by write timeout to avoid race with decommission
patch by Jaydeepkumar Chovatia; reviewed by Aleksey Yeschenko for
CASSANDRA-13740
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b2f6ce96
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b2f6ce96
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b2f6ce96
Branch: refs/heads/trunk
Commit: b2f6ce961f38a3e4cd744e102026bf7a471056c9
Parents: bc1f841
Author: Jaydeepkumar Chovatia <ch...@gmail.com>
Authored: Thu Aug 3 15:34:26 2017 -0700
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 17:32:05 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 114 ++++++++++++++++++-
6 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cf470d6..857cf96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.17
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
* Deprecate background repair and probablistic read_repair_chance table options
(CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsCatalog.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java
index 6d01629..d1f6fba 100644
--- a/src/java/org/apache/cassandra/hints/HintsCatalog.java
+++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java
@@ -23,6 +23,7 @@ import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
+import javax.annotation.Nullable;
import com.google.common.collect.ImmutableMap;
@@ -94,6 +95,12 @@ final class HintsCatalog
: store;
}
+ @Nullable
+ HintsStore getNullable(UUID hostId)
+ {
+ return stores.get(hostId);
+ }
+
/**
* Delete all hints for all host ids.
*
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index 9cd4ed3..268ee1f 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -302,7 +302,7 @@ public final class HintsService implements HintsServiceMBean
*/
public void excise(UUID hostId)
{
- HintsStore store = catalog.get(hostId);
+ HintsStore store = catalog.getNullable(hostId);
if (store == null)
return;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java
index c066331..bb3aa0f 100644
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +78,12 @@ final class HintsStore
return new HintsStore(hostId, hintsDirectory, writerParams, descriptors);
}
+ @VisibleForTesting
+ int getDispatchQueueSize()
+ {
+ return dispatchDequeue.size();
+ }
+
InetAddress address()
{
return StorageService.instance.getEndpointForHostId(hostId);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 77fcb81..5f76f7d 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2276,7 +2276,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
UUID hostId = tokenMetadata.getHostId(endpoint);
if (hostId != null && tokenMetadata.isMember(endpoint))
- HintsService.instance.excise(hostId);
+ {
+ // enough time for writes to expire and MessagingService timeout reporter callback to fire, which is where
+ // hints are mostly written from - using getMinRpcTimeout() / 2 for the interval.
+ long delay = DatabaseDescriptor.getMinRpcTimeout() + DatabaseDescriptor.getWriteRpcTimeout();
+ ScheduledExecutors.optionalTasks.schedule(() -> HintsService.instance.excise(hostId), delay, TimeUnit.MILLISECONDS);
+ }
removeEndpoint(endpoint);
tokenMetadata.removeEndpoint(endpoint);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index 51b6aa3..928fd31 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@ -19,16 +19,42 @@ package org.apache.cassandra.hints;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.*;
import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.Assert.*;
+import static org.apache.cassandra.Util.dk;
public class HintsCatalogTest
{
+ private static final String KEYSPACE = "hint_test";
+ private static final String TABLE0 = "table_0";
+ private static final String TABLE1 = "table_1";
+ private static final String TABLE2 = "table_2";
+ private static final int WRITE_BUFFER_SIZE = 256 << 10;
+
+ @BeforeClass
+ public static void defineSchema()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE0),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE2));
+ }
+
@Test
public void loadCompletenessAndOrderTest() throws IOException
{
@@ -43,7 +69,21 @@ public class HintsCatalogTest
}
}
- public static void loadCompletenessAndOrderTest(File directory) throws IOException
+ @Test
+ public void exciseHintFiles() throws IOException
+ {
+ File directory = Files.createTempDirectory(null).toFile();
+ try
+ {
+ exciseHintFiles(directory);
+ }
+ finally
+ {
+ directory.deleteOnExit();
+ }
+ }
+
+ private void loadCompletenessAndOrderTest(File directory) throws IOException
{
UUID hostId1 = UUID.randomUUID();
UUID hostId2 = UUID.randomUUID();
@@ -79,6 +119,39 @@ public class HintsCatalogTest
assertNull(store2.poll());
}
+ private static void exciseHintFiles(File directory) throws IOException
+ {
+ UUID hostId = UUID.randomUUID();
+
+ HintsDescriptor descriptor1 = new HintsDescriptor(hostId, System.currentTimeMillis());
+ HintsDescriptor descriptor2 = new HintsDescriptor(hostId, System.currentTimeMillis() + 1);
+ HintsDescriptor descriptor3 = new HintsDescriptor(hostId, System.currentTimeMillis() + 2);
+ HintsDescriptor descriptor4 = new HintsDescriptor(hostId, System.currentTimeMillis() + 3);
+
+ createHintFile(directory, descriptor1);
+ createHintFile(directory, descriptor2);
+ createHintFile(directory, descriptor3);
+ createHintFile(directory, descriptor4);
+
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(1, catalog.stores().count());
+
+ HintsStore store = catalog.get(hostId);
+
+ //should have 4 hint files
+ assertEquals(4, store.getDispatchQueueSize());
+
+ //excise store as a result it should remove all the hint files
+ catalog.exciseStore(hostId);
+
+ catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(0, catalog.stores().count());
+ store = catalog.get(hostId);
+
+ //should have 0 hint files now
+ assertEquals(0, store.getDispatchQueueSize());
+ }
+
@SuppressWarnings("EmptyTryBlock")
private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
{
@@ -86,4 +159,43 @@ public class HintsCatalogTest
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
+ Mutation mutation = new Mutation(KEYSPACE, dk(key));
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
+ .clustering("column0")
+ .add("val", "value0")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
+ .clustering("column1")
+ .add("val", "value1")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
+ .clustering("column2")
+ .add("val", "value2")
+ .build();
+
+ return mutation;
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[2/6] cassandra git commit: Delay hints store excise by write timeout
to avoid race with decommission
Posted by al...@apache.org.
Delay hints store excise by write timeout to avoid race with decommission
patch by Jaydeepkumar Chovatia; reviewed by Aleksey Yeschenko for
CASSANDRA-13740
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b2f6ce96
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b2f6ce96
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b2f6ce96
Branch: refs/heads/cassandra-3.11
Commit: b2f6ce961f38a3e4cd744e102026bf7a471056c9
Parents: bc1f841
Author: Jaydeepkumar Chovatia <ch...@gmail.com>
Authored: Thu Aug 3 15:34:26 2017 -0700
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 17:32:05 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 114 ++++++++++++++++++-
6 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cf470d6..857cf96 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.17
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
* Deprecate background repair and probablistic read_repair_chance table options
(CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsCatalog.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsCatalog.java b/src/java/org/apache/cassandra/hints/HintsCatalog.java
index 6d01629..d1f6fba 100644
--- a/src/java/org/apache/cassandra/hints/HintsCatalog.java
+++ b/src/java/org/apache/cassandra/hints/HintsCatalog.java
@@ -23,6 +23,7 @@ import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
+import javax.annotation.Nullable;
import com.google.common.collect.ImmutableMap;
@@ -94,6 +95,12 @@ final class HintsCatalog
: store;
}
+ @Nullable
+ HintsStore getNullable(UUID hostId)
+ {
+ return stores.get(hostId);
+ }
+
/**
* Delete all hints for all host ids.
*
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java
index 9cd4ed3..268ee1f 100644
--- a/src/java/org/apache/cassandra/hints/HintsService.java
+++ b/src/java/org/apache/cassandra/hints/HintsService.java
@@ -302,7 +302,7 @@ public final class HintsService implements HintsServiceMBean
*/
public void excise(UUID hostId)
{
- HintsStore store = catalog.get(hostId);
+ HintsStore store = catalog.getNullable(hostId);
if (store == null)
return;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsStore.java b/src/java/org/apache/cassandra/hints/HintsStore.java
index c066331..bb3aa0f 100644
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentLinkedQueue;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,6 +78,12 @@ final class HintsStore
return new HintsStore(hostId, hintsDirectory, writerParams, descriptors);
}
+ @VisibleForTesting
+ int getDispatchQueueSize()
+ {
+ return dispatchDequeue.size();
+ }
+
InetAddress address()
{
return StorageService.instance.getEndpointForHostId(hostId);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 77fcb81..5f76f7d 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2276,7 +2276,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
UUID hostId = tokenMetadata.getHostId(endpoint);
if (hostId != null && tokenMetadata.isMember(endpoint))
- HintsService.instance.excise(hostId);
+ {
+ // enough time for writes to expire and MessagingService timeout reporter callback to fire, which is where
+ // hints are mostly written from - using getMinRpcTimeout() / 2 for the interval.
+ long delay = DatabaseDescriptor.getMinRpcTimeout() + DatabaseDescriptor.getWriteRpcTimeout();
+ ScheduledExecutors.optionalTasks.schedule(() -> HintsService.instance.excise(hostId), delay, TimeUnit.MILLISECONDS);
+ }
removeEndpoint(endpoint);
tokenMetadata.removeEndpoint(endpoint);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b2f6ce96/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index 51b6aa3..928fd31 100644
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@ -19,16 +19,42 @@ package org.apache.cassandra.hints;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.*;
import com.google.common.collect.ImmutableMap;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.Assert.*;
+import static org.apache.cassandra.Util.dk;
public class HintsCatalogTest
{
+ private static final String KEYSPACE = "hint_test";
+ private static final String TABLE0 = "table_0";
+ private static final String TABLE1 = "table_1";
+ private static final String TABLE2 = "table_2";
+ private static final int WRITE_BUFFER_SIZE = 256 << 10;
+
+ @BeforeClass
+ public static void defineSchema()
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE0),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE1),
+ SchemaLoader.standardCFMD(KEYSPACE, TABLE2));
+ }
+
@Test
public void loadCompletenessAndOrderTest() throws IOException
{
@@ -43,7 +69,21 @@ public class HintsCatalogTest
}
}
- public static void loadCompletenessAndOrderTest(File directory) throws IOException
+ @Test
+ public void exciseHintFiles() throws IOException
+ {
+ File directory = Files.createTempDirectory(null).toFile();
+ try
+ {
+ exciseHintFiles(directory);
+ }
+ finally
+ {
+ directory.deleteOnExit();
+ }
+ }
+
+ private void loadCompletenessAndOrderTest(File directory) throws IOException
{
UUID hostId1 = UUID.randomUUID();
UUID hostId2 = UUID.randomUUID();
@@ -79,6 +119,39 @@ public class HintsCatalogTest
assertNull(store2.poll());
}
+ private static void exciseHintFiles(File directory) throws IOException
+ {
+ UUID hostId = UUID.randomUUID();
+
+ HintsDescriptor descriptor1 = new HintsDescriptor(hostId, System.currentTimeMillis());
+ HintsDescriptor descriptor2 = new HintsDescriptor(hostId, System.currentTimeMillis() + 1);
+ HintsDescriptor descriptor3 = new HintsDescriptor(hostId, System.currentTimeMillis() + 2);
+ HintsDescriptor descriptor4 = new HintsDescriptor(hostId, System.currentTimeMillis() + 3);
+
+ createHintFile(directory, descriptor1);
+ createHintFile(directory, descriptor2);
+ createHintFile(directory, descriptor3);
+ createHintFile(directory, descriptor4);
+
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(1, catalog.stores().count());
+
+ HintsStore store = catalog.get(hostId);
+
+ //should have 4 hint files
+ assertEquals(4, store.getDispatchQueueSize());
+
+ //excise store as a result it should remove all the hint files
+ catalog.exciseStore(hostId);
+
+ catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(0, catalog.stores().count());
+ store = catalog.get(hostId);
+
+ //should have 0 hint files now
+ assertEquals(0, store.getDispatchQueueSize());
+ }
+
@SuppressWarnings("EmptyTryBlock")
private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
{
@@ -86,4 +159,43 @@ public class HintsCatalogTest
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
+ Mutation mutation = new Mutation(KEYSPACE, dk(key));
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
+ .clustering("column0")
+ .add("val", "value0")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
+ .clustering("column1")
+ .add("val", "value1")
+ .build();
+
+ new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
+ .clustering("column2")
+ .add("val", "value2")
+ .build();
+
+ return mutation;
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by al...@apache.org.
Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9fecf947
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9fecf947
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9fecf947
Branch: refs/heads/trunk
Commit: 9fecf947729e6113a79cd8069a52082b4c335629
Parents: 71a27ee 32f9e67
Author: Aleksey Yeshchenko <al...@apple.com>
Authored: Mon Apr 30 18:52:43 2018 +0100
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 18:52:43 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 119 ++++++++++++++++++-
6 files changed, 137 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9fecf947/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 15c2c2e,c392059..33c81d1
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -245,6 -10,9 +245,7 @@@
* RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
* Fix wildcard GROUP BY queries (CASSANDRA-14209)
Merged from 3.0:
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
- * Deprecate background repair and probablistic read_repair_chance table options
- (CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
* Fix unbounded validation compactions on repair / revert CASSANDRA-13797 (CASSANDRA-14332)
* Avoid deadlock when running nodetool refresh before node is fully up (CASSANDRA-14310)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9fecf947/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9fecf947/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/hints/HintsStore.java
index bbf57f5,032de5a..9f95884
--- a/src/java/org/apache/cassandra/hints/HintsStore.java
+++ b/src/java/org/apache/cassandra/hints/HintsStore.java
@@@ -77,7 -78,13 +78,13 @@@ final class HintsStor
return new HintsStore(hostId, hintsDirectory, writerParams, descriptors);
}
+ @VisibleForTesting
+ int getDispatchQueueSize()
+ {
+ return dispatchDequeue.size();
+ }
+
- InetAddress address()
+ InetAddressAndPort address()
{
return StorageService.instance.getEndpointForHostId(hostId);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9fecf947/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9fecf947/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index a255338,68acd0c..92cfc71
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@@ -23,6 -24,13 +24,12 @@@ import java.nio.file.Files
import java.util.*;
import com.google.common.collect.ImmutableMap;
+ import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.Schema;
+ import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.RowUpdateBuilder;
+ import org.apache.cassandra.schema.KeyspaceParams;
++import org.apache.cassandra.schema.Schema;
+ import org.apache.cassandra.utils.FBUtilities;
+ import org.junit.BeforeClass;
import org.junit.Test;
import static junit.framework.Assert.*;
@@@ -117,4 -190,43 +189,43 @@@ public class HintsCatalogTes
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
+ Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key));
+
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE0))
++ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE0))
+ .timestamp(now)
+ .row("column0")
+ .add("val", "value0");
+
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE1))
++ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE1))
+ .timestamp(now + 1)
+ .row("column1")
+ .add("val", "value1");
+
- builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE2))
++ builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE2))
+ .timestamp(now + 2)
+ .row("column2")
+ .add("val", "value2");
+
+ return builder.build();
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by al...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/32f9e676
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/32f9e676
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/32f9e676
Branch: refs/heads/cassandra-3.11
Commit: 32f9e6763b7a857a015da0b20af1d3ff486da0c9
Parents: 684120d b2f6ce9
Author: Aleksey Yeshchenko <al...@apple.com>
Authored: Mon Apr 30 18:48:08 2018 +0100
Committer: Aleksey Yeshchenko <al...@apple.com>
Committed: Mon Apr 30 18:48:08 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/hints/HintsCatalog.java | 7 ++
.../apache/cassandra/hints/HintsService.java | 2 +-
.../org/apache/cassandra/hints/HintsStore.java | 7 ++
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/hints/HintsCatalogTest.java | 120 ++++++++++++++++++-
6 files changed, 138 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 5450322,857cf96..c392059
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,15 -1,5 +1,16 @@@
-3.0.17
+3.11.3
+ * Allow existing nodes to use all peers in shadow round (CASSANDRA-13851)
+ * Fix cqlsh to read connection.ssl cqlshrc option again (CASSANDRA-14299)
+ * Downgrade log level to trace for CommitLogSegmentManager (CASSANDRA-14370)
+ * CQL fromJson(null) throws NullPointerException (CASSANDRA-13891)
+ * Serialize empty buffer as empty string for json output format (CASSANDRA-14245)
+ * Allow logging implementation to be interchanged for embedded testing (CASSANDRA-13396)
+ * SASI tokenizer for simple delimiter based entries (CASSANDRA-14247)
+ * Fix Loss of digits when doing CAST from varint/bigint to decimal (CASSANDRA-14170)
+ * RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
+ * Fix wildcard GROUP BY queries (CASSANDRA-14209)
+Merged from 3.0:
+ * Delay hints store excise by write timeout to avoid race with decommission (CASSANDRA-13740)
* Deprecate background repair and probablistic read_repair_chance table options
(CASSANDRA-13910)
* Add missed CQL keywords to documentation (CASSANDRA-14359)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/hints/HintsService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/hints/HintsStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/32f9e676/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
index a255338,928fd31..68acd0c
--- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
+++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java
@@@ -43,7 -69,21 +69,7 @@@ public class HintsCatalogTes
}
}
- public static void loadCompletenessAndOrderTest(File directory) throws IOException
- @Test
- public void exciseHintFiles() throws IOException
- {
- File directory = Files.createTempDirectory(null).toFile();
- try
- {
- exciseHintFiles(directory);
- }
- finally
- {
- directory.deleteOnExit();
- }
- }
-
+ private void loadCompletenessAndOrderTest(File directory) throws IOException
{
UUID hostId1 = UUID.randomUUID();
UUID hostId2 = UUID.randomUUID();
@@@ -79,37 -119,39 +105,84 @@@
assertNull(store2.poll());
}
+ @Test
+ public void deleteHintsTest() throws IOException
+ {
+ File directory = Files.createTempDirectory(null).toFile();
+ UUID hostId1 = UUID.randomUUID();
+ UUID hostId2 = UUID.randomUUID();
+ long now = System.currentTimeMillis();
+ writeDescriptor(directory, new HintsDescriptor(hostId1, now));
- writeDescriptor(directory, new HintsDescriptor(hostId1, now+1));
- writeDescriptor(directory, new HintsDescriptor(hostId2, now+2));
- writeDescriptor(directory, new HintsDescriptor(hostId2, now+3));
++ writeDescriptor(directory, new HintsDescriptor(hostId1, now + 1));
++ writeDescriptor(directory, new HintsDescriptor(hostId2, now + 2));
++ writeDescriptor(directory, new HintsDescriptor(hostId2, now + 3));
+
+ // load catalog containing two stores (one for each host)
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(2, catalog.stores().count());
+ assertTrue(catalog.hasFiles());
+
+ // delete all hints from store 1
+ assertTrue(catalog.get(hostId1).hasFiles());
+ catalog.deleteAllHints(hostId1);
+ assertFalse(catalog.get(hostId1).hasFiles());
+ // stores are still keepts for each host, even after deleting hints
+ assertEquals(2, catalog.stores().count());
+ assertTrue(catalog.hasFiles());
+
+ // delete all hints from all stores
+ catalog.deleteAllHints();
+ assertEquals(2, catalog.stores().count());
+ assertFalse(catalog.hasFiles());
+ }
+
++ @Test
++ public void exciseHintFiles() throws IOException
++ {
++ File directory = Files.createTempDirectory(null).toFile();
++ try
++ {
++ exciseHintFiles(directory);
++ }
++ finally
++ {
++ directory.deleteOnExit();
++ }
++ }
++
+ private static void exciseHintFiles(File directory) throws IOException
+ {
+ UUID hostId = UUID.randomUUID();
+
+ HintsDescriptor descriptor1 = new HintsDescriptor(hostId, System.currentTimeMillis());
+ HintsDescriptor descriptor2 = new HintsDescriptor(hostId, System.currentTimeMillis() + 1);
+ HintsDescriptor descriptor3 = new HintsDescriptor(hostId, System.currentTimeMillis() + 2);
+ HintsDescriptor descriptor4 = new HintsDescriptor(hostId, System.currentTimeMillis() + 3);
+
+ createHintFile(directory, descriptor1);
+ createHintFile(directory, descriptor2);
+ createHintFile(directory, descriptor3);
+ createHintFile(directory, descriptor4);
+
+ HintsCatalog catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(1, catalog.stores().count());
+
+ HintsStore store = catalog.get(hostId);
+
+ //should have 4 hint files
+ assertEquals(4, store.getDispatchQueueSize());
+
+ //excise store as a result it should remove all the hint files
+ catalog.exciseStore(hostId);
+
+ catalog = HintsCatalog.load(directory, ImmutableMap.of());
+ assertEquals(0, catalog.stores().count());
+ store = catalog.get(hostId);
+
+ //should have 0 hint files now
+ assertEquals(0, store.getDispatchQueueSize());
+ }
+
@SuppressWarnings("EmptyTryBlock")
private static void writeDescriptor(File directory, HintsDescriptor descriptor) throws IOException
{
@@@ -117,4 -159,43 +190,43 @@@
{
}
}
+
+ private static Mutation createMutation(String key, long now)
+ {
- Mutation mutation = new Mutation(KEYSPACE, dk(key));
++ Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key));
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE0), now, mutation)
- .clustering("column0")
- .add("val", "value0")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE0))
++ .timestamp(now)
++ .row("column0")
++ .add("val", "value0");
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE1), now + 1, mutation)
- .clustering("column1")
- .add("val", "value1")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE1))
++ .timestamp(now + 1)
++ .row("column1")
++ .add("val", "value1");
+
- new RowUpdateBuilder(Schema.instance.getCFMetaData(KEYSPACE, TABLE2), now + 2, mutation)
- .clustering("column2")
- .add("val", "value2")
- .build();
++ builder.update(Schema.instance.getCFMetaData(KEYSPACE, TABLE2))
++ .timestamp(now + 2)
++ .row("column2")
++ .add("val", "value2");
+
- return mutation;
++ return builder.build();
+ }
+
+ @SuppressWarnings("EmptyTryBlock")
+ private static void createHintFile(File directory, HintsDescriptor descriptor) throws IOException
+ {
+ try (HintsWriter writer = HintsWriter.create(directory, descriptor))
+ {
+ ByteBuffer writeBuffer = ByteBuffer.allocateDirect(WRITE_BUFFER_SIZE);
+ try (HintsWriter.Session session = writer.newSession(writeBuffer))
+ {
+ long now = FBUtilities.timestampMicros();
+ Mutation mutation = createMutation("testSerializer", now);
+ Hint hint = Hint.create(mutation, now / 1000);
+
+ session.append(hint);
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org