You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2017/02/23 14:22:08 UTC
[5/5] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Merge branch 'cassandra-3.0' into trunk
* cassandra-3.0:
Legacy deserializer can create unexpected boundary range tombstones
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/831c05b1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/831c05b1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/831c05b1
Branch: refs/heads/trunk
Commit: 831c05b1c047cdccbc6a82fe0d3f72302fdfb18a
Parents: cd29d44 ab71748
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Feb 23 15:21:47 2017 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu Feb 23 15:21:47 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/rows/RangeTombstoneMarker.java | 1 -
.../apache/cassandra/service/DataResolver.java | 31 +++--
.../cassandra/service/DataResolverTest.java | 129 ++++++++++++++++++-
4 files changed, 142 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/831c05b1/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 0641011,386029e..f541c9a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,55 -1,19 +1,56 @@@
-3.0.12
+4.0
+ * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520)
+ * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226)
+ * Remove unused method (CASSANDRA-13227)
+ * Fix minor bugs related to #9143 (CASSANDRA-13217)
+ * Output warning if user increases RF (CASSANDRA-13079)
+ * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081)
+ * Add support for + and - operations on dates (CASSANDRA-11936)
+ * Fix consistency of incrementally repaired data (CASSANDRA-9143)
+ * Increase commitlog version (CASSANDRA-13161)
+ * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425)
+ * Refactor ColumnCondition (CASSANDRA-12981)
+ * Parallelize streaming of different keyspaces (CASSANDRA-4663)
+ * Improved compactions metrics (CASSANDRA-13015)
+ * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
+ * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
+ * Thrift removal (CASSANDRA-11115)
+ * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
+ * Add column definition kind to dropped columns in schema (CASSANDRA-12705)
+ * Add (automate) Nodetool Documentation (CASSANDRA-12672)
+ * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+ * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422)
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+
+
+3.11.0
+ * Fix equality comparisons of columns using the duration type (CASSANDRA-13174)
+ * Move to FastThreadLocalThread and FastThreadLocal (CASSANDRA-13034)
+ * nodetool stopdaemon errors out (CASSANDRA-13030)
+ * Tables in system_distributed should not use gcgs of 0 (CASSANDRA-12954)
+ * Fix primary index calculation for SASI (CASSANDRA-12910)
+ * More fixes to the TokenAllocator (CASSANDRA-12990)
+ * NoReplicationTokenAllocator should work with zero replication factor (CASSANDRA-12983)
+Merged from 3.0:
+ * Legacy deserializer can create unexpected boundary range tombstones (CASSANDRA-13237)
* Remove unnecessary assertion from AntiCompactionTest (CASSANDRA-13070)
* Fix cqlsh COPY for dates before 1900 (CASSANDRA-13185)
-Merged from 2.2
- * Fix flaky LongLeveledCompactionStrategyTest (CASSANDRA-12202)
- * Fix failing COPY TO STDOUT (CASSANDRA-12497)
- * Fix ColumnCounter::countAll behaviour for reverse queries (CASSANDRA-13222)
- * Exceptions encountered calling getSeeds() breaks OTC thread (CASSANDRA-13018)
-Merged from 2.1:
- * Log stacktrace of uncaught exceptions (CASSANDRA-13108)
-
-3.0.11
* Use keyspace replication settings on system.size_estimates table (CASSANDRA-9639)
* Add vm.max_map_count StartupCheck (CASSANDRA-13008)
- * Hint related logging should include the IP address of the destination in addition to
+ * Obfuscate password in stress-graphs (CASSANDRA-12233)
+ * Hint related logging should include the IP address of the destination in addition to
host ID (CASSANDRA-13205)
* Reloading logback.xml does not work (CASSANDRA-13173)
* Lightweight transactions temporarily fail after upgrade from 2.1 to 3.0 (CASSANDRA-13109)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/831c05b1/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/831c05b1/src/java/org/apache/cassandra/service/DataResolver.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/831c05b1/test/unit/org/apache/cassandra/service/DataResolverTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/service/DataResolverTest.java
index 413f032,2f72093..3916a30
--- a/test/unit/org/apache/cassandra/service/DataResolverTest.java
+++ b/test/unit/org/apache/cassandra/service/DataResolverTest.java
@@@ -556,6 -554,73 +556,73 @@@ public class DataResolverTes
assertRepairContainsDeletions(msg2, null, one_two, withExclusiveEndIf(three_four, timestamp2 >= timestamp1), five_six);
}
+ /**
+ * Test cases where a boundary of a source is covered by another source deletion and timestamp on one or both side
+ * of the boundary are equal to the "merged" deletion.
+ * This is a test for CASSANDRA-13237 to make sure we handle this case properly.
+ */
+ @Test
+ public void testRepairRangeTombstoneBoundary() throws UnknownHostException
+ {
+ testRepairRangeTombstoneBoundary(1, 0, 1);
+ messageRecorder.sent.clear();
+ testRepairRangeTombstoneBoundary(1, 1, 0);
+ messageRecorder.sent.clear();
+ testRepairRangeTombstoneBoundary(1, 1, 1);
+ }
+
+ /**
+ * Test for CASSANDRA-13237, checking we don't fail (and handle correctly) the case where a RT boundary has the
+ * same deletion on both side (while is useless but could be created by legacy code pre-CASSANDRA-13237 and could
+ * thus still be sent).
+ */
+ public void testRepairRangeTombstoneBoundary(int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException
+ {
- DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2);
++ DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime());
+ InetAddress peer1 = peer();
+ InetAddress peer2 = peer();
+
+ // 1st "stream"
+ RangeTombstone one_nine = tombstone("0", true , "9", true, timestamp1, nowInSec);
+ UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk)
+ .addRangeTombstone(one_nine)
+ .buildUpdate());
+
+ // 2nd "stream" (build more manually to ensure we have the boundary we want)
+ RangeTombstoneBoundMarker open_one = marker("0", true, true, timestamp2, nowInSec);
+ RangeTombstoneBoundaryMarker boundary_five = boundary("5", false, timestamp2, nowInSec, timestamp3, nowInSec);
+ RangeTombstoneBoundMarker close_nine = marker("9", false, true, timestamp3, nowInSec);
+ UnfilteredPartitionIterator iter2 = iter(dk, open_one, boundary_five, close_nine);
+
+ resolver.preprocess(readResponseMessage(peer1, iter1));
+ resolver.preprocess(readResponseMessage(peer2, iter2));
+
+ boolean shouldHaveRepair = timestamp1 != timestamp2 || timestamp1 != timestamp3;
+
+ // No results, we've only reconciled tombstones.
+ try (PartitionIterator data = resolver.resolve())
+ {
+ assertFalse(data.hasNext());
+ assertRepairFuture(resolver, shouldHaveRepair ? 1 : 0);
+ }
+
+ assertEquals(shouldHaveRepair? 1 : 0, messageRecorder.sent.size());
+
+ if (!shouldHaveRepair)
+ return;
+
+ MessageOut msg = getSentMessage(peer2);
+ assertRepairMetadata(msg);
+ assertRepairContainsNoColumns(msg);
+
+ RangeTombstone expected = timestamp1 != timestamp2
+ // We've repaired the 1st part
+ ? tombstone("0", true, "5", false, timestamp1, nowInSec)
+ // We've repaired the 2nd part
+ : tombstone("5", true, "9", true, timestamp1, nowInSec);
+ assertRepairContainsDeletions(msg, null, expected);
+ }
+
// Forces the start to be exclusive if the condition holds
private static RangeTombstone withExclusiveStartIf(RangeTombstone rt, boolean condition)
{
@@@ -883,17 -940,43 +950,43 @@@
private RangeTombstone tombstone(Object start, boolean inclusiveStart, Object end, boolean inclusiveEnd, long markedForDeleteAt, int localDeletionTime)
{
- Kind startKind = inclusiveStart ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND;
- Kind endKind = inclusiveEnd ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND;
-
- ClusteringBound startBound = ClusteringBound.create(startKind, cfm.comparator.make(start).getRawValues());
- ClusteringBound endBound = ClusteringBound.create(endKind, cfm.comparator.make(end).getRawValues());
- RangeTombstone.Bound startBound = rtBound(start, true, inclusiveStart);
- RangeTombstone.Bound endBound = rtBound(end, false, inclusiveEnd);
++ ClusteringBound startBound = rtBound(start, true, inclusiveStart);
++ ClusteringBound endBound = rtBound(end, false, inclusiveEnd);
return new RangeTombstone(Slice.make(startBound, endBound), new DeletionTime(markedForDeleteAt, localDeletionTime));
}
- private UnfilteredPartitionIterator fullPartitionDelete(TableMetadata cfm, DecoratedKey dk, long timestamp, int nowInSec)
- private RangeTombstone.Bound rtBound(Object value, boolean isStart, boolean inclusive)
++ private ClusteringBound rtBound(Object value, boolean isStart, boolean inclusive)
{
- return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(cfm, dk, timestamp, nowInSec).unfilteredIterator());
- RangeTombstone.Bound.Kind kind = isStart
- ? (inclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND)
- : (inclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND);
++ ClusteringBound.Kind kind = isStart
++ ? (inclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND)
++ : (inclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND);
+
- return new RangeTombstone.Bound(kind, cfm.comparator.make(value).getRawValues());
++ return ClusteringBound.create(kind, cfm.comparator.make(value).getRawValues());
+ }
+
- private RangeTombstone.Bound rtBoundary(Object value, boolean inclusiveOnEnd)
++ private ClusteringBoundary rtBoundary(Object value, boolean inclusiveOnEnd)
+ {
- RangeTombstone.Bound.Kind kind = inclusiveOnEnd
- ? Kind.INCL_END_EXCL_START_BOUNDARY
- : Kind.EXCL_END_INCL_START_BOUNDARY;
- return new RangeTombstone.Bound(kind, cfm.comparator.make(value).getRawValues());
++ ClusteringBound.Kind kind = inclusiveOnEnd
++ ? Kind.INCL_END_EXCL_START_BOUNDARY
++ : Kind.EXCL_END_INCL_START_BOUNDARY;
++ return ClusteringBoundary.create(kind, cfm.comparator.make(value).getRawValues());
+ }
+
+ private RangeTombstoneBoundMarker marker(Object value, boolean isStart, boolean inclusive, long markedForDeleteAt, int localDeletionTime)
+ {
+ return new RangeTombstoneBoundMarker(rtBound(value, isStart, inclusive), new DeletionTime(markedForDeleteAt, localDeletionTime));
+ }
+
+ private RangeTombstoneBoundaryMarker boundary(Object value, boolean inclusiveOnEnd, long markedForDeleteAt1, int localDeletionTime1, long markedForDeleteAt2, int localDeletionTime2)
+ {
+ return new RangeTombstoneBoundaryMarker(rtBoundary(value, inclusiveOnEnd),
+ new DeletionTime(markedForDeleteAt1, localDeletionTime1),
+ new DeletionTime(markedForDeleteAt2, localDeletionTime2));
+ }
+
- private UnfilteredPartitionIterator fullPartitionDelete(CFMetaData cfm, DecoratedKey dk, long timestamp, int nowInSec)
++ private UnfilteredPartitionIterator fullPartitionDelete(TableMetadata table, DecoratedKey dk, long timestamp, int nowInSec)
+ {
- return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(cfm, dk, timestamp, nowInSec).unfilteredIterator(), false);
++ return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(table, dk, timestamp, nowInSec).unfilteredIterator());
}
private static class MessageRecorder implements IMessageSink
@@@ -913,6 -996,28 +1006,28 @@@
private UnfilteredPartitionIterator iter(PartitionUpdate update)
{
- return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator(), false);
+ return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator());
}
+
+ private UnfilteredPartitionIterator iter(DecoratedKey key, Unfiltered... unfiltereds)
+ {
+ SortedSet<Unfiltered> s = new TreeSet<>(cfm.comparator);
+ Collections.addAll(s, unfiltereds);
+ final Iterator<Unfiltered> iterator = s.iterator();
+
+ UnfilteredRowIterator rowIter = new AbstractUnfilteredRowIterator(cfm,
+ key,
+ DeletionTime.LIVE,
- cfm.partitionColumns(),
++ cfm.regularAndStaticColumns(),
+ Rows.EMPTY_STATIC_ROW,
+ false,
+ EncodingStats.NO_STATS)
+ {
+ protected Unfiltered computeNext()
+ {
+ return iterator.hasNext() ? iterator.next() : endOfData();
+ }
+ };
- return new SingletonUnfilteredPartitionIterator(rowIter, false);
++ return new SingletonUnfilteredPartitionIterator(rowIter);
+ }
}