You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bd...@apache.org on 2018/11/06 19:25:30 UTC
[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a6a9dce1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a6a9dce1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a6a9dce1
Branch: refs/heads/cassandra-3.11
Commit: a6a9dce157a4ed14e7d08e854c504423dd199daa
Parents: 69f8cc7 d60c783
Author: Blake Eggleston <bd...@gmail.com>
Authored: Tue Nov 6 11:17:47 2018 -0800
Committer: Blake Eggleston <bd...@gmail.com>
Committed: Tue Nov 6 11:19:04 2018 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
src/java/org/apache/cassandra/db/Slice.java | 25 +-----
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 9 ++
.../io/sstable/metadata/MetadataCollector.java | 25 +++---
.../io/sstable/metadata/StatsMetadata.java | 14 ++-
.../mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mc-1-big-Data.db | Bin 0 -> 65 bytes
.../mc-1-big-Digest.crc32 | 1 +
.../mc-1-big-Filter.db | Bin 0 -> 16 bytes
.../mc-1-big-Index.db | Bin 0 -> 8 bytes
.../mc-1-big-Statistics.db | Bin 0 -> 4789 bytes
.../mc-1-big-Summary.db | Bin 0 -> 56 bytes
.../mc-1-big-TOC.txt | 8 ++
.../db/SinglePartitionSliceCommandTest.java | 87 +++++++++++++++++++
.../cassandra/io/sstable/LegacySSTableTest.java | 34 +++++++-
16 files changed, 166 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 03abb5b,0fb1b86..f923fa0
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,5 -1,5 +1,6 @@@
-3.0.18
+3.11.4
+Merged from 3.0:
+ * Sstable min/max metadata can cause data loss (CASSANDRA-14861)
* Dropped columns can cause reverse sstable iteration to return prematurely (CASSANDRA-14838)
* Legacy sstables with multi block range tombstones create invalid bound sequences (CASSANDRA-14823)
* Expand range tombstone validation checks to multiple interim request stages (CASSANDRA-14824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/db/Slice.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index e61f4b3,f48d0a6..a618c96
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -19,8 -19,8 +19,9 @@@ package org.apache.cassandra.io.sstable
import java.nio.ByteBuffer;
import java.util.ArrayList;
+ import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumMap;
import java.util.List;
import java.util.Map;
@@@ -93,8 -95,8 +97,8 @@@ public class MetadataCollector implemen
protected double compressionRatio = NO_COMPRESSION_RATIO;
protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogramBuilder();
protected int sstableLevel;
- protected ByteBuffer[] minClusteringValues;
- protected ByteBuffer[] maxClusteringValues;
- private ClusteringPrefix minClustering = Slice.Bound.TOP;
- private ClusteringPrefix maxClustering = Slice.Bound.BOTTOM;
++ private ClusteringPrefix minClustering = ClusteringBound.TOP;
++ private ClusteringPrefix maxClustering = ClusteringBound.BOTTOM;
protected boolean hasLegacyCounterShards = false;
protected long totalColumnsSet;
protected long totalRows;
@@@ -277,7 -271,8 +273,8 @@@
public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, SerializationHeader header)
{
+ Preconditions.checkState(comparator.compare(maxClustering, minClustering) >= 0);
- Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
+ Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
estimatedCellPerPartitionCount,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index 7ad6198,2891687..1bdbcb2
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@@ -37,11 -43,12 +43,13 @@@ import org.apache.cassandra.SchemaLoade
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
+ import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+ import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.filter.AbstractClusteringIndexFilter;
import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@@ -387,4 -388,81 +397,81 @@@ public class SinglePartitionSliceComman
Assert.assertNotNull(ret);
Assert.assertFalse(ret.isEmpty());
}
+
+
+ public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q)
+ {
+ SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement;
+
+ List<Unfiltered> unfiltereds = new ArrayList<>();
+ SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT, FBUtilities.nowInSeconds());
+ Assert.assertEquals(1, query.commands.size());
+ SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands);
- try (ReadOrderGroup group = ReadOrderGroup.forCommand(command);
- UnfilteredPartitionIterator partitions = command.executeLocally(group))
++ try (ReadExecutionController controller = ReadExecutionController.forCommand(command);
++ UnfilteredPartitionIterator partitions = command.executeLocally(controller))
+ {
+ assert partitions.hasNext();
+ try (UnfilteredRowIterator partition = partitions.next())
+ {
+ while (partition.hasNext())
+ {
+ Unfiltered next = partition.next();
+ unfiltereds.add(next);
+ }
+ }
+ assert !partitions.hasNext();
+ }
+ return unfiltereds;
+ }
+
+ private static void assertQueryReturnsSingleRT(String query)
+ {
+ List<Unfiltered> unfiltereds = getUnfilteredsFromSinglePartition(query);
+ Assert.assertEquals(2, unfiltereds.size());
+ Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+ Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+ Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+ Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+ }
+
+ private static ByteBuffer bb(int v)
+ {
+ return Int32Type.instance.decompose(v);
+ }
+
+ /**
+ * tests the bug raised in CASSANDRA-14861, where the sstable min/max can
+ * exclude range tombstones for clustering ranges not also covered by rows
+ */
+ @Test
+ public void sstableFiltering()
+ {
+ QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+ CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max");
+ ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+
+ QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)");
+ QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1");
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1");
+ cfs.forceBlockingFlush();
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1");
+
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names
+
+ cfs.truncateBlocking();
+
+ long nowMillis = System.currentTimeMillis();
- Slice slice = Slice.make(new Clustering(bb(2), bb(3)), new Clustering(bb(10), bb(10)));
++ Slice slice = Slice.make(Clustering.make(bb(2), bb(3)), Clustering.make(bb(10), bb(10)));
+ RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis),
+ Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis))));
+ PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(), 1);
+ update.add(rt);
+ new Mutation(update).apply();
+
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2");
+ cfs.forceBlockingFlush();
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2");
+ assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2 AND c3=2"); // clustering names
+
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index dae7823,5c65b31..bd51c0f
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@@ -35,12 -36,20 +36,21 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.cql3.QueryOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+ import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.ReadOrderGroup;
++import org.apache.cassandra.db.ReadExecutionController;
+ import org.apache.cassandra.db.SinglePartitionReadCommand;
+ import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
import org.apache.cassandra.db.compaction.Verifier;
+ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+ import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+ import org.apache.cassandra.db.rows.Unfiltered;
+ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@@@ -278,7 -268,29 +289,28 @@@ public class LegacySSTableTes
}
}
-
@Test
+ public void testInaccurateSSTableMinMax() throws Exception
+ {
+ QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max (k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+ loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", "");
+
+ /*
+ sstable has the following mutations:
+ INSERT INTO legacy_tables.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 4, 4, 4, 4)
+ DELETE FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1<3
+ */
+
+ String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1";
+ List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query);
+ Assert.assertEquals(2, unfiltereds.size());
+ Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+ Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+ Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+ Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+ }
+
+ @Test
public void testVerifyOldSSTables() throws Exception
{
for (String legacyVersion : legacyVersions)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org