You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2016/05/12 13:23:06 UTC
[01/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 e614433ce -> 849a43869
refs/heads/cassandra-3.0 e9657a43a -> 78a3d2bba
refs/heads/cassandra-3.7 902877a65 -> c9ac0506c
refs/heads/trunk fbc07eb27 -> bd39d73a6
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 30ed85b..ca7fe82 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.FileDataInput;
@@ -49,7 +50,7 @@ public class MetadataSerializer implements IMetadataSerializer
{
private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
// sort components by type
List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
@@ -66,12 +67,12 @@ public class MetadataSerializer implements IMetadataSerializer
out.writeInt(type.ordinal());
// serialize position
out.writeInt(lastPosition);
- lastPosition += type.serializer.serializedSize(component);
+ lastPosition += type.serializer.serializedSize(component, version);
}
// serialize components
for (MetadataComponent component : sortedComponents)
{
- component.getType().serializer.serialize(component, out);
+ component.getType().serializer.serialize(component, version, out);
}
}
@@ -153,7 +154,7 @@ public class MetadataSerializer implements IMetadataSerializer
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(tmpDescriptor.filenameFor(Component.STATS))))
{
- serialize(currentComponents, out);
+ serialize(currentComponents, descriptor.version, out);
out.flush();
}
// we cant move a file on top of another file in windows:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index f2eb1af..3d48e34 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -43,7 +43,8 @@ public class StatsMetadata extends MetadataComponent
public final EstimatedHistogram estimatedRowSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
public final int maxLocalDeletionTime;
@@ -57,7 +58,8 @@ public class StatsMetadata extends MetadataComponent
public StatsMetadata(EstimatedHistogram estimatedRowSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
int maxLocalDeletionTime,
@@ -71,7 +73,8 @@ public class StatsMetadata extends MetadataComponent
{
this.estimatedRowSize = estimatedRowSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
this.maxLocalDeletionTime = maxLocalDeletionTime;
@@ -117,7 +120,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -134,7 +138,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -157,7 +162,8 @@ public class StatsMetadata extends MetadataComponent
return new EqualsBuilder()
.append(estimatedRowSize, that.estimatedRowSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
@@ -177,7 +183,8 @@ public class StatsMetadata extends MetadataComponent
return new HashCodeBuilder()
.append(estimatedRowSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
.append(maxLocalDeletionTime)
@@ -193,12 +200,12 @@ public class StatsMetadata extends MetadataComponent
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component) throws IOException
+ public int serializedSize(StatsMetadata component, Version version) throws IOException
{
int size = 0;
size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition, TypeSizes.NATIVE);
+ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
@@ -211,14 +218,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
size += 2 + columnName.remaining(); // with short length
size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
return size;
}
- public void serialize(StatsMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
{
EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
out.writeInt(component.maxLocalDeletionTime);
@@ -233,13 +242,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
out.writeBoolean(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
public StatsMetadata deserialize(Version version, DataInput in) throws IOException
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -264,9 +276,12 @@ public class StatsMetadata extends MetadataComponent
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
return new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index 603732b..4ca078b 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -71,12 +71,12 @@ public class ValidationMetadata extends MetadataComponent
public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
{
- public int serializedSize(ValidationMetadata component) throws IOException
+ public int serializedSize(ValidationMetadata component, Version version) throws IOException
{
return TypeSizes.NATIVE.sizeof(component.partitioner) + 8;
}
- public void serialize(ValidationMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(ValidationMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeUTF(component.partitioner);
out.writeDouble(component.bloomFilterFPChance);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
deleted file mode 100644
index 1a15d6f..0000000
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public abstract class DiskAwareRunnable extends WrappedRunnable
-{
- protected Directories.DataDirectory getWriteDirectory(long writeSize)
- {
- Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
- if (directory == null)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
-
- return directory;
- }
-
- /**
- * Get sstable directories for the CF.
- * @return Directories instance for the CF.
- */
- protected abstract Directories getDirectories();
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index dd56b8b..6911ec6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -122,14 +122,11 @@ public class StreamReceiveTask extends StreamTask
for (SSTableWriter writer : task.sstables)
writer.abort();
task.sstables.clear();
- task.session.taskCompleted(task);
return;
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
File lockfiledir = cfs.directories.getWriteableLocationAsFile(task.sstables.size() * 256L);
- if (lockfiledir == null)
- throw new IOError(new IOException("All disks full"));
StreamLockfile lockfile = new StreamLockfile(lockfiledir, UUID.randomUUID());
lockfile.create(task.sstables);
List<SSTableReader> readers = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index 2665f40..8319014 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -70,7 +70,8 @@ public class SSTableMetadataViewer
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
out.println("Estimated tombstone drop times:");
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 1b4edee..f9b4156 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -452,7 +452,7 @@ public class CommitLogStressTest
int cells = 0;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index c377a21..0c46061 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -59,7 +59,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
Mutation mutation;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 93b88b4..adeb778 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.MockSchema;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
@@ -263,15 +264,15 @@ public class TrackerTest
Tracker tracker = cfs.getTracker();
tracker.subscribe(listener);
- Memtable prev1 = tracker.switchMemtable(true);
+ Memtable prev1 = tracker.switchMemtable(true, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write1 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier1 = cfs.keyspace.writeOrder.newBarrier();
- prev1.setDiscarding(barrier1, new AtomicReference<ReplayPosition>());
+ prev1.setDiscarding(barrier1, new AtomicReference<>(CommitLog.instance.getContext()));
barrier1.issue();
- Memtable prev2 = tracker.switchMemtable(false);
+ Memtable prev2 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write2 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier2 = cfs.keyspace.writeOrder.newBarrier();
- prev2.setDiscarding(barrier2, new AtomicReference<ReplayPosition>());
+ prev2.setDiscarding(barrier2, new AtomicReference<>(CommitLog.instance.getContext()));
barrier2.issue();
Memtable cur = tracker.getView().getCurrentMemtable();
OpOrder.Group writecur = cfs.keyspace.writeOrder.getCurrent();
@@ -297,6 +298,9 @@ public class TrackerTest
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
tracker.replaceFlushed(prev2, reader);
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(reader);
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
@@ -307,16 +311,17 @@ public class TrackerTest
tracker = cfs.getTracker();
listener = new MockListener(false);
tracker.subscribe(listener);
- prev1 = tracker.switchMemtable(false);
+ prev1 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
tracker.replaceFlushed(prev1, reader);
+ tracker.permitCompactionOfFlushed(reader);
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
+ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 32a81e2..5706598 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -208,6 +208,6 @@ public class ViewTest
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
+ Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 931422b..3bef89e 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -113,7 +113,7 @@ public class CompressedRandomAccessReaderTest
ChannelProxy channel = new ChannelProxy(f);
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
SequentialWriter writer = compressed
? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
: SequentialWriter.open(f);
@@ -166,7 +166,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
@@ -251,7 +251,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 27b866d..43c44fd 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -85,7 +85,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index eda4f17..19fa7c4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.EnumSet;
@@ -27,12 +28,16 @@ import java.util.Set;
import com.google.common.collect.Sets;
import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
@@ -45,18 +50,51 @@ public class MetadataSerializerTest
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
+
+ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
+
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
+ // Serialize to tmp file
+ File statsFile = File.createTempFile(Component.STATS.name, null);
+ try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
+ {
+ serializer.serialize(metadata, version, out);
+ }
+ return statsFile;
+ }
+
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
new long[] { 3L, 4L, 5L });
EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
new long[] { 8L, 9L, 10L });
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ ReplayPosition start = new ReplayPosition(11L, 12);
+ ReplayPosition end = new ReplayPosition(15L, 9);
long minTimestamp = 2162517136L;
long maxTimestamp = 4162517136L;
MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
.estimatedRowSize(rowSizes)
.estimatedColumnCount(columnCounts)
- .replayPosition(rp);
+ .commitLogLowerBound(start)
+ .commitLogUpperBound(end);
collector.updateMinTimestamp(minTimestamp);
collector.updateMaxTimestamp(maxTimestamp);
@@ -67,23 +105,35 @@ public class MetadataSerializerTest
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
+ return originalMetadata;
+ }
+
+ @Test
+ public void testLaReadsLb() throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
MetadataSerializer serializer = new MetadataSerializer();
- // Serialize to tmp file
- File statsFile = File.createTempFile(Component.STATS.name, null);
- try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
- {
- serializer.serialize(originalMetadata, out);
- }
+ // Write metadata in two minor formats.
+ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
+ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ // Reading both as earlier version should yield identical results.
+ Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
- assertEquals(originalMetadata.get(type), deserialized.get(type));
+ assertEquals(deserializedLa.get(type), deserializedLb.get(type));
+ if (!originalMetadata.get(type).equals(deserializedLb.get(type)))
+ {
+ // Currently only STATS can be different. Change if no longer the case
+ assertEquals(MetadataType.STATS, type);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
index 8409a26..1f66fb7 100644
--- a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
@@ -157,7 +157,7 @@ public class IntervalTreeTest
public String deserialize(DataInput in) throws IOException { return in.readUTF(); }
public long serializedSize(String v, TypeSizes s) { return v.length(); }
},
- (Constructor<Interval<Integer, String>>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class)
+ (Constructor<Interval<Integer, String>>) (Constructor<?>) Interval.class.getConstructor(Object.class, Object.class, Object.class)
);
DataOutputBuffer out = new DataOutputBuffer();
[12/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
index 0000000,0000000..288663f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
index 0000000,0000000..6a2f28e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
index 0000000,0000000..fb255bd
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++655951031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
index 0000000,0000000..b3094bf
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
index 0000000,0000000..55dcdce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..adb7fc4
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
index 0000000,0000000..5cd0481
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
index 0000000,0000000..7d2bdbe
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3883019031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
index 0000000,0000000..56f29df
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
index 0000000,0000000..dff24fe
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..19d25c9
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
index 0000000,0000000..77a1394
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
index 0000000,0000000..172b695
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++2876949266
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
index 0000000,0000000..59e65ca
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
index 0000000,0000000..4c67715
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..56c95a8
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
index 0000000,0000000..00a7a39
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..f9f2fff
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++1214766167
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
index 0000000,0000000..d094f73
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..33c33f2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index fea83c1,0c46061..e690785
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@@ -59,9 -59,9 +59,9 @@@ public class CommitLogTestReplayer exte
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
- FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
+ RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
Mutation mutation;
try
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 7b9b19c,adeb778..b8de711
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@@ -38,8 -38,10 +38,9 @@@ import org.apache.cassandra.MockSchema
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+ import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@@ -292,17 -291,19 +293,20 @@@ public class TrackerTes
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev1));
Assert.assertEquals(2, tracker.getView().flushingMemtables.size());
-- tracker.replaceFlushed(prev1, null);
++ tracker.replaceFlushed(prev1, Collections.emptyList());
Assert.assertEquals(1, tracker.getView().flushingMemtables.size());
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
- tracker.replaceFlushed(prev2, reader);
+ tracker.replaceFlushed(prev2, Collections.singleton(reader));
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
- tracker.permitCompactionOfFlushed(reader);
++ tracker.permitCompactionOfFlushed(singleton(reader));
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
+ Assert.assertTrue(reader.isKeyCacheSetup());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
// test invalidated CFS
@@@ -314,14 -315,13 +318,13 @@@
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
- tracker.replaceFlushed(prev1, reader);
- tracker.permitCompactionOfFlushed(reader);
+ tracker.replaceFlushed(prev1, Collections.singleton(reader));
++ tracker.permitCompactionOfFlushed(Collections.singleton(reader));
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(3, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertTrue(listener.received.get(1) instanceof SSTableDeletingNotification);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
- Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
++ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
++ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 523c203,5706598..8a5e00e
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@@ -212,6 -208,6 +212,6 @@@ public class ViewTes
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader, SSTableReader>emptyMap(), SSTableIntervalTree.build(sstables));
- Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
++ Collections.<SSTableReader, SSTableReader>emptyMap(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 74b5c74,3bef89e..802d9c8
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@@ -115,56 -107,40 +115,56 @@@ public class CompressedRandomAccessRead
}
}
- private void testResetAndTruncate(File f, boolean compressed, int junkSize) throws IOException
+ private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException
{
final String filename = f.getAbsolutePath();
- ChannelProxy channel = new ChannelProxy(f);
- try
+ try(ChannelProxy channel = new ChannelProxy(f))
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
- : SequentialWriter.open(f);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = compressed
+ ? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector)
+ : SequentialWriter.open(f))
+ {
+ writer.write("The quick ".getBytes());
+ DataPosition mark = writer.mark();
+ writer.write("blue fox jumps over the lazy dog".getBytes());
+
+ // write enough to be sure to change chunk
+ for (int i = 0; i < junkSize; ++i)
+ {
+ writer.write((byte) 1);
+ }
+
+ writer.resetAndTruncate(mark);
+ writer.write("brown fox jumps over the lazy dog".getBytes());
+ writer.finish();
+ }
+ assert f.exists();
- writer.write("The quick ".getBytes());
- FileMark mark = writer.mark();
- writer.write("blue fox jumps over the lazy dog".getBytes());
+ CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
+ RandomAccessReader.Builder builder = compressed
+ ? new CompressedRandomAccessReader.Builder(channel, compressionMetadata)
+ : new RandomAccessReader.Builder(channel);
- // write enough to be sure to change chunk
- for (int i = 0; i < junkSize; ++i)
+ if (usemmap)
{
- writer.write((byte)1);
+ if (compressed)
+ builder.regions(MmappedRegions.map(channel, compressionMetadata));
+ else
+ builder.regions(MmappedRegions.map(channel, f.length()));
}
- writer.resetAndTruncate(mark);
- writer.write("brown fox jumps over the lazy dog".getBytes());
- writer.finish();
+ try(RandomAccessReader reader = builder.build())
+ {
+ String expected = "The quick brown fox jumps over the lazy dog";
+ assertEquals(expected.length(), reader.length());
+ byte[] b = new byte[expected.length()];
+ reader.readFully(b);
+ assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + '\'';
+ }
- assert f.exists();
- RandomAccessReader reader = compressed
- ? CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length()))
- : RandomAccessReader.open(f);
- String expected = "The quick brown fox jumps over the lazy dog";
- assertEquals(expected.length(), reader.length());
- byte[] b = new byte[expected.length()];
- reader.readFully(b);
- assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + "'";
+ if (usemmap)
+ builder.regions.close();
}
finally
{
@@@ -187,11 -166,8 +187,11 @@@
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
+ assertTrue(file.createNewFile());
+ assertTrue(metadata.createNewFile());
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), CompressionParams.snappy(), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
writer.finish();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 9b09f0b,43c44fd..e045aad
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@@ -88,13 -85,12 +88,13 @@@ public class CompressedSequentialWriter
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance))).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance)));
+
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
- try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", compressionParameters, sstableMetadataCollector);)
{
- Random r = new Random();
+ Random r = new Random(42);
// Test both write with byte[] and ByteBuffer
r.nextBytes(dataPre);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 4b9a769,f4b9617..62228e3
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@@ -60,33 -58,13 +60,33 @@@ import org.apache.cassandra.utils.FBUti
*/
public class LegacySSTableTest
{
+ private static final Logger logger = LoggerFactory.getLogger(LegacySSTableTest.class);
+
public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root";
- public static final String KSNAME = "Keyspace1";
- public static final String CFNAME = "Standard1";
- public static Set<String> TEST_DATA;
public static File LEGACY_SSTABLE_ROOT;
+ /**
+ * When adding a new sstable version, add that one here.
+ * See {@link #testGenerateSstables()} to generate sstables.
+ * Take care on commit as you need to add the sstable files using {@code git add -f}
+ */
- public static final String[] legacyVersions = {"ma", "la", "ka", "jb"};
++ public static final String[] legacyVersions = {"mb", "ma", "la", "ka", "jb"};
+
+ // 1200 chars
+ static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
+
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 4c10882,19fa7c4..93365ef
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@@ -19,25 -18,30 +19,28 @@@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
import org.junit.Test;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
-import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.EstimatedHistogram;
import static org.junit.Assert.assertEquals;
@@@ -46,29 -50,81 +49,78 @@@ public class MetadataSerializerTes
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
- CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
-
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
- MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp);
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
++ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
- String partitioner = RandomPartitioner.class.getCanonicalName();
- double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST));
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
- MetadataSerializer serializer = new MetadataSerializer();
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
// Serialize to tmp file
File statsFile = File.createTempFile(Component.STATS.name, null);
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
{
- serializer.serialize(originalMetadata, out, BigFormat.latestVersion);
- serializer.serialize(metadata, version, out);
++ serializer.serialize(metadata, out, version);
}
+ return statsFile;
+ }
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
- EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
- new long[] { 3L, 4L, 5L });
- EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
- new long[] { 8L, 9L, 10L });
- ReplayPosition start = new ReplayPosition(11L, 12);
- ReplayPosition end = new ReplayPosition(15L, 9);
- long minTimestamp = 2162517136L;
- long maxTimestamp = 4162517136L;
-
- MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
- .estimatedRowSize(rowSizes)
- .estimatedColumnCount(columnCounts)
- .commitLogLowerBound(start)
- .commitLogUpperBound(end);
- collector.updateMinTimestamp(minTimestamp);
- collector.updateMaxTimestamp(maxTimestamp);
-
- Set<Integer> ancestors = Sets.newHashSet(1, 2, 3, 4);
- for (int i : ancestors)
- collector.addAncestor(i);
++ ReplayPosition club = new ReplayPosition(11L, 12);
++ ReplayPosition cllb = new ReplayPosition(9L, 12);
++
++ CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
++ MetadataCollector collector = new MetadataCollector(cfm.comparator)
++ .commitLogLowerBound(cllb)
++ .commitLogUpperBound(club);
+
+ String partitioner = RandomPartitioner.class.getCanonicalName();
+ double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
++ Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.emptyList()));
+ return originalMetadata;
+ }
+
+ @Test
- public void testLaReadsLb() throws IOException
++ public void testLaReadLb() throws IOException
++ {
++ testOldReadsNew("la", "lb");
++ }
++
++ @Test
++ public void testMaReadMb() throws IOException
++ {
++ testOldReadsNew("ma", "mb");
++ }
++
++ public void testOldReadsNew(String oldV, String newV) throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ // Write metadata in two minor formats.
- File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
- File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
-
++ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
++ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
+ // Reading both as earlier version should yield identical results.
- Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
++ Descriptor desc = new Descriptor(oldV, statsFileLb.getParentFile(), "", "", 0, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index 9df3fed,0000000..7cf7bd3
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@@ -1,375 -1,0 +1,374 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MmappedRegionsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class);
+
+ private static ByteBuffer allocateBuffer(int size)
+ {
+ ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
+ long seed = System.nanoTime();
+ //seed = 365238103404423L;
+ logger.info("Seed {}", seed);
+
+ new Random(seed).nextBytes(ret.array());
+ return ret;
+ }
+
+ private static File writeFile(String fileName, ByteBuffer buffer) throws IOException
+ {
+ File ret = File.createTempFile(fileName, "1");
+ ret.deleteOnExit();
+
+ try (SequentialWriter writer = SequentialWriter.open(ret))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ assert ret.exists();
+ assert ret.length() >= buffer.capacity();
+ return ret;
+
+ }
+
+ @Test
+ public void testEmpty() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ assertTrue(regions.isEmpty());
+ assertTrue(regions.isValid(channel));
+ }
+ }
+
+ @Test
+ public void testTwoSegments() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(2048);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ for (int i = 0; i < 1024; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+
+ regions.extend(2048);
+ for (int i = 0; i < 2048; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ if (i < 1024)
+ {
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+ else
+ {
+ assertEquals(1024, region.bottom());
+ assertEquals(2048, region.top());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSmallSegmentSize() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ regions.extend(2048);
+ regions.extend(4096);
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testAllocRegions() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity());
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testCopy() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+ {
+ // create 3 more segments, one per quater capacity
+ regions.extend(buffer.capacity() / 2);
+ regions.extend(3 * buffer.capacity() / 4);
+ regions.extend(buffer.capacity());
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ assertFalse(snapshot.isCleanedUp());
+
+ final int SIZE = buffer.capacity() / 4;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = snapshot.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+
+ // check we can access the buffer
+ assertNotNull(region.buffer.duplicate().getInt());
+ }
+
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ assertTrue(snapshot.isCleanedUp());
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testCopyCannotExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity() / 2);
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ try
+ {
+ snapshot.extend(buffer.capacity());
+ }
+ finally
+ {
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ }
+ }
+
+ @Test
+ public void testExtendOutOfOrder() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(4096);
+ regions.extend(1024);
+ regions.extend(2048);
+
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(4096, region.top());
+ }
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNegativeExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(-1);
+ }
+ }
+
+ @Test
+ public void testMapForCompressionMetadata() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+ File f = File.createTempFile("testMapForCompressionMetadata", "1");
+ f.deleteOnExit();
+
+ File cf = File.createTempFile(f.getName() + ".metadata", "1");
+ cf.deleteOnExit();
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance))
- .replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = new CompressedSequentialWriter(f,
+ cf.getAbsolutePath(),
+ CompressionParams.snappy(),
+ sstableMetadataCollector))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
+ try(ChannelProxy channel = new ChannelProxy(f);
+ MmappedRegions regions = MmappedRegions.map(channel, metadata))
+ {
+
+ assertFalse(regions.isEmpty());
+ int i = 0;
+ while(i < buffer.capacity())
+ {
+ CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
+
+ MmappedRegions.Region region = regions.floor(chunk.offset);
+ assertNotNull(region);
+
+ ByteBuffer compressedChunk = region.buffer.duplicate();
+ assertNotNull(compressedChunk);
+ assertEquals(chunk.length + 4, compressedChunk.capacity());
+
+ assertEquals(chunk.offset, region.bottom());
+ assertEquals(chunk.offset + chunk.length + 4, region.top());
+
+ i += metadata.chunkLength();
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ metadata.close();
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap1() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, 0))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap2() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, -1L))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap3() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, null))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+}
[07/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 30ed85b..ca7fe82 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.FileDataInput;
@@ -49,7 +50,7 @@ public class MetadataSerializer implements IMetadataSerializer
{
private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
// sort components by type
List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
@@ -66,12 +67,12 @@ public class MetadataSerializer implements IMetadataSerializer
out.writeInt(type.ordinal());
// serialize position
out.writeInt(lastPosition);
- lastPosition += type.serializer.serializedSize(component);
+ lastPosition += type.serializer.serializedSize(component, version);
}
// serialize components
for (MetadataComponent component : sortedComponents)
{
- component.getType().serializer.serialize(component, out);
+ component.getType().serializer.serialize(component, version, out);
}
}
@@ -153,7 +154,7 @@ public class MetadataSerializer implements IMetadataSerializer
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(tmpDescriptor.filenameFor(Component.STATS))))
{
- serialize(currentComponents, out);
+ serialize(currentComponents, descriptor.version, out);
out.flush();
}
// we cant move a file on top of another file in windows:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index f2eb1af..3d48e34 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -43,7 +43,8 @@ public class StatsMetadata extends MetadataComponent
public final EstimatedHistogram estimatedRowSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
public final int maxLocalDeletionTime;
@@ -57,7 +58,8 @@ public class StatsMetadata extends MetadataComponent
public StatsMetadata(EstimatedHistogram estimatedRowSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
int maxLocalDeletionTime,
@@ -71,7 +73,8 @@ public class StatsMetadata extends MetadataComponent
{
this.estimatedRowSize = estimatedRowSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
this.maxLocalDeletionTime = maxLocalDeletionTime;
@@ -117,7 +120,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -134,7 +138,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -157,7 +162,8 @@ public class StatsMetadata extends MetadataComponent
return new EqualsBuilder()
.append(estimatedRowSize, that.estimatedRowSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
@@ -177,7 +183,8 @@ public class StatsMetadata extends MetadataComponent
return new HashCodeBuilder()
.append(estimatedRowSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
.append(maxLocalDeletionTime)
@@ -193,12 +200,12 @@ public class StatsMetadata extends MetadataComponent
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component) throws IOException
+ public int serializedSize(StatsMetadata component, Version version) throws IOException
{
int size = 0;
size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition, TypeSizes.NATIVE);
+ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
@@ -211,14 +218,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
size += 2 + columnName.remaining(); // with short length
size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
return size;
}
- public void serialize(StatsMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
{
EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
out.writeInt(component.maxLocalDeletionTime);
@@ -233,13 +242,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
out.writeBoolean(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
public StatsMetadata deserialize(Version version, DataInput in) throws IOException
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -264,9 +276,12 @@ public class StatsMetadata extends MetadataComponent
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
return new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index 603732b..4ca078b 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -71,12 +71,12 @@ public class ValidationMetadata extends MetadataComponent
public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
{
- public int serializedSize(ValidationMetadata component) throws IOException
+ public int serializedSize(ValidationMetadata component, Version version) throws IOException
{
return TypeSizes.NATIVE.sizeof(component.partitioner) + 8;
}
- public void serialize(ValidationMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(ValidationMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeUTF(component.partitioner);
out.writeDouble(component.bloomFilterFPChance);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
deleted file mode 100644
index 1a15d6f..0000000
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public abstract class DiskAwareRunnable extends WrappedRunnable
-{
- protected Directories.DataDirectory getWriteDirectory(long writeSize)
- {
- Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
- if (directory == null)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
-
- return directory;
- }
-
- /**
- * Get sstable directories for the CF.
- * @return Directories instance for the CF.
- */
- protected abstract Directories getDirectories();
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index dd56b8b..6911ec6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -122,14 +122,11 @@ public class StreamReceiveTask extends StreamTask
for (SSTableWriter writer : task.sstables)
writer.abort();
task.sstables.clear();
- task.session.taskCompleted(task);
return;
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
File lockfiledir = cfs.directories.getWriteableLocationAsFile(task.sstables.size() * 256L);
- if (lockfiledir == null)
- throw new IOError(new IOException("All disks full"));
StreamLockfile lockfile = new StreamLockfile(lockfiledir, UUID.randomUUID());
lockfile.create(task.sstables);
List<SSTableReader> readers = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index 2665f40..8319014 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -70,7 +70,8 @@ public class SSTableMetadataViewer
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
out.println("Estimated tombstone drop times:");
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 1b4edee..f9b4156 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -452,7 +452,7 @@ public class CommitLogStressTest
int cells = 0;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index c377a21..0c46061 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -59,7 +59,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
Mutation mutation;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 93b88b4..adeb778 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.MockSchema;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
@@ -263,15 +264,15 @@ public class TrackerTest
Tracker tracker = cfs.getTracker();
tracker.subscribe(listener);
- Memtable prev1 = tracker.switchMemtable(true);
+ Memtable prev1 = tracker.switchMemtable(true, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write1 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier1 = cfs.keyspace.writeOrder.newBarrier();
- prev1.setDiscarding(barrier1, new AtomicReference<ReplayPosition>());
+ prev1.setDiscarding(barrier1, new AtomicReference<>(CommitLog.instance.getContext()));
barrier1.issue();
- Memtable prev2 = tracker.switchMemtable(false);
+ Memtable prev2 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write2 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier2 = cfs.keyspace.writeOrder.newBarrier();
- prev2.setDiscarding(barrier2, new AtomicReference<ReplayPosition>());
+ prev2.setDiscarding(barrier2, new AtomicReference<>(CommitLog.instance.getContext()));
barrier2.issue();
Memtable cur = tracker.getView().getCurrentMemtable();
OpOrder.Group writecur = cfs.keyspace.writeOrder.getCurrent();
@@ -297,6 +298,9 @@ public class TrackerTest
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
tracker.replaceFlushed(prev2, reader);
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(reader);
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
@@ -307,16 +311,17 @@ public class TrackerTest
tracker = cfs.getTracker();
listener = new MockListener(false);
tracker.subscribe(listener);
- prev1 = tracker.switchMemtable(false);
+ prev1 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
tracker.replaceFlushed(prev1, reader);
+ tracker.permitCompactionOfFlushed(reader);
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
+ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 32a81e2..5706598 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -208,6 +208,6 @@ public class ViewTest
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
+ Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 931422b..3bef89e 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -113,7 +113,7 @@ public class CompressedRandomAccessReaderTest
ChannelProxy channel = new ChannelProxy(f);
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
SequentialWriter writer = compressed
? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
: SequentialWriter.open(f);
@@ -166,7 +166,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
@@ -251,7 +251,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 27b866d..43c44fd 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -85,7 +85,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index eda4f17..19fa7c4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.EnumSet;
@@ -27,12 +28,16 @@ import java.util.Set;
import com.google.common.collect.Sets;
import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
@@ -45,18 +50,51 @@ public class MetadataSerializerTest
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
+
+ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
+
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
+ // Serialize to tmp file
+ File statsFile = File.createTempFile(Component.STATS.name, null);
+ try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
+ {
+ serializer.serialize(metadata, version, out);
+ }
+ return statsFile;
+ }
+
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
new long[] { 3L, 4L, 5L });
EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
new long[] { 8L, 9L, 10L });
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ ReplayPosition start = new ReplayPosition(11L, 12);
+ ReplayPosition end = new ReplayPosition(15L, 9);
long minTimestamp = 2162517136L;
long maxTimestamp = 4162517136L;
MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
.estimatedRowSize(rowSizes)
.estimatedColumnCount(columnCounts)
- .replayPosition(rp);
+ .commitLogLowerBound(start)
+ .commitLogUpperBound(end);
collector.updateMinTimestamp(minTimestamp);
collector.updateMaxTimestamp(maxTimestamp);
@@ -67,23 +105,35 @@ public class MetadataSerializerTest
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
+ return originalMetadata;
+ }
+
+ @Test
+ public void testLaReadsLb() throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
MetadataSerializer serializer = new MetadataSerializer();
- // Serialize to tmp file
- File statsFile = File.createTempFile(Component.STATS.name, null);
- try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
- {
- serializer.serialize(originalMetadata, out);
- }
+ // Write metadata in two minor formats.
+ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
+ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ // Reading both as earlier version should yield identical results.
+ Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
- assertEquals(originalMetadata.get(type), deserialized.get(type));
+ assertEquals(deserializedLa.get(type), deserializedLb.get(type));
+ if (!originalMetadata.get(type).equals(deserializedLb.get(type)))
+ {
+ // Currently only STATS can be different. Change if no longer the case
+ assertEquals(MetadataType.STATS, type);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
index 8409a26..1f66fb7 100644
--- a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
@@ -157,7 +157,7 @@ public class IntervalTreeTest
public String deserialize(DataInput in) throws IOException { return in.readUTF(); }
public long serializedSize(String v, TypeSizes s) { return v.length(); }
},
- (Constructor<Interval<Integer, String>>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class)
+ (Constructor<Interval<Integer, String>>) (Constructor<?>) Interval.class.getConstructor(Object.class, Object.class, Object.class)
);
DataOutputBuffer out = new DataOutputBuffer();
[18/20] cassandra git commit: Merge commit
'78a3d2bba95b9efcda152a157f822f4970f22636' into cassandra-3.7
Posted by sl...@apache.org.
Merge commit '78a3d2bba95b9efcda152a157f822f4970f22636' into cassandra-3.7
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c9ac0506
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c9ac0506
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c9ac0506
Branch: refs/heads/cassandra-3.7
Commit: c9ac0506c41667ebe9b59acb73d6329401461289
Parents: 902877a 78a3d2b
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:19:52 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:20:41 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 283 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 70 +++--
.../db/commitlog/CommitLogReplayer.java | 59 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../cassandra/db/RecoveryManagerTest.java | 2 +-
.../cassandra/db/commitlog/CommitLogTest.java | 4 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 30 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
94 files changed, 644 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d138389,677ea11..fb8d272
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -3,78 -2,9 +3,79 @@@ Merged from 3.0
* Refactor Materialized View code (CASSANDRA-11475)
* Update Java Driver (CASSANDRA-11615)
Merged from 2.2:
++ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
* Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
-3.0.6
+3.6
+ * Allow server startup if JMX is configured directly (CASSANDRA-11725)
+ * Prevent direct memory OOM on buffer pool allocations (CASSANDRA-11710)
+ * Enhanced Compaction Logging (CASSANDRA-10805)
+ * Make prepared statement cache size configurable (CASSANDRA-11555)
+ * Integrated JMX authentication and authorization (CASSANDRA-10091)
+ * Add units to stress ouput (CASSANDRA-11352)
+ * Fix PER PARTITION LIMIT for single and multi partitions queries (CASSANDRA-11603)
+ * Add uncompressed chunk cache for RandomAccessReader (CASSANDRA-5863)
+ * Clarify ClusteringPrefix hierarchy (CASSANDRA-11213)
+ * Always perform collision check before joining ring (CASSANDRA-10134)
+ * SSTableWriter output discrepancy (CASSANDRA-11646)
+ * Fix potential timeout in NativeTransportService.testConcurrentDestroys (CASSANDRA-10756)
+ * Support large partitions on the 3.0 sstable format (CASSANDRA-11206)
+ * Add support to rebuild from specific range (CASSANDRA-10406)
+ * Optimize the overlapping lookup by calculating all the
+ bounds in advance (CASSANDRA-11571)
+ * Support json/yaml output in noetool tablestats (CASSANDRA-5977)
+ * (stress) Add datacenter option to -node options (CASSANDRA-11591)
+ * Fix handling of empty slices (CASSANDRA-11513)
+ * Make number of cores used by cqlsh COPY visible to testing code (CASSANDRA-11437)
+ * Allow filtering on clustering columns for queries without secondary indexes (CASSANDRA-11310)
+ * Refactor Restriction hierarchy (CASSANDRA-11354)
+ * Eliminate allocations in R/W path (CASSANDRA-11421)
+ * Update Netty to 4.0.36 (CASSANDRA-11567)
+ * Fix PER PARTITION LIMIT for queries requiring post-query ordering (CASSANDRA-11556)
+ * Allow instantiation of UDTs and tuples in UDFs (CASSANDRA-10818)
+ * Support UDT in CQLSSTableWriter (CASSANDRA-10624)
+ * Support for non-frozen user-defined types, updating
+ individual fields of user-defined types (CASSANDRA-7423)
+ * Make LZ4 compression level configurable (CASSANDRA-11051)
+ * Allow per-partition LIMIT clause in CQL (CASSANDRA-7017)
+ * Make custom filtering more extensible with UserExpression (CASSANDRA-11295)
+ * Improve field-checking and error reporting in cassandra.yaml (CASSANDRA-10649)
+ * Print CAS stats in nodetool proxyhistograms (CASSANDRA-11507)
+ * More user friendly error when providing an invalid token to nodetool (CASSANDRA-9348)
+ * Add static column support to SASI index (CASSANDRA-11183)
+ * Support EQ/PREFIX queries in SASI CONTAINS mode without tokenization (CASSANDRA-11434)
+ * Support LIKE operator in prepared statements (CASSANDRA-11456)
+ * Add a command to see if a Materialized View has finished building (CASSANDRA-9967)
+ * Log endpoint and port associated with streaming operation (CASSANDRA-8777)
+ * Print sensible units for all log messages (CASSANDRA-9692)
+ * Upgrade Netty to version 4.0.34 (CASSANDRA-11096)
+ * Break the CQL grammar into separate Parser and Lexer (CASSANDRA-11372)
+ * Compress only inter-dc traffic by default (CASSANDRA-8888)
+ * Add metrics to track write amplification (CASSANDRA-11420)
+ * cassandra-stress: cannot handle "value-less" tables (CASSANDRA-7739)
+ * Add/drop multiple columns in one ALTER TABLE statement (CASSANDRA-10411)
+ * Add require_endpoint_verification opt for internode encryption (CASSANDRA-9220)
+ * Add auto import java.util for UDF code block (CASSANDRA-11392)
+ * Add --hex-format option to nodetool getsstables (CASSANDRA-11337)
+ * sstablemetadata should print sstable min/max token (CASSANDRA-7159)
+ * Do not wrap CassandraException in TriggerExecutor (CASSANDRA-9421)
+ * COPY TO should have higher double precision (CASSANDRA-11255)
+ * Stress should exit with non-zero status after failure (CASSANDRA-10340)
+ * Add client to cqlsh SHOW_SESSION (CASSANDRA-8958)
+ * Fix nodetool tablestats keyspace level metrics (CASSANDRA-11226)
+ * Store repair options in parent_repair_history (CASSANDRA-11244)
+ * Print current leveling in sstableofflinerelevel (CASSANDRA-9588)
+ * Change repair message for keyspaces with RF 1 (CASSANDRA-11203)
+ * Remove hard-coded SSL cipher suites and protocols (CASSANDRA-10508)
+ * Improve concurrency in CompactionStrategyManager (CASSANDRA-10099)
+ * (cqlsh) interpret CQL type for formatting blobs (CASSANDRA-11274)
+ * Refuse to start and print txn log information in case of disk
+ corruption (CASSANDRA-10112)
+ * Resolve some eclipse-warnings (CASSANDRA-11086)
+ * (cqlsh) Show static columns in a different color (CASSANDRA-11059)
+ * Allow to remove TTLs on table with default_time_to_live (CASSANDRA-11207)
+Merged from 3.0:
* Disallow creating view with a static column (CASSANDRA-11602)
* Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593)
* Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index fffd87f,e9a2938..8dedd23
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -920,10 -903,13 +931,13 @@@ public class ColumnFamilyStore implemen
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
- volatile FSWriteError flushFailure = null;
+ volatile Throwable flushFailure = null;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<Collection<SSTableReader>> readers;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<Collection<SSTableReader>> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
@@@ -966,7 -962,9 +990,9 @@@
metric.pendingFlushes.dec();
if (flushFailure != null)
- throw flushFailure;
+ throw Throwables.propagate(flushFailure);
+
+ return commitLogUpperBound;
}
}
@@@ -1052,7 -1041,8 +1069,7 @@@
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, Collections.emptyList());
- memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, Collections.emptyList());
reclaim(memtable);
iter.remove();
}
@@@ -1070,108 -1054,23 +1081,109 @@@
{
for (Memtable memtable : memtables)
{
- flushMemtable(memtable);
- Collection<SSTableReader> readers = memtable.flush();
- memtable.cfs.data.replaceFlushed(memtable, readers);
- reclaim(memtable);
- this.readers.add(readers);
++ this.readers.add(flushMemtable(memtable));
}
}
- catch (FSWriteError e)
+ catch (Throwable t)
{
- JVMStabilityInspector.inspectThrowable(e);
- // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
- postFlush.flushFailure = e;
+ JVMStabilityInspector.inspectThrowable(t);
+ postFlush.flushFailure = t;
}
-
// signal the post-flush we've done our work
postFlush.latch.countDown();
}
- public void flushMemtable(Memtable memtable)
++ public Collection<SSTableReader> flushMemtable(Memtable memtable)
+ {
+ List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
+ long totalBytesOnDisk = 0;
+ long maxBytesOnDisk = 0;
+ long minBytesOnDisk = Long.MAX_VALUE;
+ List<SSTableReader> sstables = new ArrayList<>();
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
+ {
+ List<Memtable.FlushRunnable> flushRunnables = null;
+ List<SSTableMultiWriter> flushResults = null;
+
+ try
+ {
+ // flush the memtable
+ flushRunnables = memtable.flushRunnables(txn);
+
+ for (int i = 0; i < flushRunnables.size(); i++)
+ futures.add(perDiskflushExecutors[i].submit(flushRunnables.get(i)));
+
+ flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
+ }
+ catch (Throwable t)
+ {
+ t = memtable.abortRunnables(flushRunnables, t);
+ t = txn.abort(t);
+ throw Throwables.propagate(t);
+ }
+
+ try
+ {
+ Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
+ while (writerIterator.hasNext())
+ {
+ @SuppressWarnings("resource")
+ SSTableMultiWriter writer = writerIterator.next();
+ if (writer.getFilePointer() > 0)
+ {
+ writer.setOpenResult(true).prepareToCommit();
+ }
+ else
+ {
+ maybeFail(writer.abort(null));
+ writerIterator.remove();
+ }
+ }
+ }
+ catch (Throwable t)
+ {
+ for (SSTableMultiWriter writer : flushResults)
+ t = writer.abort(t);
+ t = txn.abort(t);
+ Throwables.propagate(t);
+ }
+
+ txn.prepareToCommit();
+
+ Throwable accumulate = null;
+ for (SSTableMultiWriter writer : flushResults)
+ accumulate = writer.commit(accumulate);
+
+ maybeFail(txn.commit(accumulate));
+
+ for (SSTableMultiWriter writer : flushResults)
+ {
+ Collection<SSTableReader> flushedSSTables = writer.finished();
+ for (SSTableReader sstable : flushedSSTables)
+ {
+ if (sstable != null)
+ {
+ sstables.add(sstable);
+ long size = sstable.bytesOnDisk();
+ totalBytesOnDisk += size;
+ maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
+ minBytesOnDisk = Math.min(minBytesOnDisk, size);
+ }
+ }
+ }
+ }
- memtable.cfs.replaceFlushed(memtable, sstables);
++ memtable.cfs.data.replaceFlushed(memtable, sstables);
+ reclaim(memtable);
+ memtable.cfs.compactionStrategyManager.compactionLogger.flush(sstables);
+ logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
+ sstables,
+ sstables.size(),
+ FBUtilities.prettyPrintMemory(totalBytesOnDisk),
+ FBUtilities.prettyPrintMemory(maxBytesOnDisk),
+ FBUtilities.prettyPrintMemory(minBytesOnDisk));
++ return sstables;
+ }
+
private void reclaim(final Memtable memtable)
{
// issue a read barrier for reclaiming the memory, and offload the wait to another thread
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 6fe89a9,93dc5af..27c5372
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -34,7 -34,7 +34,6 @@@ import org.apache.cassandra.config.Colu
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
--import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@@ -44,19 -44,16 +43,15 @@@ import org.apache.cassandra.db.rows.Unf
import org.apache.cassandra.dht.*;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.index.transactions.UpdateTransaction;
- import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableTxnWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
- import org.apache.cassandra.io.sstable.SSTableTxnWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.concurrent.OpOrder;
- import org.apache.cassandra.utils.memory.HeapAllocator;
import org.apache.cassandra.utils.memory.MemtableAllocator;
import org.apache.cassandra.utils.memory.MemtablePool;
@@@ -257,49 -263,6 +261,48 @@@ public class Memtable implements Compar
return partitions.size();
}
+ public List<FlushRunnable> flushRunnables(LifecycleTransaction txn)
+ {
+ List<Range<Token>> localRanges = Range.sort(StorageService.instance.getLocalRanges(cfs.keyspace.getName()));
+
+ if (!cfs.getPartitioner().splitter().isPresent() || localRanges.isEmpty())
- return Collections.singletonList(new FlushRunnable(lastReplayPosition.get(), txn));
++ return Collections.singletonList(new FlushRunnable(txn));
+
+ return createFlushRunnables(localRanges, txn);
+ }
+
+ private List<FlushRunnable> createFlushRunnables(List<Range<Token>> localRanges, LifecycleTransaction txn)
+ {
+ assert cfs.getPartitioner().splitter().isPresent();
+
+ Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
+ List<PartitionPosition> boundaries = StorageService.getDiskBoundaries(localRanges, cfs.getPartitioner(), locations);
+ List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
+ PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound();
- ReplayPosition context = lastReplayPosition.get();
+ try
+ {
+ for (int i = 0; i < boundaries.size(); i++)
+ {
+ PartitionPosition t = boundaries.get(i);
- runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
++ runnables.add(new FlushRunnable(rangeStart, t, locations[i], txn));
+ rangeStart = t;
+ }
+ return runnables;
+ }
+ catch (Throwable e)
+ {
+ throw Throwables.propagate(abortRunnables(runnables, e));
+ }
+ }
+
+ public Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
+ {
+ if (runnables != null)
+ for (FlushRunnable runnable : runnables)
+ t = runnable.writer.abort(t);
+ return t;
+ }
+
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@@ -352,75 -315,46 +355,68 @@@
return partitions.get(key);
}
- public long creationTime()
- public Collection<SSTableReader> flush()
-- {
- return creationTime;
- long estimatedSize = estimatedSize();
- Directories.DataDirectory dataDirectory = cfs.getDirectories().getWriteableLocation(estimatedSize);
- if (dataDirectory == null)
- throw new RuntimeException("Insufficient disk space to write " + estimatedSize + " bytes");
- File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- return writeSortedContents(sstableDirectory);
-- }
--
public long getMinTimestamp()
{
return minTimestamp;
}
- private long estimatedSize()
+ class FlushRunnable implements Callable<SSTableMultiWriter>
{
- public final ReplayPosition context;
- long keySize = 0;
- for (PartitionPosition key : partitions.keySet())
+ private final long estimatedSize;
+ private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush;
+
+ private final boolean isBatchLogTable;
+ private final SSTableMultiWriter writer;
+
+ // keeping these to be able to log what we are actually flushing
+ private final PartitionPosition from;
+ private final PartitionPosition to;
+
- FlushRunnable(ReplayPosition context, PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn)
++ FlushRunnable(PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn)
{
- this(context, partitions.subMap(from, to), flushLocation, from, to, txn);
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
++ this(partitions.subMap(from, to), flushLocation, from, to, txn);
}
- return (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
- FlushRunnable(ReplayPosition context, LifecycleTransaction txn)
- private Collection<SSTableReader> writeSortedContents(File sstableDirectory)
- {
- boolean isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
++ FlushRunnable(LifecycleTransaction txn)
+ {
- this(context, partitions, null, null, null, txn);
++ this(partitions, null, null, null, txn);
+ }
+
- FlushRunnable(ReplayPosition context, ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn)
++ FlushRunnable(ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn)
+ {
- this.context = context;
+ this.toFlush = toFlush;
+ this.from = from;
+ this.to = to;
+ long keySize = 0;
+ for (PartitionPosition key : toFlush.keySet())
+ {
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey) key).getKey().remaining();
+ }
+ estimatedSize = (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+
+ this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- logger.debug("Writing {}", Memtable.this.toString());
+ if (flushLocation == null)
+ writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
+ else
+ writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
+
+ }
- Collection<SSTableReader> ssTables;
- try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
+ protected Directories getDirectories()
{
+ return cfs.getDirectories();
+ }
+
- private void writeSortedContents(ReplayPosition context)
++ private void writeSortedContents()
+ {
+ logger.debug("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to);
+
boolean trackContention = logger.isTraceEnabled();
int heavilyContendedRowCount = 0;
// (we can't clear out the map as-we-go to free up memory,
@@@ -447,37 -381,59 +443,39 @@@
}
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getFilePointer()),
- commitLogUpperBound));
-
- // sstables should contain non-repaired data.
- ssTables = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), commitLogUpperBound);
- writer.abort();
- ssTables = Collections.emptyList();
- }
+ long bytesFlushed = writer.getFilePointer();
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(bytesFlushed),
- context));
++ commitLogUpperBound));
+ // Update the metrics
+ cfs.metric.bytesFlushed.inc(bytesFlushed);
if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
-
- return ssTables;
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, toFlush.size(), Memtable.this.toString()));
}
- }
- @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
- public SSTableTxnWriter createFlushWriter(String filename,
- PartitionColumns columns,
- EncodingStats stats)
- {
- // we operate "offline" here, as we expose the resulting reader consciously when done
- // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
- LifecycleTransaction txn = null;
- try
+ public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn,
+ String filename,
+ PartitionColumns columns,
+ EncodingStats stats)
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- txn = LifecycleTransaction.offline(OperationType.FLUSH);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
- .commitLogLowerBound(commitLogLowerBound.get())
- .commitLogUpperBound(commitLogUpperBound.get());
-
- return new SSTableTxnWriter(txn,
- cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
- (long) partitions.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats),
- txn));
++ .commitLogLowerBound(commitLogLowerBound.get())
++ .commitLogUpperBound(commitLogUpperBound.get());
+ return cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
+ (long)toFlush.size(),
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ sstableMetadataCollector,
+ new SerializationHeader(true, cfs.metadata, columns, stats), txn);
+
}
- catch (Throwable t)
+
+ @Override
+ public SSTableMultiWriter call()
{
- writeSortedContents(context);
- if (txn != null)
- txn.close();
- throw t;
++ writeSortedContents();
+ return writer;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index e00a0a4,f45a47a..93c6c50
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@@ -23,17 -23,8 +23,16 @@@ import java.io.EOFException
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
- import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@@@ -45,27 -35,23 +44,25 @@@ import com.google.common.base.Throwable
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
- import com.google.common.collect.Ordering;
import com.google.common.util.concurrent.Uninterruptibles;
-
import org.apache.commons.lang3.StringUtils;
+import org.cliffc.high_scale_lib.NonBlockingHashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.UnknownColumnFamilyException;
+import org.apache.cassandra.db.commitlog.SegmentReader.SyncSegment;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.FileSegmentInputStream;
-import org.apache.cassandra.io.util.RebufferingInputStream;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.db.rows.SerializationHelper;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
@@@ -89,10 -71,10 +86,10 @@@ public class CommitLogReplaye
private static final int LEGACY_END_OF_SEGMENT_MARKER = 0;
private final Set<Keyspace> keyspacesRecovered;
- private final List<Future<?>> futures;
+ private final Queue<Future<Integer>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final CRC32 checksum;
private byte[] buffer;
@@@ -101,79 -83,12 +98,77 @@@
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
+ /*
+ * Wrapper around initiating mutations read from the log to make it possible
+ * to spy on initiated mutations for test
+ */
+ @VisibleForTesting
+ public static class MutationInitiator
+ {
+ protected Future<Integer> initiateMutation(final Mutation mutation,
+ final long segmentId,
+ final int serializedSize,
- final long entryLocation,
++ final int entryLocation,
+ final CommitLogReplayer clr)
+ {
+ Runnable runnable = new WrappedRunnable()
+ {
+ public void runMayThrow()
+ {
+ if (Schema.instance.getKSMetaData(mutation.getKeyspaceName()) == null)
+ return;
+ if (clr.pointInTimeExceeded(mutation))
+ return;
+
+ final Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
+
+ // Rebuild the mutation, omitting column families that
+ // a) the user has requested that we ignore,
+ // b) have already been flushed,
+ // or c) are part of a cf that was dropped.
+ // Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
+ Mutation newMutation = null;
+ for (PartitionUpdate update : clr.replayFilter.filter(mutation))
+ {
+ if (Schema.instance.getCF(update.metadata().cfId) == null)
+ continue; // dropped
+
- ReplayPosition rp = clr.cfPositions.get(update.metadata().cfId);
-
+ // replay if current segment is newer than last flushed one or,
+ // if it is the last known segment, if we are after the replay position
- if (segmentId > rp.segment || (segmentId == rp.segment && entryLocation > rp.position))
++ if (clr.shouldReplay(update.metadata().cfId, new ReplayPosition(segmentId, entryLocation)))
+ {
+ if (newMutation == null)
+ newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
+ newMutation.add(update);
+ clr.replayedCount.incrementAndGet();
+ }
+ }
+ if (newMutation != null)
+ {
+ assert !newMutation.isEmpty();
+
+ try
+ {
+ Uninterruptibles.getUninterruptibly(Keyspace.open(newMutation.getKeyspaceName()).applyFromCommitLog(newMutation));
+ }
+ catch (ExecutionException e)
+ {
+ throw Throwables.propagate(e.getCause());
+ }
+
+ clr.keyspacesRecovered.add(keyspace);
+ }
+ }
+ };
+ return StageManager.getStage(Stage.MUTATION).submit(runnable, serializedSize);
+ }
+ }
+
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
- this.futures = new ArrayList<Future<?>>();
+ this.futures = new ArrayDeque<Future<Integer>>();
this.buffer = new byte[4096];
- this.uncompressedBuffer = new byte[4096];
this.invalidMutations = new HashMap<UUID, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
@@@ -342,11 -285,22 +334,24 @@@
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
+ @SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
+ // just transform from the file name (no reading of headers) to determine version
CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName());
+
try(ChannelProxy channel = new ChannelProxy(file);
RandomAccessReader reader = RandomAccessReader.open(channel))
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index dd07b19,16090a1..be1436c
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@@ -329,10 -332,10 +334,10 @@@ public class Tracke
apply(View.markFlushing(memtable));
}
- public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ public void replaceFlushed(Memtable memtable, Iterable<SSTableReader> sstables)
{
assert !isDummy();
- if (sstables == null || Iterables.isEmpty(sstables))
- if (sstables.isEmpty())
++ if (Iterables.isEmpty(sstables))
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
// if it's null, we don't care what state the cfstore is in, we just replace it and continue
@@@ -348,14 -351,7 +353,9 @@@
Throwable fail;
fail = updateSizeTracking(emptySet(), sstables, null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
+ notifyDiscarded(memtable);
+
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
-
maybeFail(fail);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index 63926ed,17062b4..cde6363
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -319,12 -321,14 +321,14 @@@ public class Vie
List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
- if (flushed == null)
- if (flushed == null || flushed.isEmpty())
++ if (flushed == null || Iterables.isEmpty(flushed))
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compactingMap, view.intervalTree);
+ view.compactingMap, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), flushed);
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compactingMap,
+ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, emptySet(), flushed);
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), flushed);
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compactingMap, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/view/TableViews.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index e030b5b,e0fb3b1..9b6f491
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@@ -120,8 -121,10 +120,10 @@@ public class BigFormat implements SSTab
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
// ma (3.0.0): swap bf hash order
// store rows natively
- // mb (3.0.6): commit log lower bound included
++ // mb (3.0.7, 3.7): commit log lower bound included
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index c574a48,53cf0b0..ca50a44
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -20,10 -20,14 +20,11 @@@ package org.apache.cassandra.io.sstable
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import com.google.common.collect.Maps;
+ import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index d2e0513,420b802..3b3d7e1
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@@ -116,11 -70,9 +116,12 @@@ public class SSTableMetadataViewe
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
+ out.printf("totalColumnsSet: %s%n", stats.totalColumnsSet);
+ out.printf("totalRows: %s%n", stats.totalRows);
out.println("Estimated tombstone drop times:");
+
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
out.printf("%-10s:%10s%n",entry.getKey().intValue(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --cc test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index e6f9499,31dea3e..8e45eea
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@@ -461,13 -448,11 +461,13 @@@ public class CommitLogStressTes
super(log, discardedPos, null, ReplayFilter.create());
}
- int hash = 0;
- int cells = 0;
+ int hash;
+ int cells;
+ int discarded;
+ int skipped;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
index 788757c,baf9466..5ac53f6
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
@@@ -56,65 -50,6 +56,65 @@@ import org.apache.cassandra.db.commitlo
public class RecoveryManagerTest
{
private static Logger logger = LoggerFactory.getLogger(RecoveryManagerTest.class);
+ static final Semaphore blocker = new Semaphore(0);
+ static final Semaphore blocked = new Semaphore(0);
+ static CommitLogReplayer.MutationInitiator originalInitiator = null;
+ static final CommitLogReplayer.MutationInitiator mockInitiator = new CommitLogReplayer.MutationInitiator()
+ {
+ @Override
+ protected Future<Integer> initiateMutation(final Mutation mutation,
+ final long segmentId,
+ final int serializedSize,
- final long entryLocation,
++ final int entryLocation,
+ final CommitLogReplayer clr)
+ {
+ final Future<Integer> toWrap = super.initiateMutation(mutation,
+ segmentId,
+ serializedSize,
+ entryLocation,
+ clr);
+ return new Future<Integer>()
+ {
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isCancelled()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDone()
+ {
+ return blocker.availablePermits() > 0 && toWrap.isDone();
+ }
+
+ @Override
+ public Integer get() throws InterruptedException, ExecutionException
+ {
+ System.out.println("Got blocker once");
+ blocked.release();
+ blocker.acquire();
+ return toWrap.get();
+ }
+
+ @Override
+ public Integer get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException
+ {
+ blocked.release();
+ blocker.tryAcquire(1, timeout, unit);
+ return toWrap.get(timeout, unit);
+ }
+
+ };
+ }
+ };
private static final String KEYSPACE1 = "RecoveryManagerTest1";
private static final String CF_STANDARD1 = "Standard1";
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index b5cbf8b,555cdda..1ea0eb1
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@@ -584,147 -555,21 +584,147 @@@ public class CommitLogTes
}
@Test
- public void testDescriptorInvalidParametersSize() throws IOException
+ public void replay_Compressed_Deflate() throws IOException
{
- Map<String, String> params = new HashMap<>();
- for (int i=0; i<65535; ++i)
- params.put("key"+i, Integer.toString(i, 16));
- try {
- CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.VERSION_30,
- 21,
- new ParameterizedClass("LZ4Compressor", params));
- ByteBuffer buf = ByteBuffer.allocate(1024000);
- CommitLogDescriptor.writeHeader(buf, desc);
- Assert.fail("Parameter object too long should fail on writing descriptor.");
- } catch (ConfigurationException e)
+ replay_Compressed(new ParameterizedClass(DeflateCompressor.class.getName(), Collections.<String, String>emptyMap()));
+ }
+
+ private void replay_Compressed(ParameterizedClass parameterizedClass) throws IOException
+ {
+ ParameterizedClass originalCompression = DatabaseDescriptor.getCommitLogCompression();
+ EncryptionContext originalEncryptionContext = DatabaseDescriptor.getEncryptionContext();
+ try
+ {
+ DatabaseDescriptor.setCommitLogCompression(parameterizedClass);
+ DatabaseDescriptor.setEncryptionContext(EncryptionContextGenerator.createDisabledContext());
+ CommitLog.instance.resetUnsafe(true);
+
+ replaySimple(CommitLog.instance);
+ replayWithDiscard(CommitLog.instance);
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitLogCompression(originalCompression);
+ DatabaseDescriptor.setEncryptionContext(originalEncryptionContext);
+ CommitLog.instance.resetUnsafe(true);
+ }
+ }
+
+ @Test
+ public void replay_Encrypted() throws IOException
+ {
+ ParameterizedClass originalCompression = DatabaseDescriptor.getCommitLogCompression();
+ EncryptionContext originalEncryptionContext = DatabaseDescriptor.getEncryptionContext();
+ try
+ {
+ DatabaseDescriptor.setCommitLogCompression(null);
+ DatabaseDescriptor.setEncryptionContext(EncryptionContextGenerator.createContext(true));
+ CommitLog.instance.resetUnsafe(true);
+
+ replaySimple(CommitLog.instance);
+ replayWithDiscard(CommitLog.instance);
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitLogCompression(originalCompression);
+ DatabaseDescriptor.setEncryptionContext(originalEncryptionContext);
+ CommitLog.instance.resetUnsafe(true);
+ }
+ }
+
+ private void replaySimple(CommitLog commitLog) throws IOException
+ {
+ int cellCount = 0;
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k1")
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ cellCount += 1;
+ commitLog.add(rm1);
+
+ final Mutation rm2 = new RowUpdateBuilder(cfs.metadata, 0, "k2")
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ cellCount += 1;
+ commitLog.add(rm2);
+
+ commitLog.sync(true);
+
+ Replayer replayer = new Replayer(commitLog, ReplayPosition.NONE);
+ List<String> activeSegments = commitLog.getActiveSegmentNames();
+ Assert.assertFalse(activeSegments.isEmpty());
+
+ File[] files = new File(commitLog.location).listFiles((file, name) -> activeSegments.contains(name));
+ replayer.recover(files);
+
+ assertEquals(cellCount, replayer.cells);
+ }
+
+ private void replayWithDiscard(CommitLog commitLog) throws IOException
+ {
+ int cellCount = 0;
+ int max = 1024;
+ int discardPosition = (int)(max * .8); // an arbitrary number of entries that we'll skip on the replay
+ ReplayPosition replayPosition = null;
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+ for (int i = 0; i < max; i++)
+ {
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k" + 1)
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ ReplayPosition position = commitLog.add(rm1);
+
+ if (i == discardPosition)
+ replayPosition = position;
+ if (i > discardPosition)
+ {
+ cellCount += 1;
+ }
+ }
+
+ commitLog.sync(true);
+
+ Replayer replayer = new Replayer(commitLog, replayPosition);
+ List<String> activeSegments = commitLog.getActiveSegmentNames();
+ Assert.assertFalse(activeSegments.isEmpty());
+
+ File[] files = new File(commitLog.location).listFiles((file, name) -> activeSegments.contains(name));
+ replayer.recover(files);
+
+ assertEquals(cellCount, replayer.cells);
+ }
+
+ class Replayer extends CommitLogReplayer
+ {
+ private final ReplayPosition filterPosition;
+ int cells;
+ int skipped;
+
+ Replayer(CommitLog commitLog, ReplayPosition filterPosition)
+ {
- super(commitLog, filterPosition, null, ReplayFilter.create());
++ super(commitLog, filterPosition, Collections.emptyMap(), ReplayFilter.create());
+ this.filterPosition = filterPosition;
+ }
+
+ @SuppressWarnings("resource")
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc) throws IOException
++ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
- // correct path
+ if (entryLocation <= filterPosition.position)
+ {
+ // Skip over this mutation.
+ skipped++;
+ return;
+ }
+
+ FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
+ Mutation mutation = Mutation.serializer.deserialize(new DataInputPlus.DataInputStreamPlus(bufIn), desc.getMessagingVersion(), SerializationHelper.Flag.LOCAL);
+ for (PartitionUpdate partitionUpdate : mutation.getPartitionUpdates())
+ for (Row row : partitionUpdate)
+ cells += Iterables.size(row.cells());
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 0294115,b8de711..902e17a
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@@ -298,11 -298,13 +299,14 @@@ public class TrackerTes
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
- tracker.replaceFlushed(prev2, Collections.singleton(reader));
+ tracker.replaceFlushed(prev2, singleton(reader));
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(singleton(reader));
+ Assert.assertEquals(0, tracker.getView().premature.size());
- Assert.assertEquals(1, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertEquals(2, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
++ Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(0)).memtable);
++ Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
listener.received.clear();
Assert.assertTrue(reader.isKeyCacheSetup());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
@@@ -316,17 -318,13 +320,17 @@@
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
- tracker.replaceFlushed(prev1, Collections.singleton(reader));
+ tracker.replaceFlushed(prev1, singleton(reader));
+ tracker.permitCompactionOfFlushed(Collections.singleton(reader));
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ System.out.println(listener.received);
- Assert.assertEquals(5, listener.received.size());
++ Assert.assertEquals(4, listener.received.size());
+ Assert.assertEquals(prev1, ((MemtableSwitchedNotification) listener.received.get(0)).memtable);
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
- Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(2)).memtable);
- Assert.assertTrue(listener.received.get(3) instanceof SSTableDeletingNotification);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(4)).removed.size());
++ Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
++ Assert.assertTrue(listener.received.get(2) instanceof SSTableDeletingNotification);
++ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(3)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
----------------------------------------------------------------------
[10/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index b62c7e3,fba1627..17062b4
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -33,16 -33,14 +33,16 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.utils.Interval;
import static com.google.common.base.Predicates.equalTo;
++import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableList.copyOf;
import static com.google.common.collect.ImmutableList.of;
import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.filter;
-import static java.util.Collections.singleton;
+import static com.google.common.collect.Iterables.transform;
- import static java.util.Collections.singleton;
import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.filterOut;
import static org.apache.cassandra.db.lifecycle.Helpers.replace;
/**
@@@ -68,18 -66,18 +68,19 @@@ public class Vie
* flushed. In chronologically ascending order.
*/
public final List<Memtable> flushingMemtables;
- public final Set<SSTableReader> compacting;
- public final Set<SSTableReader> sstables;
- public final Set<SSTableReader> premature;
+ final Set<SSTableReader> compacting;
+ final Set<SSTableReader> sstables;
++ final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
// the one we made our decision to compact against.
- public final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> compactingMap;
- public final SSTableIntervalTree intervalTree;
+ final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, SSTableIntervalTree intervalTree)
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
++ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@@ -92,8 -90,8 +93,9 @@@
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
- this.compacting = compacting;
+ this.compactingMap = compacting;
+ this.compacting = compactingMap.keySet();
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@@ -238,8 -158,8 +240,8 @@@
{
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
- replace(view.compacting, unmark, mark),
+ replace(view.compactingMap, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@@ -270,7 -190,7 +272,7 @@@
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap,
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
++ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@@ -285,7 -205,7 +287,7 @@@
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -304,7 -224,7 +306,7 @@@
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -319,17 -239,35 +321,34 @@@
List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
-- if (flushed == null)
++ if (flushed == null || flushed.isEmpty())
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compactingMap, view.intervalTree);
- view.compacting, view.premature, view.intervalTree);
++ view.compactingMap, view.premature, view.intervalTree);
- Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
- Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), flushed);
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compactingMap,
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, emptySet(), flushed);
++ Set<SSTableReader> premature = replace(view.premature, emptySet(), flushed);
++ return new View(view.liveMemtables, flushingMemtables, sstableMap, compactingMap, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
- static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
++ static Function<View, View> permitCompactionOfFlushed(final Collection<SSTableReader> readers)
+ {
++ Set<SSTableReader> expectAndRemove = ImmutableSet.copyOf(readers);
+ return new Function<View, View>()
+ {
-
- @Nullable
+ public View apply(View view)
+ {
- Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
- Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
- return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
++ Set<SSTableReader> premature = replace(view.premature, expectAndRemove, emptySet());
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, expectAndRemove, emptySet());
++ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compactingMap, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/view/TableViews.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/view/TableViews.java
index 893bdd5,0000000..7feb67c
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@@ -1,481 -1,0 +1,481 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.view;
+
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.commitlog.ReplayPosition;
+import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+
+/**
+ * Groups all the views for a given table.
+ */
+public class TableViews extends AbstractCollection<View>
+{
+ private final CFMetaData baseTableMetadata;
+
+ // We need this to be thread-safe, but the number of times this is changed (when a view is created in the keyspace)
+ // massively exceeds the number of time it's read (for every mutation on the keyspace), so a copy-on-write list is the best option.
+ private final List<View> views = new CopyOnWriteArrayList();
+
+ public TableViews(CFMetaData baseTableMetadata)
+ {
+ this.baseTableMetadata = baseTableMetadata;
+ }
+
+ public int size()
+ {
+ return views.size();
+ }
+
+ public Iterator<View> iterator()
+ {
+ return views.iterator();
+ }
+
+ public boolean contains(String viewName)
+ {
+ return Iterables.any(views, view -> view.name.equals(viewName));
+ }
+
+ public boolean add(View view)
+ {
+ // We should have validated that there is no existing view with this name at this point
+ assert !contains(view.name);
+ return views.add(view);
+ }
+
+ public Iterable<ColumnFamilyStore> allViewsCfs()
+ {
+ Keyspace keyspace = Keyspace.open(baseTableMetadata.ksName);
+ return Iterables.transform(views, view -> keyspace.getColumnFamilyStore(view.getDefinition().viewName));
+ }
+
+ public void forceBlockingFlush()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.forceBlockingFlush();
+ }
+
+ public void dumpMemtables()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.dumpMemtable();
+ }
+
- public void truncateBlocking(long truncatedAt)
++ public void truncateBlocking(ReplayPosition replayAfter, long truncatedAt)
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ {
- ReplayPosition replayAfter = viewCfs.discardSSTables(truncatedAt);
++ viewCfs.discardSSTables(truncatedAt);
+ SystemKeyspace.saveTruncationRecord(viewCfs, truncatedAt, replayAfter);
+ }
+ }
+
+ public void removeByName(String viewName)
+ {
+ views.removeIf(v -> v.name.equals(viewName));
+ }
+
+ /**
+ * Calculates and pushes updates to the views replicas. The replicas are determined by
+ * {@link ViewUtils#getViewNaturalEndpoint(String, Token, Token)}.
+ *
+ * @param update an update on the base table represented by this object.
+ * @param writeCommitLog whether we should write the commit log for the view updates.
+ * @param baseComplete time from epoch in ms that the local base mutation was (or will be) completed
+ */
+ public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLog, AtomicLong baseComplete)
+ {
+ assert update.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ Collection<View> views = updatedViews(update);
+ if (views.isEmpty())
+ return;
+
+ // Read modified rows
+ int nowInSec = FBUtilities.nowInSeconds();
+ SinglePartitionReadCommand command = readExistingRowsCommand(update, views, nowInSec);
+ if (command == null)
+ return;
+
+ ColumnFamilyStore cfs = Keyspace.openAndGetStore(update.metadata());
+ long start = System.nanoTime();
+ Collection<Mutation> mutations;
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ UnfilteredRowIterator existings = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command);
+ UnfilteredRowIterator updates = update.unfilteredIterator())
+ {
+ mutations = generateViewUpdates(views, updates, existings, nowInSec);
+ }
+ Keyspace.openAndGetStore(update.metadata()).metric.viewReadTime.update(System.nanoTime() - start, TimeUnit.NANOSECONDS);
+
+ if (!mutations.isEmpty())
+ StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete);
+ }
+
+ /**
+ * Given some updates on the base table of this object and the existing values for the rows affected by that update, generates the
+ * mutation to be applied to the provided views.
+ *
+ * @param views the views potentially affected by {@code updates}.
+ * @param updates the base table updates being applied.
+ * @param existings the existing values for the rows affected by {@code updates}. This is used to decide if a view is
+ * obsoleted by the update and should be removed, gather the values for columns that may not be part of the update if
+ * a new view entry needs to be created, and compute the minimal updates to be applied if the view entry isn't changed
+ * but has simply some updated values. This will be empty for view building as we want to assume anything we'll pass
+ * to {@code updates} is new.
+ * @param nowInSec the current time in seconds.
+ * @return the mutations to apply to the {@code views}. This can be empty.
+ */
+ public Collection<Mutation> generateViewUpdates(Collection<View> views, UnfilteredRowIterator updates, UnfilteredRowIterator existings, int nowInSec)
+ {
+ assert updates.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ List<ViewUpdateGenerator> generators = new ArrayList<>(views.size());
+ for (View view : views)
+ generators.add(new ViewUpdateGenerator(view, updates.partitionKey(), nowInSec));
+
+ DeletionTracker existingsDeletion = new DeletionTracker(existings.partitionLevelDeletion());
+ DeletionTracker updatesDeletion = new DeletionTracker(updates.partitionLevelDeletion());
+
+ /*
+ * We iterate through the updates and the existing rows in parallel. This allows us to know the consequence
+ * on the view of each update.
+ */
+ PeekingIterator<Unfiltered> existingsIter = Iterators.peekingIterator(existings);
+ PeekingIterator<Unfiltered> updatesIter = Iterators.peekingIterator(updates);
+
+ while (existingsIter.hasNext() && updatesIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.peek();
+ Unfiltered update = updatesIter.peek();
+
+ Row existingRow;
+ Row updateRow;
+ int cmp = baseTableMetadata.comparator.compare(update, existing);
+ if (cmp < 0)
+ {
+ // We have an update where there was nothing before
+ if (update.isRangeTombstoneMarker())
+ {
+ updatesDeletion.update(updatesIter.next());
+ continue;
+ }
+
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ existingRow = emptyRow(updateRow.clustering(), existingsDeletion.currentDeletion());
+ }
+ else if (cmp > 0)
+ {
+ // We have something existing but no update (which will happen either because it's a range tombstone marker in
+ // existing, or because we've fetched the existing row due to some partition/range deletion in the updates)
+ if (existing.isRangeTombstoneMarker())
+ {
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion());
+
+ // The way we build the read command used for existing rows, we should always have updatesDeletion.currentDeletion()
+ // that is not live, since we wouldn't have read the existing row otherwise. And we could assert that, but if we ever
+ // change the read method so that it can slightly over-read in some case, that would be an easily avoiding bug lurking,
+ // so we just handle the case.
+ if (updateRow == null)
+ continue;
+ }
+ else
+ {
+ // We're updating a row that had pre-existing data
+ if (update.isRangeTombstoneMarker())
+ {
+ assert existing.isRangeTombstoneMarker();
+ updatesDeletion.update(updatesIter.next());
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ assert !existing.isRangeTombstoneMarker();
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ }
+
+ addToViewUpdateGenerators(existingRow, updateRow, generators, nowInSec);
+ }
+
+ // We only care about more existing rows if the update deletion isn't live, i.e. if we had a partition deletion
+ if (!updatesDeletion.currentDeletion().isLive())
+ {
+ while (existingsIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.next();
+ // If it's a range tombstone, we don't care, we're only looking for existing entry that gets deleted by
+ // the new partition deletion
+ if (existing.isRangeTombstoneMarker())
+ continue;
+
+ Row existingRow = (Row)existing;
+ addToViewUpdateGenerators(existingRow, emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion()), generators, nowInSec);
+ }
+ }
+ while (updatesIter.hasNext())
+ {
+ Unfiltered update = updatesIter.next();
+ // If it's a range tombstone, it removes nothing pre-exisiting, so we can ignore it for view updates
+ if (update.isRangeTombstoneMarker())
+ continue;
+
+ Row updateRow = (Row)update;
+ addToViewUpdateGenerators(emptyRow(updateRow.clustering(), DeletionTime.LIVE), updateRow, generators, nowInSec);
+ }
+
+ return buildMutations(baseTableMetadata, generators);
+ }
+
+ /**
+ * Return the views that are potentially updated by the provided updates.
+ *
+ * @param updates the updates applied to the base table.
+ * @return the views affected by {@code updates}.
+ */
+ public Collection<View> updatedViews(PartitionUpdate updates)
+ {
+ List<View> matchingViews = new ArrayList<>(views.size());
+
+ for (View view : views)
+ {
+ ReadQuery selectQuery = view.getReadQuery();
+ if (!selectQuery.selectsKey(updates.partitionKey()))
+ continue;
+
+ matchingViews.add(view);
+ }
+ return matchingViews;
+ }
+
+ /**
+ * Returns the command to use to read the existing rows required to generate view updates for the provided base
+ * base updates.
+ *
+ * @param updates the base table updates being applied.
+ * @param views the views potentially affected by {@code updates}.
+ * @param nowInSec the current time in seconds.
+ * @return the command to use to read the base table rows required to generate view updates for {@code updates}.
+ */
+ private SinglePartitionReadCommand readExistingRowsCommand(PartitionUpdate updates, Collection<View> views, int nowInSec)
+ {
+ Slices.Builder sliceBuilder = null;
+ DeletionInfo deletionInfo = updates.deletionInfo();
+ CFMetaData metadata = updates.metadata();
+ DecoratedKey key = updates.partitionKey();
+ // TODO: This is subtle: we need to gather all the slices that we have to fetch between partition del, range tombstones and rows.
+ if (!deletionInfo.isLive())
+ {
+ sliceBuilder = new Slices.Builder(metadata.comparator);
+ // Everything covered by a deletion might invalidate an existing view entry, which means we must read it to know. In practice
+ // though, the views involved might filter some base table clustering columns, in which case we can restrict what we read
+ // using those restrictions.
+ // If there is a partition deletion, then we can simply take each slices from each view select filter. They may overlap but
+ // the Slices.Builder handles that for us. Note that in many case this will just involve reading everything (as soon as any
+ // view involved has no clustering restrictions for instance).
+ // For range tombstone, we should theoretically take the difference between the range tombstoned and the slices selected
+ // by every views, but as we don't an easy way to compute that right now, we keep it simple and just use the tombstoned
+ // range.
+ // TODO: we should improve that latter part.
+ if (!deletionInfo.getPartitionDeletion().isLive())
+ {
+ for (View view : views)
+ sliceBuilder.addAll(view.getSelectStatement().clusteringIndexFilterAsSlices());
+ }
+ else
+ {
+ assert deletionInfo.hasRanges();
+ Iterator<RangeTombstone> iter = deletionInfo.rangeIterator(false);
+ while (iter.hasNext())
+ sliceBuilder.add(iter.next().deletedSlice());
+ }
+ }
+
+ // We need to read every row that is updated, unless we can prove that it has no impact on any view entries.
+
+ // If we had some slices from the deletions above, we'll continue using that. Otherwise, it's more efficient to build
+ // a names query.
+ BTreeSet.Builder<Clustering> namesBuilder = sliceBuilder == null ? BTreeSet.builder(metadata.comparator) : null;
+ for (Row row : updates)
+ {
+ // Don't read the existing state if we can prove the update won't affect any views
+ if (!affectsAnyViews(key, row, views))
+ continue;
+
+ if (namesBuilder == null)
+ sliceBuilder.add(Slice.make(row.clustering()));
+ else
+ namesBuilder.add(row.clustering());
+ }
+
+ NavigableSet<Clustering> names = namesBuilder == null ? null : namesBuilder.build();
+ // If we have a slice builder, it means we had some deletions and we have to read. But if we had
+ // only row updates, it's possible none of them affected the views, in which case we have nothing
+ // to do.
+ if (names != null && names.isEmpty())
+ return null;
+
+ ClusteringIndexFilter clusteringFilter = names == null
+ ? new ClusteringIndexSliceFilter(sliceBuilder.build(), false)
+ : new ClusteringIndexNamesFilter(names, false);
+ // If we have more than one view, we should merge the queried columns by each views but to keep it simple we just
+ // include everything. We could change that in the future.
+ ColumnFilter queriedColumns = views.size() == 1
+ ? Iterables.getOnlyElement(views).getSelectStatement().queriedColumns()
+ : ColumnFilter.all(metadata);
+ // Note that the views could have restrictions on regular columns, but even if that's the case we shouldn't apply those
+ // when we read, because even if an existing row doesn't match the view filter, the update can change that in which
+ // case we'll need to know the existing content. There is also no easy way to merge those RowFilter when we have multiple views.
+ // TODO: we could still make sense to special case for when there is a single view and a small number of updates (and
+ // no deletions). Indeed, in that case we could check whether any of the update modify any of the restricted regular
+ // column, and if that's not the case we could use view filter. We keep it simple for now though.
+ RowFilter rowFilter = RowFilter.NONE;
+ return SinglePartitionReadCommand.create(metadata, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, clusteringFilter);
+ }
+
+ private boolean affectsAnyViews(DecoratedKey partitionKey, Row update, Collection<View> views)
+ {
+ for (View view : views)
+ {
+ if (view.mayBeAffectedBy(partitionKey, update))
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Given an existing base row and the update that we're going to apply to this row, generate the modifications
+ * to apply to MVs using the provided {@code ViewUpdateGenerator}s.
+ *
+ * @param existingBaseRow the base table row as it is before an update.
+ * @param updateBaseRow the newly updates made to {@code existingBaseRow}.
+ * @param generators the view update generators to add the new changes to.
+ * @param nowInSec the current time in seconds. Used to decide if data is live or not.
+ */
+ private static void addToViewUpdateGenerators(Row existingBaseRow, Row updateBaseRow, Collection<ViewUpdateGenerator> generators, int nowInSec)
+ {
+ // Having existing empty is useful, it just means we'll insert a brand new entry for updateBaseRow,
+ // but if we have no update at all, we shouldn't get there.
+ assert !updateBaseRow.isEmpty();
+
+ // We allow existingBaseRow to be null, which we treat the same as being empty as an small optimization
+ // to avoid allocating empty row objects when we know there was nothing existing.
+ Row mergedBaseRow = existingBaseRow == null ? updateBaseRow : Rows.merge(existingBaseRow, updateBaseRow, nowInSec);
+ for (ViewUpdateGenerator generator : generators)
+ generator.addBaseTableUpdate(existingBaseRow, mergedBaseRow);
+ }
+
+ private static Row emptyRow(Clustering clustering, DeletionTime deletion)
+ {
+ // Returning null for an empty row is slightly ugly, but the case where there is no pre-existing row is fairly common
+ // (especially when building the view), so we want to avoid a dummy allocation of an empty row every time.
+ // And MultiViewUpdateBuilder knows how to deal with that.
+ return deletion.isLive() ? null : BTreeRow.emptyDeletedRow(clustering, Row.Deletion.regular(deletion));
+ }
+
+ /**
+ * Extracts (and potentially groups) the mutations generated by the provided view update generator.
+ * Returns the mutation that needs to be done to the views given the base table updates
+ * passed to {@link #addBaseTableUpdate}.
+ *
+ * @param baseTableMetadata the metadata for the base table being updated.
+ * @param generators the generators from which to extract the view mutations from.
+ * @return the mutations created by all the generators in {@code generators}.
+ */
+ private Collection<Mutation> buildMutations(CFMetaData baseTableMetadata, List<ViewUpdateGenerator> generators)
+ {
+ // One view is probably common enough and we can optimize a bit easily
+ if (generators.size() == 1)
+ {
+ Collection<PartitionUpdate> updates = generators.get(0).generateViewUpdates();
+ List<Mutation> mutations = new ArrayList<>(updates.size());
+ for (PartitionUpdate update : updates)
+ mutations.add(new Mutation(update));
+ return mutations;
+ }
+
+ Map<DecoratedKey, Mutation> mutations = new HashMap<>();
+ for (ViewUpdateGenerator generator : generators)
+ {
+ for (PartitionUpdate update : generator.generateViewUpdates())
+ {
+ DecoratedKey key = update.partitionKey();
+ Mutation mutation = mutations.get(key);
+ if (mutation == null)
+ {
+ mutation = new Mutation(baseTableMetadata.ksName, key);
+ mutations.put(key, mutation);
+ }
+ mutation.add(update);
+ }
+ }
+ return mutations.values();
+ }
+
+ /**
+ * A simple helper that tracks for a given {@code UnfilteredRowIterator} what is the current deletion at any time of the
+ * iteration. It will be the currently open range tombstone deletion if there is one and the partition deletion otherwise.
+ */
+ private static class DeletionTracker
+ {
+ private final DeletionTime partitionDeletion;
+ private DeletionTime deletion;
+
+ public DeletionTracker(DeletionTime partitionDeletion)
+ {
+ this.partitionDeletion = partitionDeletion;
+ }
+
+ public void update(Unfiltered marker)
+ {
+ assert marker instanceof RangeTombstoneMarker;
+ RangeTombstoneMarker rtm = (RangeTombstoneMarker)marker;
+ this.deletion = rtm.isOpen(false)
+ ? rtm.openDeletionTime(false)
+ : null;
+ }
+
+ public DeletionTime currentDeletion()
+ {
+ return deletion == null ? partitionDeletion : deletion;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/Version.java
index 16829ab,41a83e1..d9e289c
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@@ -60,16 -52,8 +60,18 @@@ public abstract class Versio
public abstract boolean hasNewFileName();
+ public abstract boolean storeRows();
+
+ public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far
+
+ public abstract boolean hasOldBfHashOrder();
+
+ public abstract boolean hasCompactionAncestors();
+
+ public abstract boolean hasBoundaries();
+
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 1f2a98f,9244bbb..e0fb3b1
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@@ -111,7 -125,7 +111,7 @@@ public class BigFormat implements SSTab
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "ma";
- public static final String current_version = "lb";
++ public static final String current_version = "mb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@@ -121,10 -135,7 +121,12 @@@
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
+ // ma (3.0.0): swap bf hash order
+ // store rows natively
++ // mb (3.0.6): commit log lower bound included
+ //
+ // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@@ -134,53 -144,20 +136,56 @@@
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ public final boolean storeRows;
+ public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
+ public final boolean hasBoundaries;
+ /**
+ * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
+ * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
+ */
+ private final boolean hasOldBfHashOrder;
+ private final boolean hasCommitLogLowerBound;
- public BigVersion(String version)
+ /**
+ * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+ */
+ private final boolean hasCompactionAncestors;
+
+ BigVersion(String version)
{
- super(instance,version);
+ super(instance, version);
isLatestVersion = version.compareTo(current_version) == 0;
hasSamplingLevel = version.compareTo("ka") >= 0;
newStatsFile = version.compareTo("ka") >= 0;
- hasAllAdlerChecksums = version.compareTo("ka") >= 0;
+
+ //For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell
+ //PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684
+ ChecksumType checksumType = ChecksumType.CRC32;
+ if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.uncompressedChecksumType = checksumType;
+
+ checksumType = ChecksumType.CRC32;
+ if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.compressedChecksumType = checksumType;
+
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
+
newFileName = version.compareTo("la") >= 0;
- hasCommitLogLowerBound = version.compareTo("lb") >= 0;
+
+ hasOldBfHashOrder = version.compareTo("ma") < 0;
+ hasCompactionAncestors = version.compareTo("ma") < 0;
+ storeRows = version.compareTo("ma") >= 0;
+ correspondingMessagingVersion = storeRows
+ ? MessagingService.VERSION_30
+ : MessagingService.VERSION_21;
+
+ hasBoundaries = version.compareTo("ma") < 0;
++ hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0)
++ || version.compareTo("mb") >= 0;
}
@Override
@@@ -243,25 -202,12 +248,30 @@@
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
+ public boolean storeRows()
+ {
+ return storeRows;
+ }
+
+ @Override
+ public int correspondingMessagingVersion()
+ {
+ return correspondingMessagingVersion;
+ }
+
+ @Override
+ public boolean hasBoundaries()
+ {
+ return hasBoundaries;
+ }
+
+ @Override
public boolean isCompatible()
{
return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index 4f4c04b,c8e6ee8..ef3453a
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@@ -72,14 -72,22 +72,23 @@@ public class CompactionMetadata extend
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, CompactionMetadata component) throws IOException
{
- int size = 0;
- size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
- for (int g : component.ancestors)
- size += TypeSizes.NATIVE.sizeof(g);
++ int sz = 0;
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ sz = 4;
++ }
byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
- return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
- size += TypeSizes.NATIVE.sizeof(serializedCardinality.length) + serializedCardinality.length;
- return size;
++ return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length + sz;
}
- public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
{
- out.writeInt(component.ancestors.size());
- for (int g : component.ancestors)
- out.writeInt(g);
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ out.writeInt(0);
++ }
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 2ed845c,bfeb930..4561520
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@@ -53,24 -51,28 +53,26 @@@ public class LegacyMetadataSerializer e
assert validation != null && stats != null && compaction != null && validation.partitioner != null;
- EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(stats.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
out.writeDouble(validation.bloomFilterFPChance);
out.writeDouble(stats.compressionRatio);
out.writeUTF(validation.partitioner);
- out.writeInt(compaction.ancestors.size());
- for (Integer g : compaction.ancestors)
- out.writeInt(g);
+ out.writeInt(0); // compaction ancestors
StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
out.writeInt(stats.sstableLevel);
- out.writeInt(stats.minColumnNames.size());
- for (ByteBuffer columnName : stats.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(stats.maxColumnNames.size());
- for (ByteBuffer columnName : stats.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(stats.minClusteringValues.size());
+ for (ByteBuffer value : stats.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(stats.maxClusteringValues.size());
+ for (ByteBuffer value : stats.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@@ -88,11 -90,12 +90,12 @@@
}
else
{
- try (DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile))))
+ try (DataInputStreamPlus in = new DataInputStreamPlus(new BufferedInputStream(new FileInputStream(statsFile))))
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@@ -115,21 -120,21 +118,25 @@@
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
+
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
new ValidationMetadata(partitioner, bloomFilterFPChance));
if (types.contains(MetadataType.STATS))
components.put(MetadataType.STATS,
- new StatsMetadata(rowSizes,
+ new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ Integer.MAX_VALUE,
maxLocalDeletionTime,
+ 0,
+ Integer.MAX_VALUE,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index c2b0caf,579ff7a..53cf0b0
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -26,16 -26,17 +26,16 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+ import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.io.sstable.ColumnNameHelper;
-import org.apache.cassandra.io.sstable.ColumnStats;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
+import org.apache.cassandra.db.rows.Cell;
- import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
@@@ -67,9 -67,10 +67,10 @@@ public class MetadataCollector implemen
public static StatsMetadata defaultStatsMetadata()
{
- return new StatsMetadata(defaultRowSizeHistogram(),
- defaultColumnCountHistogram(),
+ return new StatsMetadata(defaultPartitionSizeHistogram(),
+ defaultCellPerPartitionCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@@ -82,26 -80,23 +83,27 @@@
Collections.<ByteBuffer>emptyList(),
Collections.<ByteBuffer>emptyList(),
true,
- ActiveRepairService.UNREPAIRED_SSTABLE);
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ -1,
+ -1);
}
- protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
- protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
+ protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
+ // TODO: cound the number of row per partition (either with the number of cells, or instead)
+ protected EstimatedHistogram estimatedCellPerPartitionCount = defaultCellPerPartitionCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
- protected long minTimestamp = Long.MAX_VALUE;
- protected long maxTimestamp = Long.MIN_VALUE;
- protected int maxLocalDeletionTime = Integer.MIN_VALUE;
+ protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker();
+ protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
+ protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
protected double compressionRatio = NO_COMPRESSION_RATIO;
- protected Set<Integer> ancestors = new HashSet<>();
protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram();
protected int sstableLevel;
- protected List<ByteBuffer> minColumnNames = Collections.emptyList();
- protected List<ByteBuffer> maxColumnNames = Collections.emptyList();
+ protected ByteBuffer[] minClusteringValues;
+ protected ByteBuffer[] maxClusteringValues;
protected boolean hasLegacyCounterShards = false;
+ protected long totalColumnsSet;
+ protected long totalRows;
/**
* Default cardinality estimation method is to use HyperLogLog++.
@@@ -110,22 -105,52 +112,38 @@@
* See CASSANDRA-5906 for detail.
*/
protected ICardinality cardinality = new HyperLogLogPlus(13, 25);
- private final CellNameType columnNameComparator;
+ private final ClusteringComparator comparator;
- public MetadataCollector(CellNameType columnNameComparator)
+ public MetadataCollector(ClusteringComparator comparator)
{
- this.columnNameComparator = columnNameComparator;
+ this.comparator = comparator;
+
+ this.minClusteringValues = new ByteBuffer[comparator.size()];
+ this.maxClusteringValues = new ByteBuffer[comparator.size()];
}
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level, boolean skipAncestors)
+ public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
{
- this(columnNameComparator);
+ this(comparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
- // Get the max timestamp of the precompacted sstables
- // and adds generation of live ancestors
- if (!skipAncestors)
- {
- for (SSTableReader sstable : sstables)
- {
- addAncestor(sstable.descriptor.generation);
- for (Integer i : sstable.getAncestors())
- if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
- addAncestor(i);
- }
- }
- }
-
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level)
- {
- this(sstables, columnNameComparator, level, false);
}
public MetadataCollector addKey(ByteBuffer key)
@@@ -163,57 -188,45 +181,63 @@@
return this;
}
- public MetadataCollector updateMinTimestamp(long potentialMin)
+ public void update(LivenessInfo newInfo)
{
- minTimestamp = Math.min(minTimestamp, potentialMin);
- return this;
+ if (newInfo.isEmpty())
+ return;
+
+ updateTimestamp(newInfo.timestamp());
+ updateTTL(newInfo.ttl());
+ updateLocalDeletionTime(newInfo.localExpirationTime());
}
- public MetadataCollector updateMaxTimestamp(long potentialMax)
+ public void update(Cell cell)
{
- maxTimestamp = Math.max(maxTimestamp, potentialMax);
- return this;
+ updateTimestamp(cell.timestamp());
+ updateTTL(cell.ttl());
+ updateLocalDeletionTime(cell.localDeletionTime());
}
- public MetadataCollector updateMaxLocalDeletionTime(int maxLocalDeletionTime)
+ public void update(DeletionTime dt)
{
- this.maxLocalDeletionTime = Math.max(this.maxLocalDeletionTime, maxLocalDeletionTime);
- return this;
+ if (!dt.isLive())
+ {
+ updateTimestamp(dt.markedForDeleteAt());
+ updateLocalDeletionTime(dt.localDeletionTime());
+ }
}
- public MetadataCollector estimatedRowSize(EstimatedHistogram estimatedRowSize)
+ public void updateColumnSetPerRow(long columnSetInRow)
{
- this.estimatedRowSize = estimatedRowSize;
- return this;
+ totalColumnsSet += columnSetInRow;
+ ++totalRows;
}
- public MetadataCollector estimatedColumnCount(EstimatedHistogram estimatedColumnCount)
+ private void updateTimestamp(long newTimestamp)
{
- this.estimatedColumnCount = estimatedColumnCount;
- return this;
+ timestampTracker.update(newTimestamp);
+ }
+
+ private void updateLocalDeletionTime(int newLocalDeletionTime)
+ {
+ localDeletionTimeTracker.update(newLocalDeletionTime);
+ estimatedTombstoneDropTime.update(newLocalDeletionTime);
+ }
+
+ private void updateTTL(int newTTL)
+ {
+ ttlTracker.update(newTTL);
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@@ -276,15 -280,13 +300,16 @@@
{
Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
- components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
- estimatedColumnCount,
+ components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
+ estimatedCellPerPartitionCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
- minTimestamp,
- maxTimestamp,
- maxLocalDeletionTime,
+ timestampTracker.min(),
+ timestampTracker.max(),
+ localDeletionTimeTracker.min(),
+ localDeletionTimeTracker.max(),
+ ttlTracker.min(),
+ ttlTracker.max(),
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index 824bf8b,3d48e34..07e35bb
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@@ -40,53 -41,43 +40,56 @@@ public class StatsMetadata extends Meta
{
public static final IMetadataComponentSerializer serializer = new StatsMetadataSerializer();
- public final EstimatedHistogram estimatedRowSize;
+ public final EstimatedHistogram estimatedPartitionSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
+ public final int minLocalDeletionTime;
public final int maxLocalDeletionTime;
+ public final int minTTL;
+ public final int maxTTL;
public final double compressionRatio;
public final StreamingHistogram estimatedTombstoneDropTime;
public final int sstableLevel;
- public final List<ByteBuffer> maxColumnNames;
- public final List<ByteBuffer> minColumnNames;
+ public final List<ByteBuffer> minClusteringValues;
+ public final List<ByteBuffer> maxClusteringValues;
public final boolean hasLegacyCounterShards;
public final long repairedAt;
+ public final long totalColumnsSet;
+ public final long totalRows;
- public StatsMetadata(EstimatedHistogram estimatedRowSize,
+ public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
+ int minLocalDeletionTime,
int maxLocalDeletionTime,
+ int minTTL,
+ int maxTTL,
double compressionRatio,
StreamingHistogram estimatedTombstoneDropTime,
int sstableLevel,
- List<ByteBuffer> minColumnNames,
- List<ByteBuffer> maxColumnNames,
+ List<ByteBuffer> minClusteringValues,
+ List<ByteBuffer> maxClusteringValues,
boolean hasLegacyCounterShards,
- long repairedAt)
+ long repairedAt,
+ long totalColumnsSet,
+ long totalRows)
{
- this.estimatedRowSize = estimatedRowSize;
+ this.estimatedPartitionSize = estimatedPartitionSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
+ this.minLocalDeletionTime = minLocalDeletionTime;
this.maxLocalDeletionTime = maxLocalDeletionTime;
+ this.minTTL = minTTL;
+ this.maxTTL = maxTTL;
this.compressionRatio = compressionRatio;
this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
this.sstableLevel = sstableLevel;
@@@ -129,15 -118,13 +132,16 @@@
public StatsMetadata mutateLevel(int newLevel)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
newLevel,
@@@ -151,15 -136,13 +155,16 @@@
public StatsMetadata mutateRepairedAt(long newRepairedAt)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
@@@ -179,15 -160,13 +184,16 @@@
StatsMetadata that = (StatsMetadata) o;
return new EqualsBuilder()
- .append(estimatedRowSize, that.estimatedRowSize)
+ .append(estimatedPartitionSize, that.estimatedPartitionSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
+ .append(minLocalDeletionTime, that.minLocalDeletionTime)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
+ .append(minTTL, that.minTTL)
+ .append(maxTTL, that.maxTTL)
.append(compressionRatio, that.compressionRatio)
.append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime)
.append(sstableLevel, that.sstableLevel)
@@@ -204,15 -181,13 +210,16 @@@
public int hashCode()
{
return new HashCodeBuilder()
- .append(estimatedRowSize)
+ .append(estimatedPartitionSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
+ .append(minLocalDeletionTime)
.append(maxLocalDeletionTime)
+ .append(minTTL)
+ .append(maxTTL)
.append(compressionRatio)
.append(estimatedTombstoneDropTime)
.append(sstableLevel)
@@@ -227,78 -200,61 +234,84 @@@
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, StatsMetadata component) throws IOException
{
int size = 0;
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
- size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
- size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
- size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound);
+ if (version.storeRows())
+ size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+ else
+ size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
+ size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
+ size += TypeSizes.sizeof(component.sstableLevel);
// min column names
size += 4;
- for (ByteBuffer columnName : component.minColumnNames)
- size += 2 + columnName.remaining(); // with short length
+ for (ByteBuffer value : component.minClusteringValues)
+ size += 2 + value.remaining(); // with short length
// max column names
size += 4;
- for (ByteBuffer columnName : component.maxColumnNames)
- size += 2 + columnName.remaining(); // with short length
- size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ for (ByteBuffer value : component.maxClusteringValues)
+ size += 2 + value.remaining(); // with short length
+ size += TypeSizes.sizeof(component.hasLegacyCounterShards);
+ if (version.storeRows())
+ size += 8 + 8; // totalColumnsSet, totalRows
+ if (version.hasCommitLogLowerBound())
- size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound);
return size;
}
- public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
{
- EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(component.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
+ if (version.storeRows())
+ out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.maxLocalDeletionTime);
+ if (version.storeRows())
+ {
+ out.writeInt(component.minTTL);
+ out.writeInt(component.maxTTL);
+ }
out.writeDouble(component.compressionRatio);
StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
out.writeLong(component.repairedAt);
- out.writeInt(component.minColumnNames.size());
- for (ByteBuffer columnName : component.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(component.maxColumnNames.size());
- for (ByteBuffer columnName : component.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(component.minClusteringValues.size());
+ for (ByteBuffer value : component.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(component.maxClusteringValues.size());
+ for (ByteBuffer value : component.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
out.writeBoolean(component.hasLegacyCounterShards);
+
+ if (version.storeRows())
+ {
+ out.writeLong(component.totalColumnsSet);
+ out.writeLong(component.totalRows);
+ }
++
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
- public StatsMetadata deserialize(Version version, DataInput in) throws IOException
+ public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
+ // We use MAX_VALUE as that's the default value for "no deletion time"
+ int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
int maxLocalDeletionTime = in.readInt();
+ int minTTL = version.storeRows() ? in.readInt() : 0;
+ int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
double compressionRatio = in.readDouble();
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
@@@ -320,18 -276,15 +333,22 @@@
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ long totalColumnsSet = version.storeRows() ? in.readLong() : -1L;
+ long totalRows = version.storeRows() ? in.readLong() : -1L;
+
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
- return new StatsMetadata(rowSizes,
++
+ return new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
index 0000000,0000000..e32d12f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
index 0000000,0000000..790cbe1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
index 0000000,0000000..66d1d34
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++163939259
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
index 0000000,0000000..38d0537
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
index 0000000,0000000..a53b7c1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..e3d4377
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
index 0000000,0000000..c5aa7ce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
index 0000000,0000000..4e3bf89
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3387396134
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
index 0000000,0000000..54afd0c
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
index 0000000,0000000..9d9b085
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..5a34d6e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
index 0000000,0000000..55ad88d
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
index 0000000,0000000..86240b5
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++346435755
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
index 0000000,0000000..d1a401b
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
index 0000000,0000000..4f3f45a
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..32fa731
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
index 0000000,0000000..ca10874
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..98f5784
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3740016396
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
index 0000000,0000000..3661d77
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..bb1e3fb
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
[09/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
index 0000000,0000000..288663f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
index 0000000,0000000..6a2f28e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
index 0000000,0000000..fb255bd
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++655951031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
index 0000000,0000000..b3094bf
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
index 0000000,0000000..55dcdce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..adb7fc4
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
index 0000000,0000000..5cd0481
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
index 0000000,0000000..7d2bdbe
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3883019031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
index 0000000,0000000..56f29df
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
index 0000000,0000000..dff24fe
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..19d25c9
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
index 0000000,0000000..77a1394
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
index 0000000,0000000..172b695
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++2876949266
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
index 0000000,0000000..59e65ca
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
index 0000000,0000000..4c67715
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..56c95a8
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
index 0000000,0000000..00a7a39
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..f9f2fff
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++1214766167
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
index 0000000,0000000..d094f73
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..33c33f2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index fea83c1,0c46061..e690785
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@@ -59,9 -59,9 +59,9 @@@ public class CommitLogTestReplayer exte
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
- FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
+ RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
Mutation mutation;
try
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 7b9b19c,adeb778..b8de711
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@@ -38,8 -38,10 +38,9 @@@ import org.apache.cassandra.MockSchema
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+ import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@@ -292,17 -291,19 +293,20 @@@ public class TrackerTes
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev1));
Assert.assertEquals(2, tracker.getView().flushingMemtables.size());
-- tracker.replaceFlushed(prev1, null);
++ tracker.replaceFlushed(prev1, Collections.emptyList());
Assert.assertEquals(1, tracker.getView().flushingMemtables.size());
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
- tracker.replaceFlushed(prev2, reader);
+ tracker.replaceFlushed(prev2, Collections.singleton(reader));
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
- tracker.permitCompactionOfFlushed(reader);
++ tracker.permitCompactionOfFlushed(singleton(reader));
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
+ Assert.assertTrue(reader.isKeyCacheSetup());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
// test invalidated CFS
@@@ -314,14 -315,13 +318,13 @@@
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
- tracker.replaceFlushed(prev1, reader);
- tracker.permitCompactionOfFlushed(reader);
+ tracker.replaceFlushed(prev1, Collections.singleton(reader));
++ tracker.permitCompactionOfFlushed(Collections.singleton(reader));
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(3, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertTrue(listener.received.get(1) instanceof SSTableDeletingNotification);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
- Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
++ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
++ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 523c203,5706598..8a5e00e
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@@ -212,6 -208,6 +212,6 @@@ public class ViewTes
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader, SSTableReader>emptyMap(), SSTableIntervalTree.build(sstables));
- Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
++ Collections.<SSTableReader, SSTableReader>emptyMap(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 74b5c74,3bef89e..802d9c8
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@@ -115,56 -107,40 +115,56 @@@ public class CompressedRandomAccessRead
}
}
- private void testResetAndTruncate(File f, boolean compressed, int junkSize) throws IOException
+ private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException
{
final String filename = f.getAbsolutePath();
- ChannelProxy channel = new ChannelProxy(f);
- try
+ try(ChannelProxy channel = new ChannelProxy(f))
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
- : SequentialWriter.open(f);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = compressed
+ ? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector)
+ : SequentialWriter.open(f))
+ {
+ writer.write("The quick ".getBytes());
+ DataPosition mark = writer.mark();
+ writer.write("blue fox jumps over the lazy dog".getBytes());
+
+ // write enough to be sure to change chunk
+ for (int i = 0; i < junkSize; ++i)
+ {
+ writer.write((byte) 1);
+ }
+
+ writer.resetAndTruncate(mark);
+ writer.write("brown fox jumps over the lazy dog".getBytes());
+ writer.finish();
+ }
+ assert f.exists();
- writer.write("The quick ".getBytes());
- FileMark mark = writer.mark();
- writer.write("blue fox jumps over the lazy dog".getBytes());
+ CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
+ RandomAccessReader.Builder builder = compressed
+ ? new CompressedRandomAccessReader.Builder(channel, compressionMetadata)
+ : new RandomAccessReader.Builder(channel);
- // write enough to be sure to change chunk
- for (int i = 0; i < junkSize; ++i)
+ if (usemmap)
{
- writer.write((byte)1);
+ if (compressed)
+ builder.regions(MmappedRegions.map(channel, compressionMetadata));
+ else
+ builder.regions(MmappedRegions.map(channel, f.length()));
}
- writer.resetAndTruncate(mark);
- writer.write("brown fox jumps over the lazy dog".getBytes());
- writer.finish();
+ try(RandomAccessReader reader = builder.build())
+ {
+ String expected = "The quick brown fox jumps over the lazy dog";
+ assertEquals(expected.length(), reader.length());
+ byte[] b = new byte[expected.length()];
+ reader.readFully(b);
+ assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + '\'';
+ }
- assert f.exists();
- RandomAccessReader reader = compressed
- ? CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length()))
- : RandomAccessReader.open(f);
- String expected = "The quick brown fox jumps over the lazy dog";
- assertEquals(expected.length(), reader.length());
- byte[] b = new byte[expected.length()];
- reader.readFully(b);
- assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + "'";
+ if (usemmap)
+ builder.regions.close();
}
finally
{
@@@ -187,11 -166,8 +187,11 @@@
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
+ assertTrue(file.createNewFile());
+ assertTrue(metadata.createNewFile());
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), CompressionParams.snappy(), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
writer.finish();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 9b09f0b,43c44fd..e045aad
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@@ -88,13 -85,12 +88,13 @@@ public class CompressedSequentialWriter
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance))).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance)));
+
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
- try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", compressionParameters, sstableMetadataCollector);)
{
- Random r = new Random();
+ Random r = new Random(42);
// Test both write with byte[] and ByteBuffer
r.nextBytes(dataPre);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 4b9a769,f4b9617..62228e3
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@@ -60,33 -58,13 +60,33 @@@ import org.apache.cassandra.utils.FBUti
*/
public class LegacySSTableTest
{
+ private static final Logger logger = LoggerFactory.getLogger(LegacySSTableTest.class);
+
public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root";
- public static final String KSNAME = "Keyspace1";
- public static final String CFNAME = "Standard1";
- public static Set<String> TEST_DATA;
public static File LEGACY_SSTABLE_ROOT;
+ /**
+ * When adding a new sstable version, add that one here.
+ * See {@link #testGenerateSstables()} to generate sstables.
+ * Take care on commit as you need to add the sstable files using {@code git add -f}
+ */
- public static final String[] legacyVersions = {"ma", "la", "ka", "jb"};
++ public static final String[] legacyVersions = {"mb", "ma", "la", "ka", "jb"};
+
+ // 1200 chars
+ static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
+
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 4c10882,19fa7c4..93365ef
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@@ -19,25 -18,30 +19,28 @@@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
import org.junit.Test;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
-import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.EstimatedHistogram;
import static org.junit.Assert.assertEquals;
@@@ -46,29 -50,81 +49,78 @@@ public class MetadataSerializerTes
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
- CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
-
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
- MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp);
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
++ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
- String partitioner = RandomPartitioner.class.getCanonicalName();
- double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST));
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
- MetadataSerializer serializer = new MetadataSerializer();
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
// Serialize to tmp file
File statsFile = File.createTempFile(Component.STATS.name, null);
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
{
- serializer.serialize(originalMetadata, out, BigFormat.latestVersion);
- serializer.serialize(metadata, version, out);
++ serializer.serialize(metadata, out, version);
}
+ return statsFile;
+ }
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
- EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
- new long[] { 3L, 4L, 5L });
- EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
- new long[] { 8L, 9L, 10L });
- ReplayPosition start = new ReplayPosition(11L, 12);
- ReplayPosition end = new ReplayPosition(15L, 9);
- long minTimestamp = 2162517136L;
- long maxTimestamp = 4162517136L;
-
- MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
- .estimatedRowSize(rowSizes)
- .estimatedColumnCount(columnCounts)
- .commitLogLowerBound(start)
- .commitLogUpperBound(end);
- collector.updateMinTimestamp(minTimestamp);
- collector.updateMaxTimestamp(maxTimestamp);
-
- Set<Integer> ancestors = Sets.newHashSet(1, 2, 3, 4);
- for (int i : ancestors)
- collector.addAncestor(i);
++ ReplayPosition club = new ReplayPosition(11L, 12);
++ ReplayPosition cllb = new ReplayPosition(9L, 12);
++
++ CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
++ MetadataCollector collector = new MetadataCollector(cfm.comparator)
++ .commitLogLowerBound(cllb)
++ .commitLogUpperBound(club);
+
+ String partitioner = RandomPartitioner.class.getCanonicalName();
+ double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
++ Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.emptyList()));
+ return originalMetadata;
+ }
+
+ @Test
- public void testLaReadsLb() throws IOException
++ public void testLaReadLb() throws IOException
++ {
++ testOldReadsNew("la", "lb");
++ }
++
++ @Test
++ public void testMaReadMb() throws IOException
++ {
++ testOldReadsNew("ma", "mb");
++ }
++
++ public void testOldReadsNew(String oldV, String newV) throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ // Write metadata in two minor formats.
- File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
- File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
-
++ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
++ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
+ // Reading both as earlier version should yield identical results.
- Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
++ Descriptor desc = new Descriptor(oldV, statsFileLb.getParentFile(), "", "", 0, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index 9df3fed,0000000..7cf7bd3
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@@ -1,375 -1,0 +1,374 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MmappedRegionsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class);
+
+ private static ByteBuffer allocateBuffer(int size)
+ {
+ ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
+ long seed = System.nanoTime();
+ //seed = 365238103404423L;
+ logger.info("Seed {}", seed);
+
+ new Random(seed).nextBytes(ret.array());
+ return ret;
+ }
+
+ private static File writeFile(String fileName, ByteBuffer buffer) throws IOException
+ {
+ File ret = File.createTempFile(fileName, "1");
+ ret.deleteOnExit();
+
+ try (SequentialWriter writer = SequentialWriter.open(ret))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ assert ret.exists();
+ assert ret.length() >= buffer.capacity();
+ return ret;
+
+ }
+
+ @Test
+ public void testEmpty() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ assertTrue(regions.isEmpty());
+ assertTrue(regions.isValid(channel));
+ }
+ }
+
+ @Test
+ public void testTwoSegments() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(2048);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ for (int i = 0; i < 1024; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+
+ regions.extend(2048);
+ for (int i = 0; i < 2048; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ if (i < 1024)
+ {
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+ else
+ {
+ assertEquals(1024, region.bottom());
+ assertEquals(2048, region.top());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSmallSegmentSize() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ regions.extend(2048);
+ regions.extend(4096);
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testAllocRegions() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity());
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testCopy() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+ {
+ // create 3 more segments, one per quater capacity
+ regions.extend(buffer.capacity() / 2);
+ regions.extend(3 * buffer.capacity() / 4);
+ regions.extend(buffer.capacity());
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ assertFalse(snapshot.isCleanedUp());
+
+ final int SIZE = buffer.capacity() / 4;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = snapshot.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+
+ // check we can access the buffer
+ assertNotNull(region.buffer.duplicate().getInt());
+ }
+
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ assertTrue(snapshot.isCleanedUp());
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testCopyCannotExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity() / 2);
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ try
+ {
+ snapshot.extend(buffer.capacity());
+ }
+ finally
+ {
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ }
+ }
+
+ @Test
+ public void testExtendOutOfOrder() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(4096);
+ regions.extend(1024);
+ regions.extend(2048);
+
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(4096, region.top());
+ }
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNegativeExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(-1);
+ }
+ }
+
+ @Test
+ public void testMapForCompressionMetadata() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+ File f = File.createTempFile("testMapForCompressionMetadata", "1");
+ f.deleteOnExit();
+
+ File cf = File.createTempFile(f.getName() + ".metadata", "1");
+ cf.deleteOnExit();
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance))
- .replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = new CompressedSequentialWriter(f,
+ cf.getAbsolutePath(),
+ CompressionParams.snappy(),
+ sstableMetadataCollector))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
+ try(ChannelProxy channel = new ChannelProxy(f);
+ MmappedRegions regions = MmappedRegions.map(channel, metadata))
+ {
+
+ assertFalse(regions.isEmpty());
+ int i = 0;
+ while(i < buffer.capacity())
+ {
+ CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
+
+ MmappedRegions.Region region = regions.floor(chunk.offset);
+ assertNotNull(region);
+
+ ByteBuffer compressedChunk = region.buffer.duplicate();
+ assertNotNull(compressedChunk);
+ assertEquals(chunk.length + 4, compressedChunk.capacity());
+
+ assertEquals(chunk.offset, region.bottom());
+ assertEquals(chunk.offset + chunk.length + 4, region.top());
+
+ i += metadata.chunkLength();
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ metadata.close();
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap1() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, 0))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap2() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, -1L))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap3() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, null))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+}
[06/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
Fix commit log replay after out-of-order flush completion.
Patch by Benedict Elliott Smith; reviewed by Branimir Lambov for
CASSANDRA-9669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/849a4386
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/849a4386
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/849a4386
Branch: refs/heads/cassandra-3.7
Commit: 849a438690aa97a361227781108cc90355dcbcd9
Parents: e614433
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Sep 9 15:25:59 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:17:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/db/ColumnFamilyStore.java | 258 +++++++++++--------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 218 +++++++---------
.../db/commitlog/CommitLogReplayer.java | 58 +++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++---
.../compaction/AbstractCompactionStrategy.java | 3 -
.../cassandra/db/compaction/Scrubber.java | 2 -
.../apache/cassandra/db/lifecycle/Tracker.java | 47 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 38 ++-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 10 +-
.../io/sstable/format/big/BigTableWriter.java | 2 +-
.../io/sstable/metadata/CompactionMetadata.java | 4 +-
.../metadata/IMetadataComponentSerializer.java | 4 +-
.../sstable/metadata/IMetadataSerializer.java | 3 +-
.../metadata/LegacyMetadataSerializer.java | 15 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/MetadataSerializer.java | 9 +-
.../io/sstable/metadata/StatsMetadata.java | 41 ++-
.../io/sstable/metadata/ValidationMetadata.java | 4 +-
.../cassandra/io/util/DiskAwareRunnable.java | 42 ---
.../cassandra/streaming/StreamReceiveTask.java | 3 -
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 19 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 6 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 74 +++++-
.../cassandra/utils/IntervalTreeTest.java | 2 +-
33 files changed, 600 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2bb3518..dfcad10 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,5 @@
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+
2.2.7
* Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 513a138..88e22c0 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -146,6 +146,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ @VisibleForTesting
+ public static volatile ColumnFamilyStore discardFlushResults;
+
public final Keyspace keyspace;
public final String name;
public final CFMetaData metadata;
@@ -867,14 +870,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param memtable
*/
- public Future<?> switchMemtableIfCurrent(Memtable memtable)
+ public ListenableFuture<ReplayPosition> switchMemtableIfCurrent(Memtable memtable)
{
synchronized (data)
{
if (data.getView().getCurrentMemtable() == memtable)
return switchMemtable();
}
- return Futures.immediateFuture(null);
+ return waitForFlushes();
}
/*
@@ -884,14 +887,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL
* marked clean up to the position owned by the Memtable.
*/
- public ListenableFuture<?> switchMemtable()
+ public ListenableFuture<ReplayPosition> switchMemtable()
{
synchronized (data)
{
logFlush();
Flush flush = new Flush(false);
flushExecutor.execute(flush);
- ListenableFutureTask<?> task = ListenableFutureTask.create(flush.postFlush, null);
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(flush.postFlush);
postFlushExecutor.submit(task);
return task;
}
@@ -926,77 +929,93 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
- public ListenableFuture<?> forceFlush()
+ /**
+ * Flush if there is unflushed data in the memtables
+ *
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ public ListenableFuture<ReplayPosition> forceFlush()
{
- return forceFlush(null);
+ Memtable current = data.getView().getCurrentMemtable();
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ if (!cfs.data.getView().getCurrentMemtable().isClean())
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
}
/**
* Flush if there is unflushed data that was written to the CommitLog before @param flushIfDirtyBefore
- * (inclusive). If @param flushIfDirtyBefore is null, flush if there is any unflushed data.
+ * (inclusive).
*
- * @return a Future such that when the future completes, all data inserted before forceFlush was called,
- * will be flushed.
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
*/
- public ListenableFuture<?> forceFlush(ReplayPosition flushIfDirtyBefore)
+ public ListenableFuture<ReplayPosition> forceFlush(ReplayPosition flushIfDirtyBefore)
{
- // we synchronize on the data tracker to ensure we don't race against other calls to switchMemtable(),
- // unnecessarily queueing memtables that are about to be made clean
- synchronized (data)
- {
- // during index build, 2ary index memtables can be dirty even if parent is not. if so,
- // we want to flush the 2ary index ones too.
- boolean clean = true;
- for (ColumnFamilyStore cfs : concatWithIndexes())
- clean &= cfs.data.getView().getCurrentMemtable().isCleanAfter(flushIfDirtyBefore);
+ // we don't loop through the remaining memtables since here we only care about commit log dirtiness
+ // and this does not vary between a table and its table-backed indexes
+ Memtable current = data.getView().getCurrentMemtable();
+ if (current.mayContainDataBefore(flushIfDirtyBefore))
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
+ }
- if (clean)
+ /**
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ private ListenableFuture<ReplayPosition> waitForFlushes()
+ {
+ // we grab the current memtable; once any preceding memtables have flushed, we know its
+ // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable)
+ final Memtable current = data.getView().getCurrentMemtable();
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(new Callable<ReplayPosition>()
+ {
+ public ReplayPosition call()
{
- // We could have a memtable for this column family that is being
- // flushed. Make sure the future returned wait for that so callers can
- // assume that any data inserted prior to the call are fully flushed
- // when the future returns (see #5241).
- ListenableFutureTask<?> task = ListenableFutureTask.create(new Runnable()
- {
- public void run()
- {
- logger.trace("forceFlush requested but everything is clean in {}", name);
- }
- }, null);
- postFlushExecutor.execute(task);
- return task;
+ logger.debug("forceFlush requested but everything is clean in {}", name);
+ return current.getCommitLogLowerBound();
}
-
- return switchMemtable();
- }
+ });
+ postFlushExecutor.execute(task);
+ return task;
}
- public void forceBlockingFlush()
+ public ReplayPosition forceBlockingFlush()
{
- FBUtilities.waitOnFuture(forceFlush());
+ return FBUtilities.waitOnFuture(forceFlush());
}
/**
* Both synchronises custom secondary indexes and provides ordering guarantees for futures on switchMemtable/flush
* etc, which expect to be able to wait until the flush (and all prior flushes) requested have completed.
*/
- private final class PostFlush implements Runnable
+ private final class PostFlush implements Callable<ReplayPosition>
{
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<SSTableReader> readers;
volatile FSWriteError flushFailure = null;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<SSTableReader> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
+ this.memtables = memtables;
+ this.readers = readers;
}
- public void run()
+ public ReplayPosition call()
{
+ if (discardFlushResults == ColumnFamilyStore.this)
+ return commitLogUpperBound;
+
writeBarrier.await();
/**
@@ -1018,7 +1037,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
try
{
- // we wait on the latch for the lastReplayPosition to be set, and so that waiters
+ // we wait on the latch for the commitLogUpperBound to be set, and so that waiters
// on this task can rely on all prior flushes being complete
latch.await();
}
@@ -1027,18 +1046,23 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
+ SSTableReader reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
+ }
}
-
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
+ return commitLogUpperBound;
}
}
@@ -1053,7 +1077,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
+ final List<SSTableReader> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@ -1069,43 +1094,33 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* that all write operations register themselves with, and assigning this barrier to the memtables,
* after which we *.issue()* the barrier. This barrier is used to direct write operations started prior
* to the barrier.issue() into the memtable we have switched out, and any started after to its replacement.
- * In doing so it also tells the write operations to update the lastReplayPosition of the memtable, so
+ * In doing so it also tells the write operations to update the commitLogUpperBound of the memtable, so
* that we know the CL position we are dirty to, which can be marked clean when we complete.
*/
writeBarrier = keyspace.writeOrder.newBarrier();
- memtables = new ArrayList<>();
// submit flushes for the memtable for any indexed sub-cfses, and our own
- AtomicReference<ReplayPosition> lastReplayPositionHolder = new AtomicReference<>();
+ AtomicReference<ReplayPosition> commitLogUpperBound = new AtomicReference<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
// switch all memtables, regardless of their dirty status, setting the barrier
// so that we can reach a coordinated decision about cleanliness once they
// are no longer possible to be modified
- Memtable mt = cfs.data.switchMemtable(truncate);
- mt.setDiscarding(writeBarrier, lastReplayPositionHolder);
- memtables.add(mt);
+ Memtable newMemtable = new Memtable(commitLogUpperBound, cfs);
+ Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable);
+ oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound);
+ memtables.add(oldMemtable);
}
- // we now attempt to define the lastReplayPosition; we do this by grabbing the current limit from the CL
- // and attempting to set the holder to this value. at the same time all writes to the memtables are
- // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
- // so that we know all operations prior to the position have not reached it yet
- ReplayPosition lastReplayPosition;
- while (true)
- {
- lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
- ReplayPosition currentLast = lastReplayPositionHolder.get();
- if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
- && lastReplayPositionHolder.compareAndSet(currentLast, lastReplayPosition))
- break;
- }
+ // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log
+ // records owned by this memtable
+ setCommitLogUpperBound(commitLogUpperBound);
// we then issue the barrier; this lets us wait for all operations started prior to the barrier to complete;
- // since this happens after wiring up the lastReplayPosition, we also know all operations with earlier
+ // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier
// replay positions have also completed, i.e. the memtables are done and ready to flush
writeBarrier.issue();
- postFlush = new PostFlush(!truncate, writeBarrier, lastReplayPosition);
+ postFlush = new PostFlush(!truncate, writeBarrier, commitLogUpperBound.get(), memtables, readers);
}
public void run()
@@ -1124,7 +1139,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, null);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
reclaim(memtable);
iter.remove();
}
@@ -1143,8 +1159,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (Memtable memtable : memtables)
{
// flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+ SSTableReader reader = memtable.flush();
+ memtable.cfs.data.replaceFlushed(memtable, reader);
reclaim(memtable);
+ readers.add(reader);
}
}
catch (FSWriteError e)
@@ -1174,6 +1192,38 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ // atomically set the upper bound for the commit log
+ private static void setCommitLogUpperBound(AtomicReference<ReplayPosition> commitLogUpperBound)
+ {
+ // we attempt to set the holder to the current commit log context. at the same time all writes to the memtables are
+ // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
+ // so that we know all operations prior to the position have not reached it yet
+ ReplayPosition lastReplayPosition;
+ while (true)
+ {
+ lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
+ ReplayPosition currentLast = commitLogUpperBound.get();
+ if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
+ && commitLogUpperBound.compareAndSet(currentLast, lastReplayPosition))
+ break;
+ }
+ }
+
+ @VisibleForTesting
+ // this method should ONLY be used for testing commit log behaviour; it discards the current memtable
+ // contents without marking the commit log clean, and prevents any proceeding flushes from marking
+ // the commit log as done, however they *will* terminate (unlike under typical failures) to ensure progress is made
+ public void simulateFailedFlush()
+ {
+ discardFlushResults = this;
+ data.markFlushing(data.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), this)));
+ }
+
+ public void resumeFlushing()
+ {
+ discardFlushResults = null;
+ }
+
/**
* Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
* queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
@@ -1589,11 +1639,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
maybeFail(data.dropSSTables(Predicates.in(sstables), compactionType, null));
}
- void replaceFlushed(Memtable memtable, SSTableReader sstable)
- {
- compactionStrategyWrapper.replaceFlushed(memtable, sstable);
- }
-
public boolean isValid()
{
return valid;
@@ -1615,6 +1660,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return data.getSSTables();
}
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
public Set<SSTableReader> getUncompactingSSTables()
{
return data.getUncompacting();
@@ -2670,6 +2720,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Void call()
{
cfs.data.reset();
+ cfs.getCompactionStrategy().shutdown();
+ cfs.getCompactionStrategy().startup();
return null;
}
}, true);
@@ -2707,39 +2759,42 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- // just nuke the memtable data w/o writing to disk first
- synchronized (data)
+ if (keyspace.getMetadata().durableWrites || takeSnapshot)
{
+ replayAfter = forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
+ replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
}
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
if (takeSnapshot)
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
for (SecondaryIndex index : indexManager.getIndexes())
index.truncateBlocking(truncatedAt);
@@ -2807,7 +2862,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public LifecycleTransaction call() throws Exception
{
assert data.getCompacting().isEmpty() : data.getCompacting();
- Collection<SSTableReader> sstables = Lists.newArrayList(AbstractCompactionStrategy.filterSuspectSSTables(getSSTables()));
+ Iterable<SSTableReader> sstables = getPermittedToCompactSSTables();
+ sstables = AbstractCompactionStrategy.filterSuspectSSTables(sstables);
+ sstables = ImmutableList.copyOf(sstables);
LifecycleTransaction modifier = data.tryModify(sstables, operationType);
assert modifier != null: "something marked things compacting while compactions are disabled";
return modifier;
@@ -3025,10 +3082,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param truncatedAt The timestamp of the truncation
* (all SSTables before that timestamp are going be marked as compacted)
- *
- * @return the most recent replay position of the truncated data
*/
- public ReplayPosition discardSSTables(long truncatedAt)
+ public void discardSSTables(long truncatedAt)
{
assert data.getCompacting().isEmpty() : data.getCompacting();
@@ -3040,11 +3095,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
truncatedSSTables.add(sstable);
}
- if (truncatedSSTables.isEmpty())
- return ReplayPosition.NONE;
-
- markObsolete(truncatedSSTables, OperationType.UNKNOWN);
- return ReplayPosition.getReplayPosition(truncatedSSTables);
+ if (!truncatedSSTables.isEmpty())
+ markObsolete(truncatedSSTables, OperationType.UNKNOWN);
}
public double getDroppableTombstoneRatio()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index c2901a4..2b3662f 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -346,7 +346,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- return null;
+ throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index fb4da72..b4ada09 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -29,21 +29,20 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -62,14 +61,17 @@ public class Memtable implements Comparable<Memtable>
// the write barrier for directing writes to this memtable during a switch
private volatile OpOrder.Barrier writeBarrier;
- // the last ReplayPosition owned by this Memtable; all ReplayPositions lower are owned by this or an earlier Memtable
- private volatile AtomicReference<ReplayPosition> lastReplayPosition;
- // the "first" ReplayPosition owned by this Memtable; this is inaccurate, and only used as a convenience to prevent CLSM flushing wantonly
- private final ReplayPosition minReplayPosition = CommitLog.instance.getContext();
+ // the precise upper bound of ReplayPosition owned by this memtable
+ private volatile AtomicReference<ReplayPosition> commitLogUpperBound;
+ // the precise lower bound of ReplayPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
+ private AtomicReference<ReplayPosition> commitLogLowerBound;
+ // the approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
+ // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
+ private final ReplayPosition approximateCommitLogLowerBound = CommitLog.instance.getContext();
public int compareTo(Memtable that)
{
- return this.minReplayPosition.compareTo(that.minReplayPosition);
+ return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound);
}
public static final class LastReplayPosition extends ReplayPosition
@@ -84,7 +86,6 @@ public class Memtable implements Comparable<Memtable>
// actually only store DecoratedKey.
private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@ -95,9 +96,10 @@ public class Memtable implements Comparable<Memtable>
// memtable was created with the new or old comparator.
public final CellNameType initialComparator;
- public Memtable(ColumnFamilyStore cfs)
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@ -131,7 +133,7 @@ public class Memtable implements Comparable<Memtable>
public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference<ReplayPosition> lastReplayPosition)
{
assert this.writeBarrier == null;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = lastReplayPosition;
this.writeBarrier = writeBarrier;
allocator.setDiscarding();
}
@@ -161,16 +163,21 @@ public class Memtable implements Comparable<Memtable>
// its current value and ours; if it HAS been finalised, we simply accept its judgement
// this permits us to coordinate a safe boundary, as the boundary choice is made
// atomically wrt our max() maintenance, so an operation cannot sneak into the past
- ReplayPosition currentLast = lastReplayPosition.get();
+ ReplayPosition currentLast = commitLogUpperBound.get();
if (currentLast instanceof LastReplayPosition)
return currentLast.compareTo(replayPosition) >= 0;
if (currentLast != null && currentLast.compareTo(replayPosition) >= 0)
return true;
- if (lastReplayPosition.compareAndSet(currentLast, replayPosition))
+ if (commitLogUpperBound.compareAndSet(currentLast, replayPosition))
return true;
}
}
+ public ReplayPosition getCommitLogLowerBound()
+ {
+ return commitLogLowerBound.get();
+ }
+
public boolean isLive()
{
return allocator.isLive();
@@ -181,9 +188,9 @@ public class Memtable implements Comparable<Memtable>
return rows.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@ -252,11 +259,6 @@ public class Memtable implements Comparable<Memtable>
return rows.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@ -285,20 +287,21 @@ public class Memtable implements Comparable<Memtable>
public Map.Entry<DecoratedKey, ColumnFamily> next()
{
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entry = iter.next();
+ Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
// Actual stored key should be true DecoratedKey
- assert entry.getKey() instanceof DecoratedKey;
+ assert entryRowPosition.getKey() instanceof DecoratedKey;
+ @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
+ Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
if (MEMORY_POOL.needToCopyOnHeap())
{
- DecoratedKey key = (DecoratedKey) entry.getKey();
+ DecoratedKey key = entry.getKey();
key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
}
// Store the reference to the current entry so that remove() can update the current size.
currentEntry = entry;
- // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- return (Map.Entry<DecoratedKey, ColumnFamily>) entry;
+ return entry;
}
public void remove()
@@ -315,9 +318,13 @@ public class Memtable implements Comparable<Memtable>
return rows.get(key);
}
- public long creationTime()
+ public SSTableReader flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
+ Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
+ File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@ -325,115 +332,88 @@ public class Memtable implements Comparable<Memtable>
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- FlushRunnable(ReplayPosition context)
- {
- this.context = context;
-
- long keySize = 0;
- for (RowPosition key : rows.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
- }
-
- protected void runMayThrow() throws Exception
+ long keySize = 0;
+ for (RowPosition key : rows.keySet())
{
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- SSTableReader sstable = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstable);
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected Directories getDirectories()
- {
- return cfs.directories;
- }
+ private SSTableReader writeSortedContents(File sstableDirectory)
+ {
+ logger.info("Writing {}", Memtable.this.toString());
- private SSTableReader writeSortedContents(ReplayPosition context, File sstableDirectory)
+ SSTableReader ssTable;
+ // errors when creating the writer that may leave empty temp files.
+ try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
+ for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
+ AtomicBTreeColumns cf = entry.getValue();
+
+ if (cf.isMarkedForDelete() && cf.hasColumns())
{
- AtomicBTreeColumns cf = entry.getValue();
-
- if (cf.isMarkedForDelete() && cf.hasColumns())
- {
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
- continue;
- }
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ // When every node is up, there's no reason to write batchlog data out to sstables
+ // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
+ // and BL data is strictly local, so we don't need to preserve tombstones for repair.
+ // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
+ // See CASSANDRA-4667.
+ if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
+ continue;
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
- context));
+ if (trackContention && cf.usePessimisticLocking())
+ heavilyContendedRowCount++;
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTable = null;
- }
+ if (!cf.isEmpty())
+ writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
+ commitLogUpperBound));
- return ssTable;
+ // temp sstables should contain non-repaired data.
+ ssTable = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
+ ssTable = null;
}
- }
- public SSTableWriter createFlushWriter(String filename)
- {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
+ if (heavilyContendedRowCount > 0)
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ return ssTable;
}
}
+ private SSTableWriter createFlushWriter(String filename)
+ {
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
+ .commitLogLowerBound(commitLogLowerBound.get())
+ .commitLogUpperBound(commitLogUpperBound.get());
+ return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ }
+
private static int estimateRowOverhead(final int count)
{
// calculate row overhead
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 98fb556..a58aeb4 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -33,7 +33,6 @@ import com.google.common.base.Predicate;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +69,7 @@ public class CommitLogReplayer
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final ICRC32 checksum;
private byte[] buffer;
@@ -79,7 +78,7 @@ public class CommitLogReplayer
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
this.futures = new ArrayList<Future<?>>();
@@ -89,7 +88,7 @@ public class CommitLogReplayer
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
this.checksum = CRC32Factory.instance.create();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@ -98,17 +97,12 @@ public class CommitLogReplayer
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
-
- // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@ -125,19 +119,21 @@ public class CommitLogReplayer
cfs.metadata.ksName,
cfs.metadata.cfName);
SystemKeyspace.removeTruncationRecord(cfs.metadata.cfId);
+ truncatedAt = null;
}
}
- else
- {
- rp = replayPositionOrdering.max(Arrays.asList(rp, truncatedAt));
- }
}
- cfPositions.put(cfs.metadata.cfId, rp);
+ ReplayPosition.ReplayFilter filter = new ReplayPosition.ReplayFilter(cfs.getSSTables(), truncatedAt);
+ if (!filter.isEmpty())
+ cfPersisted.put(cfs.metadata.cfId, filter);
+ else
+ globalPosition = ReplayPosition.NONE; // if we have no ranges for this CF, we must replay everything and filter
}
- ReplayPosition globalPosition = replayPositionOrdering.min(cfPositions.values());
- logger.trace("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPositions));
- return new CommitLogReplayer(commitLog, globalPosition, cfPositions, replayFilter);
+ if (globalPosition == null)
+ globalPosition = ReplayPosition.firstNotCovered(cfPersisted.values());
+ logger.debug("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPersisted));
+ return new CommitLogReplayer(commitLog, globalPosition, cfPersisted, replayFilter);
}
public void recover(File[] clogs) throws IOException
@@ -273,6 +269,18 @@ public class CommitLogReplayer
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
@SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
@@ -495,7 +503,7 @@ public class CommitLogReplayer
mutationStart, errorContext);
continue;
}
- replayMutation(buffer, serializedSize, reader.getFilePointer(), desc);
+ replayMutation(buffer, serializedSize, (int) reader.getFilePointer(), desc);
}
return true;
}
@@ -504,7 +512,7 @@ public class CommitLogReplayer
* Deserializes and replays a commit log entry.
*/
void replayMutation(byte[] inputBuffer, int size,
- final long entryLocation, final CommitLogDescriptor desc) throws IOException
+ final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
final Mutation mutation;
@@ -577,11 +585,7 @@ public class CommitLogReplayer
if (Schema.instance.getCF(columnFamily.id()) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(columnFamily.id());
-
- // replay if current segment is newer than last flushed one or,
- // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
+ if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index ca1969f..17802ad 100644
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@ -19,10 +19,10 @@ package org.apache.cassandra.db.commitlog;
import java.io.DataInput;
import java.io.IOException;
-import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
@@ -35,46 +35,78 @@ public class ReplayPosition implements Comparable<ReplayPosition>
public static final ReplayPositionSerializer serializer = new ReplayPositionSerializer();
// NONE is used for SSTables that are streamed from other nodes and thus have no relationship
- // with our local commitlog. The values satisfy the critera that
+ // with our local commitlog. The values satisfy the criteria that
// - no real commitlog segment will have the given id
// - it will sort before any real replayposition, so it will be effectively ignored by getReplayPosition
public static final ReplayPosition NONE = new ReplayPosition(-1, 0);
+ public final long segment;
+ public final int position;
+
/**
- * Convenience method to compute the replay position for a group of SSTables.
- * @param sstables
- * @return the most recent (highest) replay position
+ * A filter of known safe-to-discard commit log replay positions, based on
+ * the range covered by on disk sstables and those prior to the most recent truncation record
*/
- public static ReplayPosition getReplayPosition(Iterable<? extends SSTableReader> sstables)
+ public static class ReplayFilter
{
- if (Iterables.isEmpty(sstables))
- return NONE;
-
- Function<SSTableReader, ReplayPosition> f = new Function<SSTableReader, ReplayPosition>()
+ final NavigableMap<ReplayPosition, ReplayPosition> persisted = new TreeMap<>();
+ public ReplayFilter(Iterable<SSTableReader> onDisk, ReplayPosition truncatedAt)
{
- public ReplayPosition apply(SSTableReader sstable)
+ for (SSTableReader reader : onDisk)
{
- return sstable.getReplayPosition();
+ ReplayPosition start = reader.getSSTableMetadata().commitLogLowerBound;
+ ReplayPosition end = reader.getSSTableMetadata().commitLogUpperBound;
+ add(persisted, start, end);
}
- };
- Ordering<ReplayPosition> ordering = Ordering.from(ReplayPosition.comparator);
- return ordering.max(Iterables.transform(sstables, f));
- }
+ if (truncatedAt != null)
+ add(persisted, ReplayPosition.NONE, truncatedAt);
+ }
+ private static void add(NavigableMap<ReplayPosition, ReplayPosition> ranges, ReplayPosition start, ReplayPosition end)
+ {
+ // extend ourselves to cover any ranges we overlap
+ // record directly preceding our end may extend past us, so take the max of our end and its
+ Map.Entry<ReplayPosition, ReplayPosition> extend = ranges.floorEntry(end);
+ if (extend != null && extend.getValue().compareTo(end) > 0)
+ end = extend.getValue();
+
+ // record directly preceding our start may extend into us; if it does, we take it as our start
+ extend = ranges.lowerEntry(start);
+ if (extend != null && extend.getValue().compareTo(start) >= 0)
+ start = extend.getKey();
+
+ ranges.subMap(start, end).clear();
+ ranges.put(start, end);
+ }
- public final long segment;
- public final int position;
+ public boolean shouldReplay(ReplayPosition position)
+ {
+ // replay ranges are start exclusive, end inclusive
+ Map.Entry<ReplayPosition, ReplayPosition> range = persisted.lowerEntry(position);
+ return range == null || position.compareTo(range.getValue()) > 0;
+ }
- public static final Comparator<ReplayPosition> comparator = new Comparator<ReplayPosition>()
- {
- public int compare(ReplayPosition o1, ReplayPosition o2)
+ public boolean isEmpty()
{
- if (o1.segment != o2.segment)
- return Long.valueOf(o1.segment).compareTo(o2.segment);
+ return persisted.isEmpty();
+ }
+ }
- return Integer.valueOf(o1.position).compareTo(o2.position);
+ public static ReplayPosition firstNotCovered(Iterable<ReplayFilter> ranges)
+ {
+ ReplayPosition min = null;
+ for (ReplayFilter map : ranges)
+ {
+ ReplayPosition first = map.persisted.firstEntry().getValue();
+ if (min == null)
+ min = first;
+ else
+ min = Ordering.natural().min(min, first);
}
- };
+ if (min == null)
+ return NONE;
+ return min;
+ }
public ReplayPosition(long segment, int position)
{
@@ -83,9 +115,12 @@ public class ReplayPosition implements Comparable<ReplayPosition>
this.position = position;
}
- public int compareTo(ReplayPosition other)
+ public int compareTo(ReplayPosition that)
{
- return comparator.compare(this, other);
+ if (this.segment != that.segment)
+ return Long.compare(this.segment, that.segment);
+
+ return Integer.compare(this.position, that.position);
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 02497df..575aa51 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -232,9 +232,6 @@ public abstract class AbstractCompactionStrategy
*/
public void replaceFlushed(Memtable memtable, SSTableReader sstable)
{
- cfs.getTracker().replaceFlushed(memtable, sstable);
- if (sstable != null)
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e9137e2..d1fe702 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -98,8 +98,6 @@ public class Scrubber implements Closeable
// Calculate the expected compacted filesize
this.destination = cfs.directories.getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(toScrub, OperationType.SCRUB));
- if (destination == null)
- throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
this.controller = transaction.isOffline()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index a074216..5d5701f 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -31,11 +31,13 @@ import com.google.common.collect.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -45,6 +47,8 @@ import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.OpOrder;
import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.in;
+import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.Iterables.filter;
import static java.util.Collections.singleton;
@@ -195,10 +199,12 @@ public class Tracker
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@ -294,9 +300,8 @@ public class Tracker
*
* @return the previously active memtable
*/
- public Memtable switchMemtable(boolean truncating)
+ public Memtable switchMemtable(boolean truncating, Memtable newMemtable)
{
- Memtable newMemtable = new Memtable(cfstore);
Pair<View, View> result = apply(View.switchMemtable(newMemtable));
if (truncating)
notifyRenewed(newMemtable);
@@ -328,15 +333,35 @@ public class Tracker
Throwable fail;
fail = updateSizeTracking(emptySet(), singleton(sstable), null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstable, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
+ public void permitCompactionOfFlushed(SSTableReader sstable)
+ {
+ if (sstable == null)
+ return;
+
+ apply(View.permitCompactionOfFlushed(sstable));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
+ notifyAdded(sstable);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
+
// MISCELLANEOUS public utility calls
@@ -346,6 +371,12 @@ public class Tracker
return view.get().sstables;
}
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index 73ba131..fba1627 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.db.lifecycle;
import java.util.*;
+import javax.annotation.Nullable;
+
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
@@ -66,6 +68,7 @@ public class View
public final List<Memtable> flushingMemtables;
public final Set<SSTableReader> compacting;
public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
@@ -74,7 +77,7 @@ public class View
public final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, SSTableIntervalTree intervalTree)
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@ -88,6 +91,7 @@ public class View
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
this.compacting = compacting;
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@ -155,7 +159,7 @@ public class View
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
replace(view.compacting, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@ -169,7 +173,7 @@ public class View
public boolean apply(View view)
{
for (SSTableReader reader : readers)
- if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted())
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
return false;
return true;
}
@@ -186,7 +190,7 @@ public class View
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting,
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@ -201,7 +205,7 @@ public class View
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -220,7 +224,7 @@ public class View
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -237,15 +241,33 @@ public class View
if (flushed == null)
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compacting, view.intervalTree);
+ view.compacting, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compacting,
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e81e4e9..c303975 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1864,11 +1864,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstableMetadata.compressionRatio;
}
- public ReplayPosition getReplayPosition()
- {
- return sstableMetadata.replayPosition;
- }
-
public long getMinTimestamp()
{
return sstableMetadata.minTimestamp;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index faaa89e..41a83e1 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -52,6 +52,8 @@ public abstract class Version
public abstract boolean hasNewFileName();
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index a1e32cf..9244bbb 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -125,7 +125,7 @@ public class BigFormat implements SSTableFormat
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "la";
+ public static final String current_version = "lb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@ -135,6 +135,7 @@ public class BigFormat implements SSTableFormat
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@ -143,6 +144,7 @@ public class BigFormat implements SSTableFormat
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ private final boolean hasCommitLogLowerBound;
public BigVersion(String version)
{
@@ -155,6 +157,7 @@ public class BigFormat implements SSTableFormat
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("la") >= 0;
+ hasCommitLogLowerBound = version.compareTo("lb") >= 0;
}
@Override
@@ -199,6 +202,11 @@ public class BigFormat implements SSTableFormat
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
public boolean isCompatible()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 505bac0..3a01f87 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -432,7 +432,7 @@ public class BigTableWriter extends SSTableWriter
File file = new File(desc.filenameFor(Component.STATS));
try (SequentialWriter out = SequentialWriter.open(file);)
{
- desc.getMetadataSerializer().serialize(components, out.stream);
+ desc.getMetadataSerializer().serialize(components, desc.version, out.stream);
out.setDescriptor(desc).finish();
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index ed1f327..c8e6ee8 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -72,7 +72,7 @@ public class CompactionMetadata extends MetadataComponent
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component) throws IOException
+ public int serializedSize(CompactionMetadata component, Version version) throws IOException
{
int size = 0;
size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
@@ -83,7 +83,7 @@ public class CompactionMetadata extends MetadataComponent
return size;
}
- public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeInt(component.ancestors.size());
for (int g : component.ancestors)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
index dc8fbdf..e3d867f 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
@@ -35,7 +35,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @return serialized size of this component
* @throws IOException
*/
- int serializedSize(T component) throws IOException;
+ int serializedSize(T component, Version version) throws IOException;
/**
* Serialize metadata component to given output.
@@ -45,7 +45,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @param out serialize destination
* @throws IOException
*/
- void serialize(T component, DataOutputPlus out) throws IOException;
+ void serialize(T component, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize metadata component from given input.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index df577df..a7d23f4 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -22,6 +22,7 @@ import java.util.EnumSet;
import java.util.Map;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -37,7 +38,7 @@ public interface IMetadataSerializer
* @param out
* @throws IOException
*/
- void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException;
+ void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize specified metadata components from given descriptor.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 4bd060e..bfeb930 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Maps;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -42,7 +43,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
* Legacy serialization is only used for SSTable level reset.
*/
@Override
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
@@ -52,7 +53,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
@@ -70,6 +71,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
out.writeInt(stats.maxColumnNames.size());
for (ByteBuffer columnName : stats.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@ -91,7 +94,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -116,6 +120,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
@@ -124,7 +130,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
components.put(MetadataType.STATS,
new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 5962a46..579ff7a 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -28,6 +28,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -69,6 +70,7 @@ public class MetadataCollector
return new StatsMetadata(defaultRowSizeHistogram(),
defaultColumnCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@ -83,7 +85,8 @@ public class MetadataCollector
protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
protected long minTimestamp = Long.MAX_VALUE;
protected long maxTimestamp = Long.MIN_VALUE;
protected int maxLocalDeletionTime = Integer.MIN_VALUE;
@@ -113,7 +116,23 @@ public class MetadataCollector
{
this(columnNameComparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
// Get the max timestamp of the precompacted sstables
// and adds generation of live ancestors
@@ -199,9 +218,15 @@ public class MetadataCollector
return this;
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@ -257,7 +282,8 @@ public class MetadataCollector
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
[19/20] cassandra git commit: Merge commit
'78a3d2bba95b9efcda152a157f822f4970f22636' into cassandra-3.7
Posted by sl...@apache.org.
Merge commit '78a3d2bba95b9efcda152a157f822f4970f22636' into cassandra-3.7
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c9ac0506
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c9ac0506
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c9ac0506
Branch: refs/heads/trunk
Commit: c9ac0506c41667ebe9b59acb73d6329401461289
Parents: 902877a 78a3d2b
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:19:52 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:20:41 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 283 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 70 +++--
.../db/commitlog/CommitLogReplayer.java | 59 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../cassandra/db/RecoveryManagerTest.java | 2 +-
.../cassandra/db/commitlog/CommitLogTest.java | 4 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 30 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
94 files changed, 644 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d138389,677ea11..fb8d272
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -3,78 -2,9 +3,79 @@@ Merged from 3.0
* Refactor Materialized View code (CASSANDRA-11475)
* Update Java Driver (CASSANDRA-11615)
Merged from 2.2:
++ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
* Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395)
+ * cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
-3.0.6
+3.6
+ * Allow server startup if JMX is configured directly (CASSANDRA-11725)
+ * Prevent direct memory OOM on buffer pool allocations (CASSANDRA-11710)
+ * Enhanced Compaction Logging (CASSANDRA-10805)
+ * Make prepared statement cache size configurable (CASSANDRA-11555)
+ * Integrated JMX authentication and authorization (CASSANDRA-10091)
+ * Add units to stress ouput (CASSANDRA-11352)
+ * Fix PER PARTITION LIMIT for single and multi partitions queries (CASSANDRA-11603)
+ * Add uncompressed chunk cache for RandomAccessReader (CASSANDRA-5863)
+ * Clarify ClusteringPrefix hierarchy (CASSANDRA-11213)
+ * Always perform collision check before joining ring (CASSANDRA-10134)
+ * SSTableWriter output discrepancy (CASSANDRA-11646)
+ * Fix potential timeout in NativeTransportService.testConcurrentDestroys (CASSANDRA-10756)
+ * Support large partitions on the 3.0 sstable format (CASSANDRA-11206)
+ * Add support to rebuild from specific range (CASSANDRA-10406)
+ * Optimize the overlapping lookup by calculating all the
+ bounds in advance (CASSANDRA-11571)
+ * Support json/yaml output in noetool tablestats (CASSANDRA-5977)
+ * (stress) Add datacenter option to -node options (CASSANDRA-11591)
+ * Fix handling of empty slices (CASSANDRA-11513)
+ * Make number of cores used by cqlsh COPY visible to testing code (CASSANDRA-11437)
+ * Allow filtering on clustering columns for queries without secondary indexes (CASSANDRA-11310)
+ * Refactor Restriction hierarchy (CASSANDRA-11354)
+ * Eliminate allocations in R/W path (CASSANDRA-11421)
+ * Update Netty to 4.0.36 (CASSANDRA-11567)
+ * Fix PER PARTITION LIMIT for queries requiring post-query ordering (CASSANDRA-11556)
+ * Allow instantiation of UDTs and tuples in UDFs (CASSANDRA-10818)
+ * Support UDT in CQLSSTableWriter (CASSANDRA-10624)
+ * Support for non-frozen user-defined types, updating
+ individual fields of user-defined types (CASSANDRA-7423)
+ * Make LZ4 compression level configurable (CASSANDRA-11051)
+ * Allow per-partition LIMIT clause in CQL (CASSANDRA-7017)
+ * Make custom filtering more extensible with UserExpression (CASSANDRA-11295)
+ * Improve field-checking and error reporting in cassandra.yaml (CASSANDRA-10649)
+ * Print CAS stats in nodetool proxyhistograms (CASSANDRA-11507)
+ * More user friendly error when providing an invalid token to nodetool (CASSANDRA-9348)
+ * Add static column support to SASI index (CASSANDRA-11183)
+ * Support EQ/PREFIX queries in SASI CONTAINS mode without tokenization (CASSANDRA-11434)
+ * Support LIKE operator in prepared statements (CASSANDRA-11456)
+ * Add a command to see if a Materialized View has finished building (CASSANDRA-9967)
+ * Log endpoint and port associated with streaming operation (CASSANDRA-8777)
+ * Print sensible units for all log messages (CASSANDRA-9692)
+ * Upgrade Netty to version 4.0.34 (CASSANDRA-11096)
+ * Break the CQL grammar into separate Parser and Lexer (CASSANDRA-11372)
+ * Compress only inter-dc traffic by default (CASSANDRA-8888)
+ * Add metrics to track write amplification (CASSANDRA-11420)
+ * cassandra-stress: cannot handle "value-less" tables (CASSANDRA-7739)
+ * Add/drop multiple columns in one ALTER TABLE statement (CASSANDRA-10411)
+ * Add require_endpoint_verification opt for internode encryption (CASSANDRA-9220)
+ * Add auto import java.util for UDF code block (CASSANDRA-11392)
+ * Add --hex-format option to nodetool getsstables (CASSANDRA-11337)
+ * sstablemetadata should print sstable min/max token (CASSANDRA-7159)
+ * Do not wrap CassandraException in TriggerExecutor (CASSANDRA-9421)
+ * COPY TO should have higher double precision (CASSANDRA-11255)
+ * Stress should exit with non-zero status after failure (CASSANDRA-10340)
+ * Add client to cqlsh SHOW_SESSION (CASSANDRA-8958)
+ * Fix nodetool tablestats keyspace level metrics (CASSANDRA-11226)
+ * Store repair options in parent_repair_history (CASSANDRA-11244)
+ * Print current leveling in sstableofflinerelevel (CASSANDRA-9588)
+ * Change repair message for keyspaces with RF 1 (CASSANDRA-11203)
+ * Remove hard-coded SSL cipher suites and protocols (CASSANDRA-10508)
+ * Improve concurrency in CompactionStrategyManager (CASSANDRA-10099)
+ * (cqlsh) interpret CQL type for formatting blobs (CASSANDRA-11274)
+ * Refuse to start and print txn log information in case of disk
+ corruption (CASSANDRA-10112)
+ * Resolve some eclipse-warnings (CASSANDRA-11086)
+ * (cqlsh) Show static columns in a different color (CASSANDRA-11059)
+ * Allow to remove TTLs on table with default_time_to_live (CASSANDRA-11207)
+Merged from 3.0:
* Disallow creating view with a static column (CASSANDRA-11602)
* Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593)
* Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index fffd87f,e9a2938..8dedd23
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -920,10 -903,13 +931,13 @@@ public class ColumnFamilyStore implemen
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
- volatile FSWriteError flushFailure = null;
+ volatile Throwable flushFailure = null;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<Collection<SSTableReader>> readers;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<Collection<SSTableReader>> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
@@@ -966,7 -962,9 +990,9 @@@
metric.pendingFlushes.dec();
if (flushFailure != null)
- throw flushFailure;
+ throw Throwables.propagate(flushFailure);
+
+ return commitLogUpperBound;
}
}
@@@ -1052,7 -1041,8 +1069,7 @@@
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, Collections.emptyList());
- memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, Collections.emptyList());
reclaim(memtable);
iter.remove();
}
@@@ -1070,108 -1054,23 +1081,109 @@@
{
for (Memtable memtable : memtables)
{
- flushMemtable(memtable);
- Collection<SSTableReader> readers = memtable.flush();
- memtable.cfs.data.replaceFlushed(memtable, readers);
- reclaim(memtable);
- this.readers.add(readers);
++ this.readers.add(flushMemtable(memtable));
}
}
- catch (FSWriteError e)
+ catch (Throwable t)
{
- JVMStabilityInspector.inspectThrowable(e);
- // If we weren't killed, try to continue work but do not allow CommitLog to be discarded.
- postFlush.flushFailure = e;
+ JVMStabilityInspector.inspectThrowable(t);
+ postFlush.flushFailure = t;
}
-
// signal the post-flush we've done our work
postFlush.latch.countDown();
}
- public void flushMemtable(Memtable memtable)
++ public Collection<SSTableReader> flushMemtable(Memtable memtable)
+ {
+ List<Future<SSTableMultiWriter>> futures = new ArrayList<>();
+ long totalBytesOnDisk = 0;
+ long maxBytesOnDisk = 0;
+ long minBytesOnDisk = Long.MAX_VALUE;
+ List<SSTableReader> sstables = new ArrayList<>();
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH))
+ {
+ List<Memtable.FlushRunnable> flushRunnables = null;
+ List<SSTableMultiWriter> flushResults = null;
+
+ try
+ {
+ // flush the memtable
+ flushRunnables = memtable.flushRunnables(txn);
+
+ for (int i = 0; i < flushRunnables.size(); i++)
+ futures.add(perDiskflushExecutors[i].submit(flushRunnables.get(i)));
+
+ flushResults = Lists.newArrayList(FBUtilities.waitOnFutures(futures));
+ }
+ catch (Throwable t)
+ {
+ t = memtable.abortRunnables(flushRunnables, t);
+ t = txn.abort(t);
+ throw Throwables.propagate(t);
+ }
+
+ try
+ {
+ Iterator<SSTableMultiWriter> writerIterator = flushResults.iterator();
+ while (writerIterator.hasNext())
+ {
+ @SuppressWarnings("resource")
+ SSTableMultiWriter writer = writerIterator.next();
+ if (writer.getFilePointer() > 0)
+ {
+ writer.setOpenResult(true).prepareToCommit();
+ }
+ else
+ {
+ maybeFail(writer.abort(null));
+ writerIterator.remove();
+ }
+ }
+ }
+ catch (Throwable t)
+ {
+ for (SSTableMultiWriter writer : flushResults)
+ t = writer.abort(t);
+ t = txn.abort(t);
+ Throwables.propagate(t);
+ }
+
+ txn.prepareToCommit();
+
+ Throwable accumulate = null;
+ for (SSTableMultiWriter writer : flushResults)
+ accumulate = writer.commit(accumulate);
+
+ maybeFail(txn.commit(accumulate));
+
+ for (SSTableMultiWriter writer : flushResults)
+ {
+ Collection<SSTableReader> flushedSSTables = writer.finished();
+ for (SSTableReader sstable : flushedSSTables)
+ {
+ if (sstable != null)
+ {
+ sstables.add(sstable);
+ long size = sstable.bytesOnDisk();
+ totalBytesOnDisk += size;
+ maxBytesOnDisk = Math.max(maxBytesOnDisk, size);
+ minBytesOnDisk = Math.min(minBytesOnDisk, size);
+ }
+ }
+ }
+ }
- memtable.cfs.replaceFlushed(memtable, sstables);
++ memtable.cfs.data.replaceFlushed(memtable, sstables);
+ reclaim(memtable);
+ memtable.cfs.compactionStrategyManager.compactionLogger.flush(sstables);
+ logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}",
+ sstables,
+ sstables.size(),
+ FBUtilities.prettyPrintMemory(totalBytesOnDisk),
+ FBUtilities.prettyPrintMemory(maxBytesOnDisk),
+ FBUtilities.prettyPrintMemory(minBytesOnDisk));
++ return sstables;
+ }
+
private void reclaim(final Memtable memtable)
{
// issue a read barrier for reclaiming the memory, and offload the wait to another thread
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 6fe89a9,93dc5af..27c5372
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -34,7 -34,7 +34,6 @@@ import org.apache.cassandra.config.Colu
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
--import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
@@@ -44,19 -44,16 +43,15 @@@ import org.apache.cassandra.db.rows.Unf
import org.apache.cassandra.dht.*;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.index.transactions.UpdateTransaction;
- import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableTxnWriter;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
- import org.apache.cassandra.io.sstable.SSTableTxnWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.concurrent.OpOrder;
- import org.apache.cassandra.utils.memory.HeapAllocator;
import org.apache.cassandra.utils.memory.MemtableAllocator;
import org.apache.cassandra.utils.memory.MemtablePool;
@@@ -257,49 -263,6 +261,48 @@@ public class Memtable implements Compar
return partitions.size();
}
+ public List<FlushRunnable> flushRunnables(LifecycleTransaction txn)
+ {
+ List<Range<Token>> localRanges = Range.sort(StorageService.instance.getLocalRanges(cfs.keyspace.getName()));
+
+ if (!cfs.getPartitioner().splitter().isPresent() || localRanges.isEmpty())
- return Collections.singletonList(new FlushRunnable(lastReplayPosition.get(), txn));
++ return Collections.singletonList(new FlushRunnable(txn));
+
+ return createFlushRunnables(localRanges, txn);
+ }
+
+ private List<FlushRunnable> createFlushRunnables(List<Range<Token>> localRanges, LifecycleTransaction txn)
+ {
+ assert cfs.getPartitioner().splitter().isPresent();
+
+ Directories.DataDirectory[] locations = cfs.getDirectories().getWriteableLocations();
+ List<PartitionPosition> boundaries = StorageService.getDiskBoundaries(localRanges, cfs.getPartitioner(), locations);
+ List<FlushRunnable> runnables = new ArrayList<>(boundaries.size());
+ PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound();
- ReplayPosition context = lastReplayPosition.get();
+ try
+ {
+ for (int i = 0; i < boundaries.size(); i++)
+ {
+ PartitionPosition t = boundaries.get(i);
- runnables.add(new FlushRunnable(context, rangeStart, t, locations[i], txn));
++ runnables.add(new FlushRunnable(rangeStart, t, locations[i], txn));
+ rangeStart = t;
+ }
+ return runnables;
+ }
+ catch (Throwable e)
+ {
+ throw Throwables.propagate(abortRunnables(runnables, e));
+ }
+ }
+
+ public Throwable abortRunnables(List<FlushRunnable> runnables, Throwable t)
+ {
+ if (runnables != null)
+ for (FlushRunnable runnable : runnables)
+ t = runnable.writer.abort(t);
+ return t;
+ }
+
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@@ -352,75 -315,46 +355,68 @@@
return partitions.get(key);
}
- public long creationTime()
- public Collection<SSTableReader> flush()
-- {
- return creationTime;
- long estimatedSize = estimatedSize();
- Directories.DataDirectory dataDirectory = cfs.getDirectories().getWriteableLocation(estimatedSize);
- if (dataDirectory == null)
- throw new RuntimeException("Insufficient disk space to write " + estimatedSize + " bytes");
- File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- return writeSortedContents(sstableDirectory);
-- }
--
public long getMinTimestamp()
{
return minTimestamp;
}
- private long estimatedSize()
+ class FlushRunnable implements Callable<SSTableMultiWriter>
{
- public final ReplayPosition context;
- long keySize = 0;
- for (PartitionPosition key : partitions.keySet())
+ private final long estimatedSize;
+ private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush;
+
+ private final boolean isBatchLogTable;
+ private final SSTableMultiWriter writer;
+
+ // keeping these to be able to log what we are actually flushing
+ private final PartitionPosition from;
+ private final PartitionPosition to;
+
- FlushRunnable(ReplayPosition context, PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn)
++ FlushRunnable(PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn)
{
- this(context, partitions.subMap(from, to), flushLocation, from, to, txn);
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
++ this(partitions.subMap(from, to), flushLocation, from, to, txn);
}
- return (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
- FlushRunnable(ReplayPosition context, LifecycleTransaction txn)
- private Collection<SSTableReader> writeSortedContents(File sstableDirectory)
- {
- boolean isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
++ FlushRunnable(LifecycleTransaction txn)
+ {
- this(context, partitions, null, null, null, txn);
++ this(partitions, null, null, null, txn);
+ }
+
- FlushRunnable(ReplayPosition context, ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn)
++ FlushRunnable(ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn)
+ {
- this.context = context;
+ this.toFlush = toFlush;
+ this.from = from;
+ this.to = to;
+ long keySize = 0;
+ for (PartitionPosition key : toFlush.keySet())
+ {
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey) key).getKey().remaining();
+ }
+ estimatedSize = (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+
+ this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- logger.debug("Writing {}", Memtable.this.toString());
+ if (flushLocation == null)
+ writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
+ else
+ writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
+
+ }
- Collection<SSTableReader> ssTables;
- try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
+ protected Directories getDirectories()
{
+ return cfs.getDirectories();
+ }
+
- private void writeSortedContents(ReplayPosition context)
++ private void writeSortedContents()
+ {
+ logger.debug("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to);
+
boolean trackContention = logger.isTraceEnabled();
int heavilyContendedRowCount = 0;
// (we can't clear out the map as-we-go to free up memory,
@@@ -447,37 -381,59 +443,39 @@@
}
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getFilePointer()),
- commitLogUpperBound));
-
- // sstables should contain non-repaired data.
- ssTables = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), commitLogUpperBound);
- writer.abort();
- ssTables = Collections.emptyList();
- }
+ long bytesFlushed = writer.getFilePointer();
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(bytesFlushed),
- context));
++ commitLogUpperBound));
+ // Update the metrics
+ cfs.metric.bytesFlushed.inc(bytesFlushed);
if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
-
- return ssTables;
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, toFlush.size(), Memtable.this.toString()));
}
- }
- @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
- public SSTableTxnWriter createFlushWriter(String filename,
- PartitionColumns columns,
- EncodingStats stats)
- {
- // we operate "offline" here, as we expose the resulting reader consciously when done
- // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
- LifecycleTransaction txn = null;
- try
+ public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn,
+ String filename,
+ PartitionColumns columns,
+ EncodingStats stats)
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- txn = LifecycleTransaction.offline(OperationType.FLUSH);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
- .commitLogLowerBound(commitLogLowerBound.get())
- .commitLogUpperBound(commitLogUpperBound.get());
-
- return new SSTableTxnWriter(txn,
- cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
- (long) partitions.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats),
- txn));
++ .commitLogLowerBound(commitLogLowerBound.get())
++ .commitLogUpperBound(commitLogUpperBound.get());
+ return cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
+ (long)toFlush.size(),
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ sstableMetadataCollector,
+ new SerializationHeader(true, cfs.metadata, columns, stats), txn);
+
}
- catch (Throwable t)
+
+ @Override
+ public SSTableMultiWriter call()
{
- writeSortedContents(context);
- if (txn != null)
- txn.close();
- throw t;
++ writeSortedContents();
+ return writer;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index e00a0a4,f45a47a..93c6c50
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@@ -23,17 -23,8 +23,16 @@@ import java.io.EOFException
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
- import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@@@ -45,27 -35,23 +44,25 @@@ import com.google.common.base.Throwable
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
- import com.google.common.collect.Ordering;
import com.google.common.util.concurrent.Uninterruptibles;
-
import org.apache.commons.lang3.StringUtils;
+import org.cliffc.high_scale_lib.NonBlockingHashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.UnknownColumnFamilyException;
+import org.apache.cassandra.db.commitlog.SegmentReader.SyncSegment;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.util.FileSegmentInputStream;
-import org.apache.cassandra.io.util.RebufferingInputStream;
-import org.apache.cassandra.schema.CompressionParams;
-import org.apache.cassandra.io.compress.ICompressor;
+import org.apache.cassandra.db.rows.SerializationHelper;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
@@@ -89,10 -71,10 +86,10 @@@ public class CommitLogReplaye
private static final int LEGACY_END_OF_SEGMENT_MARKER = 0;
private final Set<Keyspace> keyspacesRecovered;
- private final List<Future<?>> futures;
+ private final Queue<Future<Integer>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final CRC32 checksum;
private byte[] buffer;
@@@ -101,79 -83,12 +98,77 @@@
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
+ /*
+ * Wrapper around initiating mutations read from the log to make it possible
+ * to spy on initiated mutations for test
+ */
+ @VisibleForTesting
+ public static class MutationInitiator
+ {
+ protected Future<Integer> initiateMutation(final Mutation mutation,
+ final long segmentId,
+ final int serializedSize,
- final long entryLocation,
++ final int entryLocation,
+ final CommitLogReplayer clr)
+ {
+ Runnable runnable = new WrappedRunnable()
+ {
+ public void runMayThrow()
+ {
+ if (Schema.instance.getKSMetaData(mutation.getKeyspaceName()) == null)
+ return;
+ if (clr.pointInTimeExceeded(mutation))
+ return;
+
+ final Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
+
+ // Rebuild the mutation, omitting column families that
+ // a) the user has requested that we ignore,
+ // b) have already been flushed,
+ // or c) are part of a cf that was dropped.
+ // Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
+ Mutation newMutation = null;
+ for (PartitionUpdate update : clr.replayFilter.filter(mutation))
+ {
+ if (Schema.instance.getCF(update.metadata().cfId) == null)
+ continue; // dropped
+
- ReplayPosition rp = clr.cfPositions.get(update.metadata().cfId);
-
+ // replay if current segment is newer than last flushed one or,
+ // if it is the last known segment, if we are after the replay position
- if (segmentId > rp.segment || (segmentId == rp.segment && entryLocation > rp.position))
++ if (clr.shouldReplay(update.metadata().cfId, new ReplayPosition(segmentId, entryLocation)))
+ {
+ if (newMutation == null)
+ newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
+ newMutation.add(update);
+ clr.replayedCount.incrementAndGet();
+ }
+ }
+ if (newMutation != null)
+ {
+ assert !newMutation.isEmpty();
+
+ try
+ {
+ Uninterruptibles.getUninterruptibly(Keyspace.open(newMutation.getKeyspaceName()).applyFromCommitLog(newMutation));
+ }
+ catch (ExecutionException e)
+ {
+ throw Throwables.propagate(e.getCause());
+ }
+
+ clr.keyspacesRecovered.add(keyspace);
+ }
+ }
+ };
+ return StageManager.getStage(Stage.MUTATION).submit(runnable, serializedSize);
+ }
+ }
+
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
- this.futures = new ArrayList<Future<?>>();
+ this.futures = new ArrayDeque<Future<Integer>>();
this.buffer = new byte[4096];
- this.uncompressedBuffer = new byte[4096];
this.invalidMutations = new HashMap<UUID, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
@@@ -342,11 -285,22 +334,24 @@@
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
+ @SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
+ // just transform from the file name (no reading of headers) to determine version
CommitLogDescriptor desc = CommitLogDescriptor.fromFileName(file.getName());
+
try(ChannelProxy channel = new ChannelProxy(file);
RandomAccessReader reader = RandomAccessReader.open(channel))
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index dd07b19,16090a1..be1436c
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@@ -329,10 -332,10 +334,10 @@@ public class Tracke
apply(View.markFlushing(memtable));
}
- public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ public void replaceFlushed(Memtable memtable, Iterable<SSTableReader> sstables)
{
assert !isDummy();
- if (sstables == null || Iterables.isEmpty(sstables))
- if (sstables.isEmpty())
++ if (Iterables.isEmpty(sstables))
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
// if it's null, we don't care what state the cfstore is in, we just replace it and continue
@@@ -348,14 -351,7 +353,9 @@@
Throwable fail;
fail = updateSizeTracking(emptySet(), sstables, null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
+ notifyDiscarded(memtable);
+
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
-
maybeFail(fail);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index 63926ed,17062b4..cde6363
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -319,12 -321,14 +321,14 @@@ public class Vie
List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
- if (flushed == null)
- if (flushed == null || flushed.isEmpty())
++ if (flushed == null || Iterables.isEmpty(flushed))
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compactingMap, view.intervalTree);
+ view.compactingMap, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), flushed);
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compactingMap,
+ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, emptySet(), flushed);
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), flushed);
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compactingMap, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/db/view/TableViews.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index e030b5b,e0fb3b1..9b6f491
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@@ -120,8 -121,10 +120,10 @@@ public class BigFormat implements SSTab
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
// ma (3.0.0): swap bf hash order
// store rows natively
- // mb (3.0.6): commit log lower bound included
++ // mb (3.0.7, 3.7): commit log lower bound included
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index c574a48,53cf0b0..ca50a44
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -20,10 -20,14 +20,11 @@@ package org.apache.cassandra.io.sstable
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import com.google.common.collect.Maps;
+ import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index d2e0513,420b802..3b3d7e1
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@@ -116,11 -70,9 +116,12 @@@ public class SSTableMetadataViewe
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
+ out.printf("totalColumnsSet: %s%n", stats.totalColumnsSet);
+ out.printf("totalRows: %s%n", stats.totalRows);
out.println("Estimated tombstone drop times:");
+
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
out.printf("%-10s:%10s%n",entry.getKey().intValue(), entry.getValue());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --cc test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index e6f9499,31dea3e..8e45eea
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@@ -461,13 -448,11 +461,13 @@@ public class CommitLogStressTes
super(log, discardedPos, null, ReplayFilter.create());
}
- int hash = 0;
- int cells = 0;
+ int hash;
+ int cells;
+ int discarded;
+ int skipped;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
index 788757c,baf9466..5ac53f6
--- a/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/RecoveryManagerTest.java
@@@ -56,65 -50,6 +56,65 @@@ import org.apache.cassandra.db.commitlo
public class RecoveryManagerTest
{
private static Logger logger = LoggerFactory.getLogger(RecoveryManagerTest.class);
+ static final Semaphore blocker = new Semaphore(0);
+ static final Semaphore blocked = new Semaphore(0);
+ static CommitLogReplayer.MutationInitiator originalInitiator = null;
+ static final CommitLogReplayer.MutationInitiator mockInitiator = new CommitLogReplayer.MutationInitiator()
+ {
+ @Override
+ protected Future<Integer> initiateMutation(final Mutation mutation,
+ final long segmentId,
+ final int serializedSize,
- final long entryLocation,
++ final int entryLocation,
+ final CommitLogReplayer clr)
+ {
+ final Future<Integer> toWrap = super.initiateMutation(mutation,
+ segmentId,
+ serializedSize,
+ entryLocation,
+ clr);
+ return new Future<Integer>()
+ {
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isCancelled()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isDone()
+ {
+ return blocker.availablePermits() > 0 && toWrap.isDone();
+ }
+
+ @Override
+ public Integer get() throws InterruptedException, ExecutionException
+ {
+ System.out.println("Got blocker once");
+ blocked.release();
+ blocker.acquire();
+ return toWrap.get();
+ }
+
+ @Override
+ public Integer get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException
+ {
+ blocked.release();
+ blocker.tryAcquire(1, timeout, unit);
+ return toWrap.get(timeout, unit);
+ }
+
+ };
+ }
+ };
private static final String KEYSPACE1 = "RecoveryManagerTest1";
private static final String CF_STANDARD1 = "Standard1";
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index b5cbf8b,555cdda..1ea0eb1
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@@ -584,147 -555,21 +584,147 @@@ public class CommitLogTes
}
@Test
- public void testDescriptorInvalidParametersSize() throws IOException
+ public void replay_Compressed_Deflate() throws IOException
{
- Map<String, String> params = new HashMap<>();
- for (int i=0; i<65535; ++i)
- params.put("key"+i, Integer.toString(i, 16));
- try {
- CommitLogDescriptor desc = new CommitLogDescriptor(CommitLogDescriptor.VERSION_30,
- 21,
- new ParameterizedClass("LZ4Compressor", params));
- ByteBuffer buf = ByteBuffer.allocate(1024000);
- CommitLogDescriptor.writeHeader(buf, desc);
- Assert.fail("Parameter object too long should fail on writing descriptor.");
- } catch (ConfigurationException e)
+ replay_Compressed(new ParameterizedClass(DeflateCompressor.class.getName(), Collections.<String, String>emptyMap()));
+ }
+
+ private void replay_Compressed(ParameterizedClass parameterizedClass) throws IOException
+ {
+ ParameterizedClass originalCompression = DatabaseDescriptor.getCommitLogCompression();
+ EncryptionContext originalEncryptionContext = DatabaseDescriptor.getEncryptionContext();
+ try
+ {
+ DatabaseDescriptor.setCommitLogCompression(parameterizedClass);
+ DatabaseDescriptor.setEncryptionContext(EncryptionContextGenerator.createDisabledContext());
+ CommitLog.instance.resetUnsafe(true);
+
+ replaySimple(CommitLog.instance);
+ replayWithDiscard(CommitLog.instance);
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitLogCompression(originalCompression);
+ DatabaseDescriptor.setEncryptionContext(originalEncryptionContext);
+ CommitLog.instance.resetUnsafe(true);
+ }
+ }
+
+ @Test
+ public void replay_Encrypted() throws IOException
+ {
+ ParameterizedClass originalCompression = DatabaseDescriptor.getCommitLogCompression();
+ EncryptionContext originalEncryptionContext = DatabaseDescriptor.getEncryptionContext();
+ try
+ {
+ DatabaseDescriptor.setCommitLogCompression(null);
+ DatabaseDescriptor.setEncryptionContext(EncryptionContextGenerator.createContext(true));
+ CommitLog.instance.resetUnsafe(true);
+
+ replaySimple(CommitLog.instance);
+ replayWithDiscard(CommitLog.instance);
+ }
+ finally
+ {
+ DatabaseDescriptor.setCommitLogCompression(originalCompression);
+ DatabaseDescriptor.setEncryptionContext(originalEncryptionContext);
+ CommitLog.instance.resetUnsafe(true);
+ }
+ }
+
+ private void replaySimple(CommitLog commitLog) throws IOException
+ {
+ int cellCount = 0;
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k1")
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ cellCount += 1;
+ commitLog.add(rm1);
+
+ final Mutation rm2 = new RowUpdateBuilder(cfs.metadata, 0, "k2")
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ cellCount += 1;
+ commitLog.add(rm2);
+
+ commitLog.sync(true);
+
+ Replayer replayer = new Replayer(commitLog, ReplayPosition.NONE);
+ List<String> activeSegments = commitLog.getActiveSegmentNames();
+ Assert.assertFalse(activeSegments.isEmpty());
+
+ File[] files = new File(commitLog.location).listFiles((file, name) -> activeSegments.contains(name));
+ replayer.recover(files);
+
+ assertEquals(cellCount, replayer.cells);
+ }
+
+ private void replayWithDiscard(CommitLog commitLog) throws IOException
+ {
+ int cellCount = 0;
+ int max = 1024;
+ int discardPosition = (int)(max * .8); // an arbitrary number of entries that we'll skip on the replay
+ ReplayPosition replayPosition = null;
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+ for (int i = 0; i < max; i++)
+ {
+ final Mutation rm1 = new RowUpdateBuilder(cfs.metadata, 0, "k" + 1)
+ .clustering("bytes")
+ .add("val", bytes("this is a string"))
+ .build();
+ ReplayPosition position = commitLog.add(rm1);
+
+ if (i == discardPosition)
+ replayPosition = position;
+ if (i > discardPosition)
+ {
+ cellCount += 1;
+ }
+ }
+
+ commitLog.sync(true);
+
+ Replayer replayer = new Replayer(commitLog, replayPosition);
+ List<String> activeSegments = commitLog.getActiveSegmentNames();
+ Assert.assertFalse(activeSegments.isEmpty());
+
+ File[] files = new File(commitLog.location).listFiles((file, name) -> activeSegments.contains(name));
+ replayer.recover(files);
+
+ assertEquals(cellCount, replayer.cells);
+ }
+
+ class Replayer extends CommitLogReplayer
+ {
+ private final ReplayPosition filterPosition;
+ int cells;
+ int skipped;
+
+ Replayer(CommitLog commitLog, ReplayPosition filterPosition)
+ {
- super(commitLog, filterPosition, null, ReplayFilter.create());
++ super(commitLog, filterPosition, Collections.emptyMap(), ReplayFilter.create());
+ this.filterPosition = filterPosition;
+ }
+
+ @SuppressWarnings("resource")
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc) throws IOException
++ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
- // correct path
+ if (entryLocation <= filterPosition.position)
+ {
+ // Skip over this mutation.
+ skipped++;
+ return;
+ }
+
+ FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
+ Mutation mutation = Mutation.serializer.deserialize(new DataInputPlus.DataInputStreamPlus(bufIn), desc.getMessagingVersion(), SerializationHelper.Flag.LOCAL);
+ for (PartitionUpdate partitionUpdate : mutation.getPartitionUpdates())
+ for (Row row : partitionUpdate)
+ cells += Iterables.size(row.cells());
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 0294115,b8de711..902e17a
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@@ -298,11 -298,13 +299,14 @@@ public class TrackerTes
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
- tracker.replaceFlushed(prev2, Collections.singleton(reader));
+ tracker.replaceFlushed(prev2, singleton(reader));
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(singleton(reader));
+ Assert.assertEquals(0, tracker.getView().premature.size());
- Assert.assertEquals(1, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertEquals(2, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
++ Assert.assertEquals(prev2, ((MemtableDiscardedNotification) listener.received.get(0)).memtable);
++ Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
listener.received.clear();
Assert.assertTrue(reader.isKeyCacheSetup());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
@@@ -316,17 -318,13 +320,17 @@@
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
- tracker.replaceFlushed(prev1, Collections.singleton(reader));
+ tracker.replaceFlushed(prev1, singleton(reader));
+ tracker.permitCompactionOfFlushed(Collections.singleton(reader));
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ System.out.println(listener.received);
- Assert.assertEquals(5, listener.received.size());
++ Assert.assertEquals(4, listener.received.size());
+ Assert.assertEquals(prev1, ((MemtableSwitchedNotification) listener.received.get(0)).memtable);
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added);
- Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(2)).memtable);
- Assert.assertTrue(listener.received.get(3) instanceof SSTableDeletingNotification);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(4)).removed.size());
++ Assert.assertEquals(prev1, ((MemtableDiscardedNotification) listener.received.get(1)).memtable);
++ Assert.assertTrue(listener.received.get(2) instanceof SSTableDeletingNotification);
++ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(3)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c9ac0506/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
----------------------------------------------------------------------
[17/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
Merge commit '849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/78a3d2bb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/78a3d2bb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/78a3d2bb
Branch: refs/heads/trunk
Commit: 78a3d2bba95b9efcda152a157f822f4970f22636
Parents: e9657a4 849a438
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:17:51 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:19:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 282 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 243 ++++++++--------
.../db/commitlog/CommitLogReplayer.java | 56 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 37 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 23 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
92 files changed, 715 insertions(+), 381 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 306f46f,dfcad10..677ea11
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,7 +1,31 @@@
- * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+3.0.7
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
+ * Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395)
-2.2.7
- * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+3.0.6
+ * Disallow creating view with a static column (CASSANDRA-11602)
+ * Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593)
+ * Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618)
+ * Fix queries with filtering on counter columns (CASSANDRA-11629)
+ * Improve tombstone printing in sstabledump (CASSANDRA-11655)
+ * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669)
+ * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600)
+ * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654)
+ * Ignore all LocalStrategy keyspaces for streaming and other related
+ operations (CASSANDRA-11627)
+ * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523)
+ * Only open one sstable scanner per sstable (CASSANDRA-11412)
+ * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410)
+ * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485)
+ * LogAwareFileLister should only use OLD sstable files in current folder to determine disk consistency (CASSANDRA-11470)
+ * Notify indexers of expired rows during compaction (CASSANDRA-11329)
+ * Properly respond with ProtocolError when a v1/v2 native protocol
+ header is received (CASSANDRA-11464)
+ * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120)
+Merged from 2.2:
++ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
* Exit JVM if JMX server fails to startup (CASSANDRA-11540)
* Produce a heap dump when exiting on OOM (CASSANDRA-9861)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 5b8de8f,88e22c0..e9a2938
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -892,10 -996,13 +903,13 @@@ public class ColumnFamilyStore implemen
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ volatile FSWriteError flushFailure = null;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
- final List<SSTableReader> readers;
- volatile FSWriteError flushFailure = null;
++ final List<Collection<SSTableReader>> readers;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
- List<Memtable> memtables, List<SSTableReader> readers)
++ List<Memtable> memtables, List<Collection<SSTableReader>> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
@@@ -927,18 -1046,23 +946,25 @@@
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
- SSTableReader reader = readers.get(i);
++ Collection<SSTableReader> reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, reader);
+ }
}
+
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
++
+ return commitLogUpperBound;
}
}
@@@ -953,7 -1077,8 +979,8 @@@
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
- final List<SSTableReader> readers = new ArrayList<>();
++ final List<Collection<SSTableReader>> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@@ -1024,27 -1139,30 +1041,23 @@@
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
- memtable.cfs.data.replaceFlushed(memtable, null);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
++ memtable.cfs.data.replaceFlushed(memtable, Collections.emptyList());
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, Collections.emptyList());
reclaim(memtable);
iter.remove();
}
}
-- if (memtables.isEmpty())
-- {
-- postFlush.latch.countDown();
-- return;
-- }
--
metric.memtableSwitchCount.inc();
try
{
for (Memtable memtable : memtables)
{
-- // flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
- SSTableReader reader = memtable.flush();
- memtable.cfs.data.replaceFlushed(memtable, reader);
++ Collection<SSTableReader> readers = memtable.flush();
++ memtable.cfs.data.replaceFlushed(memtable, readers);
reclaim(memtable);
- readers.add(reader);
++ this.readers.add(readers);
}
}
catch (FSWriteError e)
@@@ -1406,22 -1655,172 +1451,32 @@@
return data;
}
+ public Collection<SSTableReader> getSSTables()
+ {
+ return data.getSSTables();
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
- public Set<SSTableReader> getUncompactingSSTables()
+ public Set<SSTableReader> getLiveSSTables()
{
- return data.getUncompacting();
+ return data.getView().liveSSTables();
}
- public ColumnFamily getColumnFamily(DecoratedKey key,
- Composite start,
- Composite finish,
- boolean reversed,
- int limit,
- long timestamp)
+ public Iterable<SSTableReader> getSSTables(SSTableSet sstableSet)
{
- return getColumnFamily(QueryFilter.getSliceFilter(key, name, start, finish, reversed, limit, timestamp));
+ return data.getView().sstables(sstableSet);
}
- /**
- * Fetch the row and columns given by filter.key if it is in the cache; if not, read it from disk and cache it
- *
- * If row is cached, and the filter given is within its bounds, we return from cache, otherwise from disk
- *
- * If row is not cached, we figure out what filter is "biggest", read that from disk, then
- * filter the result and either cache that or return it.
- *
- * @param cfId the column family to read the row from
- * @param filter the columns being queried.
- * @return the requested data for the filter provided
- */
- private ColumnFamily getThroughCache(UUID cfId, QueryFilter filter)
+ public Iterable<SSTableReader> getUncompactingSSTables()
{
- assert isRowCacheEnabled()
- : String.format("Row cache is not enabled on table [" + name + "]");
-
- RowCacheKey key = new RowCacheKey(metadata.ksAndCFName, filter.key);
-
- // attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
- // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
- // TODO: don't evict entire rows on writes (#2864)
- IRowCacheEntry cached = CacheService.instance.rowCache.get(key);
- if (cached != null)
- {
- if (cached instanceof RowCacheSentinel)
- {
- // Some other read is trying to cache the value, just do a normal non-caching read
- Tracing.trace("Row cache miss (race)");
- metric.rowCacheMiss.inc();
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- ColumnFamily cachedCf = (ColumnFamily)cached;
- if (isFilterFullyCoveredBy(filter.filter, cachedCf, filter.timestamp))
- {
- metric.rowCacheHit.inc();
- Tracing.trace("Row cache hit");
- ColumnFamily result = filterColumnFamily(cachedCf, filter);
- metric.updateSSTableIterated(0);
- return result;
- }
-
- metric.rowCacheHitOutOfRange.inc();
- Tracing.trace("Ignoring row cache as cached value could not satisfy query");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- metric.rowCacheMiss.inc();
- Tracing.trace("Row cache miss");
- RowCacheSentinel sentinel = new RowCacheSentinel();
- boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel);
- ColumnFamily data = null;
- ColumnFamily toCache = null;
- try
- {
- // If we are explicitely asked to fill the cache with full partitions, we go ahead and query the whole thing
- if (metadata.getCaching().rowCache.cacheFullPartitions())
- {
- data = getTopLevelColumns(QueryFilter.getIdentityFilter(filter.key, name, filter.timestamp), Integer.MIN_VALUE);
- toCache = data;
- Tracing.trace("Populating row cache with the whole partition");
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return filterColumnFamily(data, filter);
- }
-
- // Otherwise, if we want to cache the result of the query we're about to do, we must make sure this query
- // covers what needs to be cached. And if the user filter does not satisfy that, we sometimes extend said
- // filter so we can populate the cache but only if:
- // 1) we can guarantee it is a strict extension, i.e. that we will still fetch the data asked by the user.
- // 2) the extension does not make us query more than getRowsPerPartitionToCache() (as a mean to limit the
- // amount of extra work we'll do on a user query for the purpose of populating the cache).
- //
- // In practice, we can only guarantee those 2 points if the filter is one that queries the head of the
- // partition (and if that filter actually counts CQL3 rows since that's what we cache and it would be
- // bogus to compare the filter count to the 'rows to cache' otherwise).
- if (filter.filter.isHeadFilter() && filter.filter.countCQL3Rows(metadata.comparator))
- {
- SliceQueryFilter sliceFilter = (SliceQueryFilter)filter.filter;
- int rowsToCache = metadata.getCaching().rowCache.rowsToCache;
-
- SliceQueryFilter cacheSlice = readFilterForCache();
- QueryFilter cacheFilter = new QueryFilter(filter.key, name, cacheSlice, filter.timestamp);
-
- // If the filter count is less than the number of rows cached, we simply extend it to make sure we do cover the
- // number of rows to cache, and if that count is greater than the number of rows to cache, we simply filter what
- // needs to be cached afterwards.
- if (sliceFilter.count < rowsToCache)
- {
- toCache = getTopLevelColumns(cacheFilter, Integer.MIN_VALUE);
- if (toCache != null)
- {
- Tracing.trace("Populating row cache ({} rows cached)", cacheSlice.lastCounted());
- data = filterColumnFamily(toCache, filter);
- }
- }
- else
- {
- data = getTopLevelColumns(filter, Integer.MIN_VALUE);
- if (data != null)
- {
- // The filter limit was greater than the number of rows to cache. But, if the filter had a non-empty
- // finish bound, we may have gotten less than what needs to be cached, in which case we shouldn't cache it
- // (otherwise a cache hit would assume the whole partition is cached which is not the case).
- if (sliceFilter.finish().isEmpty() || sliceFilter.lastCounted() >= rowsToCache)
- {
- toCache = filterColumnFamily(data, cacheFilter);
- Tracing.trace("Caching {} rows (out of {} requested)", cacheSlice.lastCounted(), sliceFilter.count);
- }
- else
- {
- Tracing.trace("Not populating row cache, not enough rows fetched ({} fetched but {} required for the cache)", sliceFilter.lastCounted(), rowsToCache);
- }
- }
- }
-
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return data;
- }
- else
- {
- Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
- }
- finally
- {
- if (sentinelSuccess && toCache == null)
- invalidateCachedRow(key);
- }
- }
-
- public SliceQueryFilter readFilterForCache()
- {
- // We create a new filter everytime before for now SliceQueryFilter is unfortunatly mutable.
- return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, metadata.getCaching().rowCache.rowsToCache, metadata.clusteringColumns().size());
+ return data.getUncompacting();
}
- public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now)
+ public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec)
{
// We can use the cached value only if we know that no data it doesn't contain could be covered
// by the query filter, that is if:
@@@ -1910,40 -2759,45 +1965,52 @@@
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- viewManager.forceBlockingFlush();
-
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- dumpMemtable();
- viewManager.dumpMemtables();
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
++ if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
+ {
+ replayAfter = forceBlockingFlush();
++ viewManager.forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
- final Flush flush = new Flush(true);
- flushExecutor.execute(flush);
- replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
++ viewManager.dumpMemtables();
++ try
++ {
++ replayAfter = dumpMemtable().get();
++ }
++ catch (Exception e)
++ {
++ throw new RuntimeException(e);
++ }
+ }
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
- if (takeSnapshot)
+ if (DatabaseDescriptor.isAutoSnapshot())
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
- for (SecondaryIndex index : indexManager.getIndexes())
- index.truncateBlocking(truncatedAt);
+ indexManager.truncateAllIndexesBlocking(truncatedAt);
-
- viewManager.truncateBlocking(truncatedAt);
++ viewManager.truncateBlocking(replayAfter, truncatedAt);
SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
logger.trace("cleaning out row cache");
@@@ -1955,20 -2809,7 +2022,20 @@@
logger.trace("truncate complete");
}
- public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation)
+ /**
+ * Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable.
+ */
- public void dumpMemtable()
++ public Future<ReplayPosition> dumpMemtable()
+ {
+ synchronized (data)
+ {
+ final Flush flush = new Flush(true);
+ flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
++ return postFlushExecutor.submit(flush.postFlush);
+ }
+ }
+
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
{
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 5d5f7bf,b4ada09..93dc5af
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -87,12 -81,11 +90,11 @@@ public class Memtable implements Compar
}
}
- // We index the memtable by RowPosition only for the purpose of being able
+ // We index the memtable by PartitionPosition only for the purpose of being able
// to select key range using Token.KeyBound. However put() ensures that we
// actually only store DecoratedKey.
- private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
+ private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@@ -101,14 -94,12 +103,16 @@@
// Record the comparator of the CFS at the creation of the memtable. This
// is only used when a user update the CF comparator, to know if the
// memtable was created with the new or old comparator.
- public final CellNameType initialComparator;
+ public final ClusteringComparator initialComparator;
+ private final ColumnsCollector columnsCollector;
+ private final StatsCollector statsCollector = new StatsCollector();
+
- public Memtable(ColumnFamilyStore cfs)
++ // only to be used by init(), to setup the very first memtable for the cfs
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@@ -191,12 -185,12 +200,12 @@@
public boolean isClean()
{
- return rows.isEmpty();
+ return partitions.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@@ -251,14 -256,9 +260,9 @@@
public int partitionCount()
{
- return rows.size();
+ return partitions.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@@ -266,54 -266,65 +270,60 @@@
100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
}
- /**
- * @param startWith Include data in the result from and including this key and to the end of the memtable
- * @return An iterator of entries with the data from the start key
- */
- public Iterator<Map.Entry<DecoratedKey, ColumnFamily>> getEntryIterator(final RowPosition startWith, final RowPosition stopAt)
+ public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange, final boolean isForThrift)
{
- return new Iterator<Map.Entry<DecoratedKey, ColumnFamily>>()
- {
- private Iterator<? extends Map.Entry<? extends RowPosition, AtomicBTreeColumns>> iter = stopAt.isMinimum()
- ? rows.tailMap(startWith).entrySet().iterator()
- : rows.subMap(startWith, true, stopAt, true).entrySet().iterator();
+ AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
- private Map.Entry<? extends RowPosition, ? extends ColumnFamily> currentEntry;
+ boolean startIsMin = keyRange.left.isMinimum();
+ boolean stopIsMin = keyRange.right.isMinimum();
- public boolean hasNext()
- {
- return iter.hasNext();
- }
+ boolean isBound = keyRange instanceof Bounds;
+ boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+ boolean includeStop = isBound || keyRange instanceof Range;
+ Map<PartitionPosition, AtomicBTreePartition> subMap;
+ if (startIsMin)
+ subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop);
+ else
+ subMap = stopIsMin
+ ? partitions.tailMap(keyRange.left, includeStart)
+ : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop);
- public Map.Entry<DecoratedKey, ColumnFamily> next()
- {
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
- // Actual stored key should be true DecoratedKey
- assert entryRowPosition.getKey() instanceof DecoratedKey;
- @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
- if (MEMORY_POOL.needToCopyOnHeap())
- {
- DecoratedKey key = entry.getKey();
- key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
- ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
- entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
- }
- // Store the reference to the current entry so that remove() can update the current size.
- currentEntry = entry;
- return entry;
- }
+ int minLocalDeletionTime = Integer.MAX_VALUE;
- public void remove()
- {
- iter.remove();
- liveDataSize.addAndGet(-currentEntry.getValue().dataSize());
- currentEntry = null;
- }
- };
+ // avoid iterating over the memtable if we purge all tombstones
+ if (cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones())
+ minLocalDeletionTime = findMinLocalDeletionTime(subMap.entrySet().iterator());
+
+ final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter = subMap.entrySet().iterator();
+
+ return new MemtableUnfilteredPartitionIterator(cfs, iter, isForThrift, minLocalDeletionTime, columnFilter, dataRange);
+ }
+
+ private int findMinLocalDeletionTime(Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iterator)
+ {
+ int minLocalDeletionTime = Integer.MAX_VALUE;
+ while (iterator.hasNext())
+ {
+ Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iterator.next();
+ minLocalDeletionTime = Math.min(minLocalDeletionTime, entry.getValue().stats().minLocalDeletionTime);
+ }
+ return minLocalDeletionTime;
}
- public ColumnFamily getColumnFamily(DecoratedKey key)
+ public Partition getPartition(DecoratedKey key)
{
- return rows.get(key);
+ return partitions.get(key);
}
- public long creationTime()
- public SSTableReader flush()
++ public Collection<SSTableReader> flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
- Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
++ Directories.DataDirectory dataDirectory = cfs.getDirectories().getWriteableLocation(estimatedSize);
++ if (dataDirectory == null)
++ throw new RuntimeException("Insufficient disk space to write " + estimatedSize + " bytes");
++ File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@@ -321,137 -332,86 +331,110 @@@
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- private final boolean isBatchLogTable;
-
- FlushRunnable(ReplayPosition context)
+ long keySize = 0;
- for (RowPosition key : rows.keySet())
++ for (PartitionPosition key : partitions.keySet())
{
- this.context = context;
-
- long keySize = 0;
- for (PartitionPosition key : partitions.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
-
- this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected void runMayThrow() throws Exception
- {
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- Collection<SSTableReader> sstables = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstables);
- }
- private SSTableReader writeSortedContents(File sstableDirectory)
++ private Collection<SSTableReader> writeSortedContents(File sstableDirectory)
+ {
- logger.info("Writing {}", Memtable.this.toString());
++ boolean isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- protected Directories getDirectories()
- {
- return cfs.getDirectories();
- }
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
++ logger.debug("Writing {}", Memtable.this.toString());
+
- private Collection<SSTableReader> writeSortedContents(ReplayPosition context, File sstableDirectory)
++ Collection<SSTableReader> ssTables;
++ try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- Collection<SSTableReader> ssTables;
- try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
++ for (AtomicBTreePartition partition : partitions.values())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (AtomicBTreePartition partition : partitions.values())
- AtomicBTreeColumns cf = entry.getValue();
++ // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
++ // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
++ // we don't need to preserve tombstones for repair. So if both operation are in this
++ // memtable (which will almost always be the case if there is no ongoing failure), we can
++ // just skip the entry (CASSANDRA-4667).
++ if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
++ continue;
++
++ if (trackContention && partition.usePessimisticLocking())
++ heavilyContendedRowCount++;
+
- if (cf.isMarkedForDelete() && cf.hasColumns())
++ if (!partition.isEmpty())
{
- // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
- // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
- // we don't need to preserve tombstones for repair. So if both operation are in this
- // memtable (which will almost always be the case if there is no ongoing failure), we can
- // just skip the entry (CASSANDRA-4667).
- if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
-- continue;
-
- if (trackContention && partition.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!partition.isEmpty())
++ try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+ {
- try (UnfilteredRowIterator iter = partition.unfilteredIterator())
- {
- writer.append(iter);
- }
++ writer.append(iter);
+ }
}
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getFilePointer()),
- context));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
++ FBUtilities.prettyPrintMemory(writer.getFilePointer()),
+ commitLogUpperBound));
- // sstables should contain non-repaired data.
- ssTables = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTables = null;
- }
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
++ // sstables should contain non-repaired data.
++ ssTables = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
- ssTable = null;
++ ssTables = Collections.emptyList();
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
+ if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
++ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
- return ssTables;
- }
- return ssTable;
++ return ssTables;
}
+ }
- @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
- public SSTableTxnWriter createFlushWriter(String filename,
- PartitionColumns columns,
- EncodingStats stats)
- private SSTableWriter createFlushWriter(String filename)
++ @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
++ public SSTableTxnWriter createFlushWriter(String filename,
++ PartitionColumns columns,
++ EncodingStats stats)
+ {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
- .commitLogLowerBound(commitLogLowerBound.get())
- .commitLogUpperBound(commitLogUpperBound.get());
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
++ // we operate "offline" here, as we expose the resulting reader consciously when done
++ // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
++ LifecycleTransaction txn = null;
++ try
+ {
- // we operate "offline" here, as we expose the resulting reader consciously when done
- // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
- LifecycleTransaction txn = null;
- try
- {
- txn = LifecycleTransaction.offline(OperationType.FLUSH);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- return new SSTableTxnWriter(txn,
- cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
- (long) partitions.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats),
- txn));
- }
- catch (Throwable t)
- {
- if (txn != null)
- txn.close();
- throw t;
- }
++ txn = LifecycleTransaction.offline(OperationType.FLUSH);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
++ .commitLogLowerBound(commitLogLowerBound.get())
++ .commitLogUpperBound(commitLogUpperBound.get());
++
++ return new SSTableTxnWriter(txn,
++ cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
++ (long) partitions.size(),
++ ActiveRepairService.UNREPAIRED_SSTABLE,
++ sstableMetadataCollector,
++ new SerializationHeader(true, cfs.metadata, columns, stats),
++ txn));
++ }
++ catch (Throwable t)
++ {
++ if (txn != null)
++ txn.close();
++ throw t;
+ }
}
private static int estimateRowOverhead(final int count)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 7169b2f,a58aeb4..f45a47a
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@@ -35,9 -33,6 +35,8 @@@ import com.google.common.base.Throwable
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
- import com.google.common.collect.Ordering;
+import com.google.common.util.concurrent.Uninterruptibles;
+
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -46,18 -43,16 +45,17 @@@ import org.apache.cassandra.concurrent.
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.compress.CompressionParameters;
+import org.apache.cassandra.io.util.FileSegmentInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.io.util.ByteBufferDataInput;
-import org.apache.cassandra.io.util.FastByteArrayInputStream;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.CRC32Factory;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.WrappedRunnable;
@@@ -76,9 -69,9 +74,9 @@@ public class CommitLogReplaye
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
- private final ICRC32 checksum;
+ private final CRC32 checksum;
private byte[] buffer;
private byte[] uncompressedBuffer;
@@@ -94,8 -87,8 +92,8 @@@
this.invalidMutations = new HashMap<UUID, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
- this.checksum = CRC32Factory.instance.create();
+ this.checksum = new CRC32();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@@ -104,17 -97,12 +102,12 @@@
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables(SSTableSet.CANONICAL));
-
- // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@@ -584,16 -580,12 +592,14 @@@
// or c) are part of a cf that was dropped.
// Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
Mutation newMutation = null;
- for (ColumnFamily columnFamily : replayFilter.filter(mutation))
+ for (PartitionUpdate update : replayFilter.filter(mutation))
{
- if (Schema.instance.getCF(columnFamily.id()) == null)
+ if (Schema.instance.getCF(update.metadata().cfId) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(update.metadata().cfId);
-
- if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
+ // replay if current segment is newer than last flushed one or,
+ // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
++ if (shouldReplay(update.metadata().cfId, new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index 28416f3,17802ad..0b21763
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@@ -17,11 -17,12 +17,11 @@@
*/
package org.apache.cassandra.db.commitlog;
-import java.io.DataInput;
import java.io.IOException;
- import java.util.Comparator;
+ import java.util.Map;
+ import java.util.NavigableMap;
+ import java.util.TreeMap;
- import com.google.common.base.Function;
- import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index c205d5c,575aa51..8928db5
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@@ -237,13 -228,10 +237,10 @@@ public abstract class AbstractCompactio
* Handle a flushed memtable.
*
* @param memtable the flushed memtable
- * @param sstable the written sstable. can be null if the memtable was clean.
+ * @param sstables the written sstables. can be null or empty if the memtable was clean.
*/
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 82fd872,0000000..f1127c9
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@@ -1,504 -1,0 +1,501 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.compaction;
+
+
+import java.util.*;
+import java.util.concurrent.Callable;
+
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * Manages the compaction strategies.
+ *
+ * Currently has two instances of actual compaction strategies - one for repaired data and one for
+ * unrepaired data. This is done to be able to totally separate the different sets of sstables.
+ */
+public class CompactionStrategyManager implements INotificationConsumer
+{
+ private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManager.class);
+ private final ColumnFamilyStore cfs;
+ private volatile AbstractCompactionStrategy repaired;
+ private volatile AbstractCompactionStrategy unrepaired;
+ private volatile boolean enabled = true;
+ public boolean isActive = true;
+ private volatile CompactionParams params;
+ /*
+ We keep a copy of the schema compaction parameters here to be able to decide if we
+ should update the compaction strategy in maybeReloadCompactionStrategy() due to an ALTER.
+
+ If a user changes the local compaction strategy and then later ALTERs a compaction parameter,
+ we will use the new compaction parameters.
+ */
+ private CompactionParams schemaCompactionParams;
+
+ public CompactionStrategyManager(ColumnFamilyStore cfs)
+ {
+ cfs.getTracker().subscribe(this);
+ logger.trace("{} subscribed to the data tracker.", this);
+ this.cfs = cfs;
+ reload(cfs.metadata);
+ params = cfs.metadata.params.compaction;
+ enabled = params.isEnabled();
+ }
+
+ /**
+ * Return the next background task
+ *
+ * Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks)
+ *
+ */
+ public synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
+ {
+ if (!isEnabled())
+ return null;
+
+ maybeReload(cfs.metadata);
+
+ if (repaired.getEstimatedRemainingTasks() > unrepaired.getEstimatedRemainingTasks())
+ {
+ AbstractCompactionTask repairedTask = repaired.getNextBackgroundTask(gcBefore);
+ if (repairedTask != null)
+ return repairedTask;
+ return unrepaired.getNextBackgroundTask(gcBefore);
+ }
+ else
+ {
+ AbstractCompactionTask unrepairedTask = unrepaired.getNextBackgroundTask(gcBefore);
+ if (unrepairedTask != null)
+ return unrepairedTask;
+ return repaired.getNextBackgroundTask(gcBefore);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled && isActive;
+ }
+
+ public synchronized void resume()
+ {
+ isActive = true;
+ }
+
+ /**
+ * pause compaction while we cancel all ongoing compactions
+ *
+ * Separate call from enable/disable to not have to save the enabled-state externally
+ */
+ public synchronized void pause()
+ {
+ isActive = false;
+ }
+
+
+ private void startup()
+ {
+ for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+ {
+ if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+ getCompactionStrategyFor(sstable).addSSTable(sstable);
+ }
+ repaired.startup();
+ unrepaired.startup();
+ }
+
+ /**
+ * return the compaction strategy for the given sstable
+ *
+ * returns differently based on the repaired status
+ * @param sstable
+ * @return
+ */
+ private AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
+ {
+ if (sstable.isRepaired())
+ return repaired;
+ else
+ return unrepaired;
+ }
+
+ public void shutdown()
+ {
+ isActive = false;
+ repaired.shutdown();
+ unrepaired.shutdown();
+ }
+
+ public synchronized void maybeReload(CFMetaData metadata)
+ {
+ // compare the old schema configuration to the new one, ignore any locally set changes.
+ if (metadata.params.compaction.equals(schemaCompactionParams))
+ return;
+ reload(metadata);
+ }
+
+ /**
+ * Reload the compaction strategies
+ *
+ * Called after changing configuration and at startup.
+ * @param metadata
+ */
+ public synchronized void reload(CFMetaData metadata)
+ {
+ boolean disabledWithJMX = !enabled && shouldBeEnabled();
+ setStrategy(metadata.params.compaction);
+ schemaCompactionParams = metadata.params.compaction;
+
+ if (disabledWithJMX || !shouldBeEnabled())
+ disable();
+ else
+ enable();
+ startup();
+ }
+
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ {
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
+ }
+
+ public int getUnleveledSSTables()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int count = 0;
+ count += ((LeveledCompactionStrategy)repaired).getLevelSize(0);
+ count += ((LeveledCompactionStrategy)unrepaired).getLevelSize(0);
+ return count;
+ }
+ return 0;
+ }
+
+ public synchronized int[] getSSTableCountPerLevel()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int [] res = new int[LeveledManifest.MAX_LEVEL_COUNT];
+ int[] repairedCountPerLevel = ((LeveledCompactionStrategy) repaired).getAllLevelSize();
+ res = sumArrays(res, repairedCountPerLevel);
+ int[] unrepairedCountPerLevel = ((LeveledCompactionStrategy) unrepaired).getAllLevelSize();
+ res = sumArrays(res, unrepairedCountPerLevel);
+ return res;
+ }
+ return null;
+ }
+
+ private static int[] sumArrays(int[] a, int[] b)
+ {
+ int[] res = new int[Math.max(a.length, b.length)];
+ for (int i = 0; i < res.length; i++)
+ {
+ if (i < a.length && i < b.length)
+ res[i] = a[i] + b[i];
+ else if (i < a.length)
+ res[i] = a[i];
+ else
+ res[i] = b[i];
+ }
+ return res;
+ }
+
+ public boolean shouldDefragment()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.shouldDefragment();
+ }
+
+ public Directories getDirectories()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.getDirectories();
+ }
+
+ public synchronized void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableAddedNotification)
+ {
+ SSTableAddedNotification flushedNotification = (SSTableAddedNotification) notification;
+ for (SSTableReader sstable : flushedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repaired.addSSTable(sstable);
+ else
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableListChangedNotification)
+ {
+ SSTableListChangedNotification listChangedNotification = (SSTableListChangedNotification) notification;
+ Set<SSTableReader> repairedRemoved = new HashSet<>();
+ Set<SSTableReader> repairedAdded = new HashSet<>();
+ Set<SSTableReader> unrepairedRemoved = new HashSet<>();
+ Set<SSTableReader> unrepairedAdded = new HashSet<>();
+
+ for (SSTableReader sstable : listChangedNotification.removed)
+ {
+ if (sstable.isRepaired())
+ repairedRemoved.add(sstable);
+ else
+ unrepairedRemoved.add(sstable);
+ }
+ for (SSTableReader sstable : listChangedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repairedAdded.add(sstable);
+ else
+ unrepairedAdded.add(sstable);
+ }
+ if (!repairedRemoved.isEmpty())
+ {
+ repaired.replaceSSTables(repairedRemoved, repairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : repairedAdded)
+ repaired.addSSTable(sstable);
+ }
+
+ if (!unrepairedRemoved.isEmpty())
+ {
+ unrepaired.replaceSSTables(unrepairedRemoved, unrepairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : unrepairedAdded)
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableRepairStatusChanged)
+ {
+ for (SSTableReader sstable : ((SSTableRepairStatusChanged) notification).sstable)
+ {
+ if (sstable.isRepaired())
+ {
+ unrepaired.removeSSTable(sstable);
+ repaired.addSSTable(sstable);
+ }
+ else
+ {
+ repaired.removeSSTable(sstable);
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ }
+ else if (notification instanceof SSTableDeletingNotification)
+ {
+ SSTableReader sstable = ((SSTableDeletingNotification)notification).deleting;
+ if (sstable.isRepaired())
+ repaired.removeSSTable(sstable);
+ else
+ unrepaired.removeSSTable(sstable);
+ }
+ }
+
+ public void enable()
+ {
+ if (repaired != null)
+ repaired.enable();
+ if (unrepaired != null)
+ unrepaired.enable();
+ // enable this last to make sure the strategies are ready to get calls.
+ enabled = true;
+ }
+
+ public void disable()
+ {
+ // disable this first avoid asking disabled strategies for compaction tasks
+ enabled = false;
+ if (repaired != null)
+ repaired.disable();
+ if (unrepaired != null)
+ unrepaired.disable();
+ }
+
+ /**
+ * Create ISSTableScanner from the given sstables
+ *
+ * Delegates the call to the compaction strategies to allow LCS to create a scanner
+ * @param sstables
+ * @param ranges
+ * @return
+ */
+ @SuppressWarnings("resource")
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ {
+ List<SSTableReader> repairedSSTables = new ArrayList<>();
+ List<SSTableReader> unrepairedSSTables = new ArrayList<>();
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.isRepaired())
+ repairedSSTables.add(sstable);
+ else
+ unrepairedSSTables.add(sstable);
+ }
+
+ Set<ISSTableScanner> scanners = new HashSet<>(sstables.size());
+ AbstractCompactionStrategy.ScannerList repairedScanners = repaired.getScanners(repairedSSTables, ranges);
+ AbstractCompactionStrategy.ScannerList unrepairedScanners = unrepaired.getScanners(unrepairedSSTables, ranges);
+ scanners.addAll(repairedScanners.scanners);
+ scanners.addAll(unrepairedScanners.scanners);
+ return new AbstractCompactionStrategy.ScannerList(new ArrayList<>(scanners));
+ }
+
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables)
+ {
+ return getScanners(sstables, null);
+ }
+
+ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+ {
+ return unrepaired.groupSSTablesForAntiCompaction(sstablesToGroup);
+ }
+
+ public long getMaxSSTableBytes()
+ {
+ return unrepaired.getMaxSSTableBytes();
+ }
+
+ public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, int gcBefore, long maxSSTableBytes)
+ {
+ return getCompactionStrategyFor(txn.originals().iterator().next()).getCompactionTask(txn, gcBefore, maxSSTableBytes);
+ }
+
+ public Collection<AbstractCompactionTask> getMaximalTasks(final int gcBefore, final boolean splitOutput)
+ {
+ // runWithCompactionsDisabled cancels active compactions and disables them, then we are able
+ // to make the repaired/unrepaired strategies mark their own sstables as compacting. Once the
+ // sstables are marked the compactions are re-enabled
+ return cfs.runWithCompactionsDisabled(new Callable<Collection<AbstractCompactionTask>>()
+ {
+ @Override
+ public Collection<AbstractCompactionTask> call() throws Exception
+ {
+ synchronized (CompactionStrategyManager.this)
+ {
+ Collection<AbstractCompactionTask> repairedTasks = repaired.getMaximalTask(gcBefore, splitOutput);
+ Collection<AbstractCompactionTask> unrepairedTasks = unrepaired.getMaximalTask(gcBefore, splitOutput);
+
+ if (repairedTasks == null && unrepairedTasks == null)
+ return null;
+
+ if (repairedTasks == null)
+ return unrepairedTasks;
+ if (unrepairedTasks == null)
+ return repairedTasks;
+
+ List<AbstractCompactionTask> tasks = new ArrayList<>();
+ tasks.addAll(repairedTasks);
+ tasks.addAll(unrepairedTasks);
+ return tasks;
+ }
+ }
+ }, false, false);
+ }
+
+ public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
+ {
+ return getCompactionStrategyFor(sstables.iterator().next()).getUserDefinedTask(sstables, gcBefore);
+ }
+
+ public int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ tasks += repaired.getEstimatedRemainingTasks();
+ tasks += unrepaired.getEstimatedRemainingTasks();
+
+ return tasks;
+ }
+
+ public boolean shouldBeEnabled()
+ {
+ return params.isEnabled();
+ }
+
+ public String getName()
+ {
+ return unrepaired.getName();
+ }
+
+ public List<AbstractCompactionStrategy> getStrategies()
+ {
+ return Arrays.asList(repaired, unrepaired);
+ }
+
+ public synchronized void setNewLocalCompactionStrategy(CompactionParams params)
+ {
+ logger.info("Switching local compaction strategy from {} to {}}", this.params, params);
+ setStrategy(params);
+ if (shouldBeEnabled())
+ enable();
+ else
+ disable();
+ startup();
+ }
+
+ private void setStrategy(CompactionParams params)
+ {
+ if (repaired != null)
+ repaired.shutdown();
+ if (unrepaired != null)
+ unrepaired.shutdown();
+ repaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ unrepaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ this.params = params;
+ }
+
+ public CompactionParams getCompactionParams()
+ {
+ return params;
+ }
+
+ public boolean onlyPurgeRepairedTombstones()
+ {
+ return Boolean.parseBoolean(params.options().get(AbstractCompactionStrategy.ONLY_PURGE_REPAIRED_TOMBSTONES));
+ }
+
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn)
+ {
+ if (repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
+ {
+ return unrepaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ else
+ {
+ return repaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index c09d49c,5d5701f..16090a1
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@@ -195,10 -199,12 +199,12 @@@ public class Tracke
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
- Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader, SSTableReader>emptyMap(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@@ -327,10 -314,10 +332,10 @@@
apply(View.markFlushing(memtable));
}
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
assert !isDummy();
- if (sstables == null || sstables.isEmpty())
- if (sstable == null)
++ if (sstables.isEmpty())
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
// if it's null, we don't care what state the cfstore is in, we just replace it and continue
@@@ -338,27 -325,58 +343,57 @@@
return;
}
- sstable.setupKeyCache();
+ sstables.forEach(SSTableReader::setupOnline);
// back up before creating a new Snapshot (which makes the new one eligible for compaction)
- maybeIncrementallyBackup(sstable);
+ maybeIncrementallyBackup(sstables);
- apply(View.replaceFlushed(memtable, sstable));
+ apply(View.replaceFlushed(memtable, sstables));
Throwable fail;
- fail = updateSizeTracking(emptySet(), singleton(sstable), null);
+ fail = updateSizeTracking(emptySet(), sstables, null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
- public void permitCompactionOfFlushed(SSTableReader sstable)
++ public void permitCompactionOfFlushed(Collection<SSTableReader> sstables)
+ {
- if (sstable == null)
++ if (sstables.isEmpty())
+ return;
+
- apply(View.permitCompactionOfFlushed(sstable));
++ apply(View.permitCompactionOfFlushed(sstables));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
- notifyAdded(sstable);
++ notifyAdded(sstables);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
-
// MISCELLANEOUS public utility calls
+ public Set<SSTableReader> getSSTables()
+ {
+ return view.get().sstables;
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
[08/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
Fix commit log replay after out-of-order flush completion.
Patch by Benedict Elliott Smith; reviewed by Branimir Lambov for
CASSANDRA-9669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/849a4386
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/849a4386
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/849a4386
Branch: refs/heads/trunk
Commit: 849a438690aa97a361227781108cc90355dcbcd9
Parents: e614433
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Sep 9 15:25:59 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:17:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/db/ColumnFamilyStore.java | 258 +++++++++++--------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 218 +++++++---------
.../db/commitlog/CommitLogReplayer.java | 58 +++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++---
.../compaction/AbstractCompactionStrategy.java | 3 -
.../cassandra/db/compaction/Scrubber.java | 2 -
.../apache/cassandra/db/lifecycle/Tracker.java | 47 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 38 ++-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 10 +-
.../io/sstable/format/big/BigTableWriter.java | 2 +-
.../io/sstable/metadata/CompactionMetadata.java | 4 +-
.../metadata/IMetadataComponentSerializer.java | 4 +-
.../sstable/metadata/IMetadataSerializer.java | 3 +-
.../metadata/LegacyMetadataSerializer.java | 15 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/MetadataSerializer.java | 9 +-
.../io/sstable/metadata/StatsMetadata.java | 41 ++-
.../io/sstable/metadata/ValidationMetadata.java | 4 +-
.../cassandra/io/util/DiskAwareRunnable.java | 42 ---
.../cassandra/streaming/StreamReceiveTask.java | 3 -
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 19 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 6 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 74 +++++-
.../cassandra/utils/IntervalTreeTest.java | 2 +-
33 files changed, 600 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2bb3518..dfcad10 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,5 @@
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+
2.2.7
* Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 513a138..88e22c0 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -146,6 +146,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ @VisibleForTesting
+ public static volatile ColumnFamilyStore discardFlushResults;
+
public final Keyspace keyspace;
public final String name;
public final CFMetaData metadata;
@@ -867,14 +870,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param memtable
*/
- public Future<?> switchMemtableIfCurrent(Memtable memtable)
+ public ListenableFuture<ReplayPosition> switchMemtableIfCurrent(Memtable memtable)
{
synchronized (data)
{
if (data.getView().getCurrentMemtable() == memtable)
return switchMemtable();
}
- return Futures.immediateFuture(null);
+ return waitForFlushes();
}
/*
@@ -884,14 +887,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL
* marked clean up to the position owned by the Memtable.
*/
- public ListenableFuture<?> switchMemtable()
+ public ListenableFuture<ReplayPosition> switchMemtable()
{
synchronized (data)
{
logFlush();
Flush flush = new Flush(false);
flushExecutor.execute(flush);
- ListenableFutureTask<?> task = ListenableFutureTask.create(flush.postFlush, null);
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(flush.postFlush);
postFlushExecutor.submit(task);
return task;
}
@@ -926,77 +929,93 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
- public ListenableFuture<?> forceFlush()
+ /**
+ * Flush if there is unflushed data in the memtables
+ *
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ public ListenableFuture<ReplayPosition> forceFlush()
{
- return forceFlush(null);
+ Memtable current = data.getView().getCurrentMemtable();
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ if (!cfs.data.getView().getCurrentMemtable().isClean())
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
}
/**
* Flush if there is unflushed data that was written to the CommitLog before @param flushIfDirtyBefore
- * (inclusive). If @param flushIfDirtyBefore is null, flush if there is any unflushed data.
+ * (inclusive).
*
- * @return a Future such that when the future completes, all data inserted before forceFlush was called,
- * will be flushed.
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
*/
- public ListenableFuture<?> forceFlush(ReplayPosition flushIfDirtyBefore)
+ public ListenableFuture<ReplayPosition> forceFlush(ReplayPosition flushIfDirtyBefore)
{
- // we synchronize on the data tracker to ensure we don't race against other calls to switchMemtable(),
- // unnecessarily queueing memtables that are about to be made clean
- synchronized (data)
- {
- // during index build, 2ary index memtables can be dirty even if parent is not. if so,
- // we want to flush the 2ary index ones too.
- boolean clean = true;
- for (ColumnFamilyStore cfs : concatWithIndexes())
- clean &= cfs.data.getView().getCurrentMemtable().isCleanAfter(flushIfDirtyBefore);
+ // we don't loop through the remaining memtables since here we only care about commit log dirtiness
+ // and this does not vary between a table and its table-backed indexes
+ Memtable current = data.getView().getCurrentMemtable();
+ if (current.mayContainDataBefore(flushIfDirtyBefore))
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
+ }
- if (clean)
+ /**
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ private ListenableFuture<ReplayPosition> waitForFlushes()
+ {
+ // we grab the current memtable; once any preceding memtables have flushed, we know its
+ // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable)
+ final Memtable current = data.getView().getCurrentMemtable();
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(new Callable<ReplayPosition>()
+ {
+ public ReplayPosition call()
{
- // We could have a memtable for this column family that is being
- // flushed. Make sure the future returned wait for that so callers can
- // assume that any data inserted prior to the call are fully flushed
- // when the future returns (see #5241).
- ListenableFutureTask<?> task = ListenableFutureTask.create(new Runnable()
- {
- public void run()
- {
- logger.trace("forceFlush requested but everything is clean in {}", name);
- }
- }, null);
- postFlushExecutor.execute(task);
- return task;
+ logger.debug("forceFlush requested but everything is clean in {}", name);
+ return current.getCommitLogLowerBound();
}
-
- return switchMemtable();
- }
+ });
+ postFlushExecutor.execute(task);
+ return task;
}
- public void forceBlockingFlush()
+ public ReplayPosition forceBlockingFlush()
{
- FBUtilities.waitOnFuture(forceFlush());
+ return FBUtilities.waitOnFuture(forceFlush());
}
/**
* Both synchronises custom secondary indexes and provides ordering guarantees for futures on switchMemtable/flush
* etc, which expect to be able to wait until the flush (and all prior flushes) requested have completed.
*/
- private final class PostFlush implements Runnable
+ private final class PostFlush implements Callable<ReplayPosition>
{
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<SSTableReader> readers;
volatile FSWriteError flushFailure = null;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<SSTableReader> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
+ this.memtables = memtables;
+ this.readers = readers;
}
- public void run()
+ public ReplayPosition call()
{
+ if (discardFlushResults == ColumnFamilyStore.this)
+ return commitLogUpperBound;
+
writeBarrier.await();
/**
@@ -1018,7 +1037,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
try
{
- // we wait on the latch for the lastReplayPosition to be set, and so that waiters
+ // we wait on the latch for the commitLogUpperBound to be set, and so that waiters
// on this task can rely on all prior flushes being complete
latch.await();
}
@@ -1027,18 +1046,23 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
+ SSTableReader reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
+ }
}
-
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
+ return commitLogUpperBound;
}
}
@@ -1053,7 +1077,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
+ final List<SSTableReader> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@ -1069,43 +1094,33 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* that all write operations register themselves with, and assigning this barrier to the memtables,
* after which we *.issue()* the barrier. This barrier is used to direct write operations started prior
* to the barrier.issue() into the memtable we have switched out, and any started after to its replacement.
- * In doing so it also tells the write operations to update the lastReplayPosition of the memtable, so
+ * In doing so it also tells the write operations to update the commitLogUpperBound of the memtable, so
* that we know the CL position we are dirty to, which can be marked clean when we complete.
*/
writeBarrier = keyspace.writeOrder.newBarrier();
- memtables = new ArrayList<>();
// submit flushes for the memtable for any indexed sub-cfses, and our own
- AtomicReference<ReplayPosition> lastReplayPositionHolder = new AtomicReference<>();
+ AtomicReference<ReplayPosition> commitLogUpperBound = new AtomicReference<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
// switch all memtables, regardless of their dirty status, setting the barrier
// so that we can reach a coordinated decision about cleanliness once they
// are no longer possible to be modified
- Memtable mt = cfs.data.switchMemtable(truncate);
- mt.setDiscarding(writeBarrier, lastReplayPositionHolder);
- memtables.add(mt);
+ Memtable newMemtable = new Memtable(commitLogUpperBound, cfs);
+ Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable);
+ oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound);
+ memtables.add(oldMemtable);
}
- // we now attempt to define the lastReplayPosition; we do this by grabbing the current limit from the CL
- // and attempting to set the holder to this value. at the same time all writes to the memtables are
- // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
- // so that we know all operations prior to the position have not reached it yet
- ReplayPosition lastReplayPosition;
- while (true)
- {
- lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
- ReplayPosition currentLast = lastReplayPositionHolder.get();
- if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
- && lastReplayPositionHolder.compareAndSet(currentLast, lastReplayPosition))
- break;
- }
+ // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log
+ // records owned by this memtable
+ setCommitLogUpperBound(commitLogUpperBound);
// we then issue the barrier; this lets us wait for all operations started prior to the barrier to complete;
- // since this happens after wiring up the lastReplayPosition, we also know all operations with earlier
+ // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier
// replay positions have also completed, i.e. the memtables are done and ready to flush
writeBarrier.issue();
- postFlush = new PostFlush(!truncate, writeBarrier, lastReplayPosition);
+ postFlush = new PostFlush(!truncate, writeBarrier, commitLogUpperBound.get(), memtables, readers);
}
public void run()
@@ -1124,7 +1139,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, null);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
reclaim(memtable);
iter.remove();
}
@@ -1143,8 +1159,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (Memtable memtable : memtables)
{
// flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+ SSTableReader reader = memtable.flush();
+ memtable.cfs.data.replaceFlushed(memtable, reader);
reclaim(memtable);
+ readers.add(reader);
}
}
catch (FSWriteError e)
@@ -1174,6 +1192,38 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ // atomically set the upper bound for the commit log
+ private static void setCommitLogUpperBound(AtomicReference<ReplayPosition> commitLogUpperBound)
+ {
+ // we attempt to set the holder to the current commit log context. at the same time all writes to the memtables are
+ // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
+ // so that we know all operations prior to the position have not reached it yet
+ ReplayPosition lastReplayPosition;
+ while (true)
+ {
+ lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
+ ReplayPosition currentLast = commitLogUpperBound.get();
+ if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
+ && commitLogUpperBound.compareAndSet(currentLast, lastReplayPosition))
+ break;
+ }
+ }
+
+ @VisibleForTesting
+ // this method should ONLY be used for testing commit log behaviour; it discards the current memtable
+ // contents without marking the commit log clean, and prevents any proceeding flushes from marking
+ // the commit log as done, however they *will* terminate (unlike under typical failures) to ensure progress is made
+ public void simulateFailedFlush()
+ {
+ discardFlushResults = this;
+ data.markFlushing(data.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), this)));
+ }
+
+ public void resumeFlushing()
+ {
+ discardFlushResults = null;
+ }
+
/**
* Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
* queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
@@ -1589,11 +1639,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
maybeFail(data.dropSSTables(Predicates.in(sstables), compactionType, null));
}
- void replaceFlushed(Memtable memtable, SSTableReader sstable)
- {
- compactionStrategyWrapper.replaceFlushed(memtable, sstable);
- }
-
public boolean isValid()
{
return valid;
@@ -1615,6 +1660,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return data.getSSTables();
}
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
public Set<SSTableReader> getUncompactingSSTables()
{
return data.getUncompacting();
@@ -2670,6 +2720,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Void call()
{
cfs.data.reset();
+ cfs.getCompactionStrategy().shutdown();
+ cfs.getCompactionStrategy().startup();
return null;
}
}, true);
@@ -2707,39 +2759,42 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- // just nuke the memtable data w/o writing to disk first
- synchronized (data)
+ if (keyspace.getMetadata().durableWrites || takeSnapshot)
{
+ replayAfter = forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
+ replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
}
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
if (takeSnapshot)
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
for (SecondaryIndex index : indexManager.getIndexes())
index.truncateBlocking(truncatedAt);
@@ -2807,7 +2862,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public LifecycleTransaction call() throws Exception
{
assert data.getCompacting().isEmpty() : data.getCompacting();
- Collection<SSTableReader> sstables = Lists.newArrayList(AbstractCompactionStrategy.filterSuspectSSTables(getSSTables()));
+ Iterable<SSTableReader> sstables = getPermittedToCompactSSTables();
+ sstables = AbstractCompactionStrategy.filterSuspectSSTables(sstables);
+ sstables = ImmutableList.copyOf(sstables);
LifecycleTransaction modifier = data.tryModify(sstables, operationType);
assert modifier != null: "something marked things compacting while compactions are disabled";
return modifier;
@@ -3025,10 +3082,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param truncatedAt The timestamp of the truncation
* (all SSTables before that timestamp are going be marked as compacted)
- *
- * @return the most recent replay position of the truncated data
*/
- public ReplayPosition discardSSTables(long truncatedAt)
+ public void discardSSTables(long truncatedAt)
{
assert data.getCompacting().isEmpty() : data.getCompacting();
@@ -3040,11 +3095,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
truncatedSSTables.add(sstable);
}
- if (truncatedSSTables.isEmpty())
- return ReplayPosition.NONE;
-
- markObsolete(truncatedSSTables, OperationType.UNKNOWN);
- return ReplayPosition.getReplayPosition(truncatedSSTables);
+ if (!truncatedSSTables.isEmpty())
+ markObsolete(truncatedSSTables, OperationType.UNKNOWN);
}
public double getDroppableTombstoneRatio()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index c2901a4..2b3662f 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -346,7 +346,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- return null;
+ throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index fb4da72..b4ada09 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -29,21 +29,20 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -62,14 +61,17 @@ public class Memtable implements Comparable<Memtable>
// the write barrier for directing writes to this memtable during a switch
private volatile OpOrder.Barrier writeBarrier;
- // the last ReplayPosition owned by this Memtable; all ReplayPositions lower are owned by this or an earlier Memtable
- private volatile AtomicReference<ReplayPosition> lastReplayPosition;
- // the "first" ReplayPosition owned by this Memtable; this is inaccurate, and only used as a convenience to prevent CLSM flushing wantonly
- private final ReplayPosition minReplayPosition = CommitLog.instance.getContext();
+ // the precise upper bound of ReplayPosition owned by this memtable
+ private volatile AtomicReference<ReplayPosition> commitLogUpperBound;
+ // the precise lower bound of ReplayPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
+ private AtomicReference<ReplayPosition> commitLogLowerBound;
+ // the approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
+ // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
+ private final ReplayPosition approximateCommitLogLowerBound = CommitLog.instance.getContext();
public int compareTo(Memtable that)
{
- return this.minReplayPosition.compareTo(that.minReplayPosition);
+ return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound);
}
public static final class LastReplayPosition extends ReplayPosition
@@ -84,7 +86,6 @@ public class Memtable implements Comparable<Memtable>
// actually only store DecoratedKey.
private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@ -95,9 +96,10 @@ public class Memtable implements Comparable<Memtable>
// memtable was created with the new or old comparator.
public final CellNameType initialComparator;
- public Memtable(ColumnFamilyStore cfs)
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@ -131,7 +133,7 @@ public class Memtable implements Comparable<Memtable>
public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference<ReplayPosition> lastReplayPosition)
{
assert this.writeBarrier == null;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = lastReplayPosition;
this.writeBarrier = writeBarrier;
allocator.setDiscarding();
}
@@ -161,16 +163,21 @@ public class Memtable implements Comparable<Memtable>
// its current value and ours; if it HAS been finalised, we simply accept its judgement
// this permits us to coordinate a safe boundary, as the boundary choice is made
// atomically wrt our max() maintenance, so an operation cannot sneak into the past
- ReplayPosition currentLast = lastReplayPosition.get();
+ ReplayPosition currentLast = commitLogUpperBound.get();
if (currentLast instanceof LastReplayPosition)
return currentLast.compareTo(replayPosition) >= 0;
if (currentLast != null && currentLast.compareTo(replayPosition) >= 0)
return true;
- if (lastReplayPosition.compareAndSet(currentLast, replayPosition))
+ if (commitLogUpperBound.compareAndSet(currentLast, replayPosition))
return true;
}
}
+ public ReplayPosition getCommitLogLowerBound()
+ {
+ return commitLogLowerBound.get();
+ }
+
public boolean isLive()
{
return allocator.isLive();
@@ -181,9 +188,9 @@ public class Memtable implements Comparable<Memtable>
return rows.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@ -252,11 +259,6 @@ public class Memtable implements Comparable<Memtable>
return rows.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@ -285,20 +287,21 @@ public class Memtable implements Comparable<Memtable>
public Map.Entry<DecoratedKey, ColumnFamily> next()
{
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entry = iter.next();
+ Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
// Actual stored key should be true DecoratedKey
- assert entry.getKey() instanceof DecoratedKey;
+ assert entryRowPosition.getKey() instanceof DecoratedKey;
+ @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
+ Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
if (MEMORY_POOL.needToCopyOnHeap())
{
- DecoratedKey key = (DecoratedKey) entry.getKey();
+ DecoratedKey key = entry.getKey();
key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
}
// Store the reference to the current entry so that remove() can update the current size.
currentEntry = entry;
- // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- return (Map.Entry<DecoratedKey, ColumnFamily>) entry;
+ return entry;
}
public void remove()
@@ -315,9 +318,13 @@ public class Memtable implements Comparable<Memtable>
return rows.get(key);
}
- public long creationTime()
+ public SSTableReader flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
+ Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
+ File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@ -325,115 +332,88 @@ public class Memtable implements Comparable<Memtable>
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- FlushRunnable(ReplayPosition context)
- {
- this.context = context;
-
- long keySize = 0;
- for (RowPosition key : rows.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
- }
-
- protected void runMayThrow() throws Exception
+ long keySize = 0;
+ for (RowPosition key : rows.keySet())
{
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- SSTableReader sstable = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstable);
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected Directories getDirectories()
- {
- return cfs.directories;
- }
+ private SSTableReader writeSortedContents(File sstableDirectory)
+ {
+ logger.info("Writing {}", Memtable.this.toString());
- private SSTableReader writeSortedContents(ReplayPosition context, File sstableDirectory)
+ SSTableReader ssTable;
+ // errors when creating the writer that may leave empty temp files.
+ try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
+ for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
+ AtomicBTreeColumns cf = entry.getValue();
+
+ if (cf.isMarkedForDelete() && cf.hasColumns())
{
- AtomicBTreeColumns cf = entry.getValue();
-
- if (cf.isMarkedForDelete() && cf.hasColumns())
- {
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
- continue;
- }
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ // When every node is up, there's no reason to write batchlog data out to sstables
+ // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
+ // and BL data is strictly local, so we don't need to preserve tombstones for repair.
+ // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
+ // See CASSANDRA-4667.
+ if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
+ continue;
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
- context));
+ if (trackContention && cf.usePessimisticLocking())
+ heavilyContendedRowCount++;
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTable = null;
- }
+ if (!cf.isEmpty())
+ writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
+ commitLogUpperBound));
- return ssTable;
+ // temp sstables should contain non-repaired data.
+ ssTable = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
+ ssTable = null;
}
- }
- public SSTableWriter createFlushWriter(String filename)
- {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
+ if (heavilyContendedRowCount > 0)
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ return ssTable;
}
}
+ private SSTableWriter createFlushWriter(String filename)
+ {
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
+ .commitLogLowerBound(commitLogLowerBound.get())
+ .commitLogUpperBound(commitLogUpperBound.get());
+ return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ }
+
private static int estimateRowOverhead(final int count)
{
// calculate row overhead
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 98fb556..a58aeb4 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -33,7 +33,6 @@ import com.google.common.base.Predicate;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +69,7 @@ public class CommitLogReplayer
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final ICRC32 checksum;
private byte[] buffer;
@@ -79,7 +78,7 @@ public class CommitLogReplayer
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
this.futures = new ArrayList<Future<?>>();
@@ -89,7 +88,7 @@ public class CommitLogReplayer
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
this.checksum = CRC32Factory.instance.create();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@ -98,17 +97,12 @@ public class CommitLogReplayer
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
-
- // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@ -125,19 +119,21 @@ public class CommitLogReplayer
cfs.metadata.ksName,
cfs.metadata.cfName);
SystemKeyspace.removeTruncationRecord(cfs.metadata.cfId);
+ truncatedAt = null;
}
}
- else
- {
- rp = replayPositionOrdering.max(Arrays.asList(rp, truncatedAt));
- }
}
- cfPositions.put(cfs.metadata.cfId, rp);
+ ReplayPosition.ReplayFilter filter = new ReplayPosition.ReplayFilter(cfs.getSSTables(), truncatedAt);
+ if (!filter.isEmpty())
+ cfPersisted.put(cfs.metadata.cfId, filter);
+ else
+ globalPosition = ReplayPosition.NONE; // if we have no ranges for this CF, we must replay everything and filter
}
- ReplayPosition globalPosition = replayPositionOrdering.min(cfPositions.values());
- logger.trace("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPositions));
- return new CommitLogReplayer(commitLog, globalPosition, cfPositions, replayFilter);
+ if (globalPosition == null)
+ globalPosition = ReplayPosition.firstNotCovered(cfPersisted.values());
+ logger.debug("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPersisted));
+ return new CommitLogReplayer(commitLog, globalPosition, cfPersisted, replayFilter);
}
public void recover(File[] clogs) throws IOException
@@ -273,6 +269,18 @@ public class CommitLogReplayer
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
@SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
@@ -495,7 +503,7 @@ public class CommitLogReplayer
mutationStart, errorContext);
continue;
}
- replayMutation(buffer, serializedSize, reader.getFilePointer(), desc);
+ replayMutation(buffer, serializedSize, (int) reader.getFilePointer(), desc);
}
return true;
}
@@ -504,7 +512,7 @@ public class CommitLogReplayer
* Deserializes and replays a commit log entry.
*/
void replayMutation(byte[] inputBuffer, int size,
- final long entryLocation, final CommitLogDescriptor desc) throws IOException
+ final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
final Mutation mutation;
@@ -577,11 +585,7 @@ public class CommitLogReplayer
if (Schema.instance.getCF(columnFamily.id()) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(columnFamily.id());
-
- // replay if current segment is newer than last flushed one or,
- // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
+ if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index ca1969f..17802ad 100644
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@ -19,10 +19,10 @@ package org.apache.cassandra.db.commitlog;
import java.io.DataInput;
import java.io.IOException;
-import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
@@ -35,46 +35,78 @@ public class ReplayPosition implements Comparable<ReplayPosition>
public static final ReplayPositionSerializer serializer = new ReplayPositionSerializer();
// NONE is used for SSTables that are streamed from other nodes and thus have no relationship
- // with our local commitlog. The values satisfy the critera that
+ // with our local commitlog. The values satisfy the criteria that
// - no real commitlog segment will have the given id
// - it will sort before any real replayposition, so it will be effectively ignored by getReplayPosition
public static final ReplayPosition NONE = new ReplayPosition(-1, 0);
+ public final long segment;
+ public final int position;
+
/**
- * Convenience method to compute the replay position for a group of SSTables.
- * @param sstables
- * @return the most recent (highest) replay position
+ * A filter of known safe-to-discard commit log replay positions, based on
+ * the range covered by on disk sstables and those prior to the most recent truncation record
*/
- public static ReplayPosition getReplayPosition(Iterable<? extends SSTableReader> sstables)
+ public static class ReplayFilter
{
- if (Iterables.isEmpty(sstables))
- return NONE;
-
- Function<SSTableReader, ReplayPosition> f = new Function<SSTableReader, ReplayPosition>()
+ final NavigableMap<ReplayPosition, ReplayPosition> persisted = new TreeMap<>();
+ public ReplayFilter(Iterable<SSTableReader> onDisk, ReplayPosition truncatedAt)
{
- public ReplayPosition apply(SSTableReader sstable)
+ for (SSTableReader reader : onDisk)
{
- return sstable.getReplayPosition();
+ ReplayPosition start = reader.getSSTableMetadata().commitLogLowerBound;
+ ReplayPosition end = reader.getSSTableMetadata().commitLogUpperBound;
+ add(persisted, start, end);
}
- };
- Ordering<ReplayPosition> ordering = Ordering.from(ReplayPosition.comparator);
- return ordering.max(Iterables.transform(sstables, f));
- }
+ if (truncatedAt != null)
+ add(persisted, ReplayPosition.NONE, truncatedAt);
+ }
+ private static void add(NavigableMap<ReplayPosition, ReplayPosition> ranges, ReplayPosition start, ReplayPosition end)
+ {
+ // extend ourselves to cover any ranges we overlap
+ // record directly preceding our end may extend past us, so take the max of our end and its
+ Map.Entry<ReplayPosition, ReplayPosition> extend = ranges.floorEntry(end);
+ if (extend != null && extend.getValue().compareTo(end) > 0)
+ end = extend.getValue();
+
+ // record directly preceding our start may extend into us; if it does, we take it as our start
+ extend = ranges.lowerEntry(start);
+ if (extend != null && extend.getValue().compareTo(start) >= 0)
+ start = extend.getKey();
+
+ ranges.subMap(start, end).clear();
+ ranges.put(start, end);
+ }
- public final long segment;
- public final int position;
+ public boolean shouldReplay(ReplayPosition position)
+ {
+ // replay ranges are start exclusive, end inclusive
+ Map.Entry<ReplayPosition, ReplayPosition> range = persisted.lowerEntry(position);
+ return range == null || position.compareTo(range.getValue()) > 0;
+ }
- public static final Comparator<ReplayPosition> comparator = new Comparator<ReplayPosition>()
- {
- public int compare(ReplayPosition o1, ReplayPosition o2)
+ public boolean isEmpty()
{
- if (o1.segment != o2.segment)
- return Long.valueOf(o1.segment).compareTo(o2.segment);
+ return persisted.isEmpty();
+ }
+ }
- return Integer.valueOf(o1.position).compareTo(o2.position);
+ public static ReplayPosition firstNotCovered(Iterable<ReplayFilter> ranges)
+ {
+ ReplayPosition min = null;
+ for (ReplayFilter map : ranges)
+ {
+ ReplayPosition first = map.persisted.firstEntry().getValue();
+ if (min == null)
+ min = first;
+ else
+ min = Ordering.natural().min(min, first);
}
- };
+ if (min == null)
+ return NONE;
+ return min;
+ }
public ReplayPosition(long segment, int position)
{
@@ -83,9 +115,12 @@ public class ReplayPosition implements Comparable<ReplayPosition>
this.position = position;
}
- public int compareTo(ReplayPosition other)
+ public int compareTo(ReplayPosition that)
{
- return comparator.compare(this, other);
+ if (this.segment != that.segment)
+ return Long.compare(this.segment, that.segment);
+
+ return Integer.compare(this.position, that.position);
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 02497df..575aa51 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -232,9 +232,6 @@ public abstract class AbstractCompactionStrategy
*/
public void replaceFlushed(Memtable memtable, SSTableReader sstable)
{
- cfs.getTracker().replaceFlushed(memtable, sstable);
- if (sstable != null)
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e9137e2..d1fe702 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -98,8 +98,6 @@ public class Scrubber implements Closeable
// Calculate the expected compacted filesize
this.destination = cfs.directories.getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(toScrub, OperationType.SCRUB));
- if (destination == null)
- throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
this.controller = transaction.isOffline()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index a074216..5d5701f 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -31,11 +31,13 @@ import com.google.common.collect.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -45,6 +47,8 @@ import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.OpOrder;
import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.in;
+import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.Iterables.filter;
import static java.util.Collections.singleton;
@@ -195,10 +199,12 @@ public class Tracker
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@ -294,9 +300,8 @@ public class Tracker
*
* @return the previously active memtable
*/
- public Memtable switchMemtable(boolean truncating)
+ public Memtable switchMemtable(boolean truncating, Memtable newMemtable)
{
- Memtable newMemtable = new Memtable(cfstore);
Pair<View, View> result = apply(View.switchMemtable(newMemtable));
if (truncating)
notifyRenewed(newMemtable);
@@ -328,15 +333,35 @@ public class Tracker
Throwable fail;
fail = updateSizeTracking(emptySet(), singleton(sstable), null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstable, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
+ public void permitCompactionOfFlushed(SSTableReader sstable)
+ {
+ if (sstable == null)
+ return;
+
+ apply(View.permitCompactionOfFlushed(sstable));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
+ notifyAdded(sstable);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
+
// MISCELLANEOUS public utility calls
@@ -346,6 +371,12 @@ public class Tracker
return view.get().sstables;
}
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index 73ba131..fba1627 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.db.lifecycle;
import java.util.*;
+import javax.annotation.Nullable;
+
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
@@ -66,6 +68,7 @@ public class View
public final List<Memtable> flushingMemtables;
public final Set<SSTableReader> compacting;
public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
@@ -74,7 +77,7 @@ public class View
public final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, SSTableIntervalTree intervalTree)
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@ -88,6 +91,7 @@ public class View
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
this.compacting = compacting;
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@ -155,7 +159,7 @@ public class View
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
replace(view.compacting, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@ -169,7 +173,7 @@ public class View
public boolean apply(View view)
{
for (SSTableReader reader : readers)
- if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted())
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
return false;
return true;
}
@@ -186,7 +190,7 @@ public class View
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting,
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@ -201,7 +205,7 @@ public class View
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -220,7 +224,7 @@ public class View
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -237,15 +241,33 @@ public class View
if (flushed == null)
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compacting, view.intervalTree);
+ view.compacting, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compacting,
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e81e4e9..c303975 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1864,11 +1864,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstableMetadata.compressionRatio;
}
- public ReplayPosition getReplayPosition()
- {
- return sstableMetadata.replayPosition;
- }
-
public long getMinTimestamp()
{
return sstableMetadata.minTimestamp;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index faaa89e..41a83e1 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -52,6 +52,8 @@ public abstract class Version
public abstract boolean hasNewFileName();
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index a1e32cf..9244bbb 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -125,7 +125,7 @@ public class BigFormat implements SSTableFormat
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "la";
+ public static final String current_version = "lb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@ -135,6 +135,7 @@ public class BigFormat implements SSTableFormat
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@ -143,6 +144,7 @@ public class BigFormat implements SSTableFormat
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ private final boolean hasCommitLogLowerBound;
public BigVersion(String version)
{
@@ -155,6 +157,7 @@ public class BigFormat implements SSTableFormat
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("la") >= 0;
+ hasCommitLogLowerBound = version.compareTo("lb") >= 0;
}
@Override
@@ -199,6 +202,11 @@ public class BigFormat implements SSTableFormat
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
public boolean isCompatible()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 505bac0..3a01f87 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -432,7 +432,7 @@ public class BigTableWriter extends SSTableWriter
File file = new File(desc.filenameFor(Component.STATS));
try (SequentialWriter out = SequentialWriter.open(file);)
{
- desc.getMetadataSerializer().serialize(components, out.stream);
+ desc.getMetadataSerializer().serialize(components, desc.version, out.stream);
out.setDescriptor(desc).finish();
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index ed1f327..c8e6ee8 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -72,7 +72,7 @@ public class CompactionMetadata extends MetadataComponent
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component) throws IOException
+ public int serializedSize(CompactionMetadata component, Version version) throws IOException
{
int size = 0;
size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
@@ -83,7 +83,7 @@ public class CompactionMetadata extends MetadataComponent
return size;
}
- public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeInt(component.ancestors.size());
for (int g : component.ancestors)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
index dc8fbdf..e3d867f 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
@@ -35,7 +35,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @return serialized size of this component
* @throws IOException
*/
- int serializedSize(T component) throws IOException;
+ int serializedSize(T component, Version version) throws IOException;
/**
* Serialize metadata component to given output.
@@ -45,7 +45,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @param out serialize destination
* @throws IOException
*/
- void serialize(T component, DataOutputPlus out) throws IOException;
+ void serialize(T component, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize metadata component from given input.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index df577df..a7d23f4 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -22,6 +22,7 @@ import java.util.EnumSet;
import java.util.Map;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -37,7 +38,7 @@ public interface IMetadataSerializer
* @param out
* @throws IOException
*/
- void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException;
+ void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize specified metadata components from given descriptor.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 4bd060e..bfeb930 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Maps;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -42,7 +43,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
* Legacy serialization is only used for SSTable level reset.
*/
@Override
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
@@ -52,7 +53,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
@@ -70,6 +71,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
out.writeInt(stats.maxColumnNames.size());
for (ByteBuffer columnName : stats.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@ -91,7 +94,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -116,6 +120,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
@@ -124,7 +130,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
components.put(MetadataType.STATS,
new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 5962a46..579ff7a 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -28,6 +28,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -69,6 +70,7 @@ public class MetadataCollector
return new StatsMetadata(defaultRowSizeHistogram(),
defaultColumnCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@ -83,7 +85,8 @@ public class MetadataCollector
protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
protected long minTimestamp = Long.MAX_VALUE;
protected long maxTimestamp = Long.MIN_VALUE;
protected int maxLocalDeletionTime = Integer.MIN_VALUE;
@@ -113,7 +116,23 @@ public class MetadataCollector
{
this(columnNameComparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
// Get the max timestamp of the precompacted sstables
// and adds generation of live ancestors
@@ -199,9 +218,15 @@ public class MetadataCollector
return this;
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@ -257,7 +282,8 @@ public class MetadataCollector
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
[04/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
Fix commit log replay after out-of-order flush completion.
Patch by Benedict Elliott Smith; reviewed by Branimir Lambov for
CASSANDRA-9669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/849a4386
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/849a4386
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/849a4386
Branch: refs/heads/cassandra-3.0
Commit: 849a438690aa97a361227781108cc90355dcbcd9
Parents: e614433
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Sep 9 15:25:59 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:17:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/db/ColumnFamilyStore.java | 258 +++++++++++--------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 218 +++++++---------
.../db/commitlog/CommitLogReplayer.java | 58 +++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++---
.../compaction/AbstractCompactionStrategy.java | 3 -
.../cassandra/db/compaction/Scrubber.java | 2 -
.../apache/cassandra/db/lifecycle/Tracker.java | 47 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 38 ++-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 10 +-
.../io/sstable/format/big/BigTableWriter.java | 2 +-
.../io/sstable/metadata/CompactionMetadata.java | 4 +-
.../metadata/IMetadataComponentSerializer.java | 4 +-
.../sstable/metadata/IMetadataSerializer.java | 3 +-
.../metadata/LegacyMetadataSerializer.java | 15 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/MetadataSerializer.java | 9 +-
.../io/sstable/metadata/StatsMetadata.java | 41 ++-
.../io/sstable/metadata/ValidationMetadata.java | 4 +-
.../cassandra/io/util/DiskAwareRunnable.java | 42 ---
.../cassandra/streaming/StreamReceiveTask.java | 3 -
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 19 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 6 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 74 +++++-
.../cassandra/utils/IntervalTreeTest.java | 2 +-
33 files changed, 600 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2bb3518..dfcad10 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,5 @@
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+
2.2.7
* Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 513a138..88e22c0 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -146,6 +146,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ @VisibleForTesting
+ public static volatile ColumnFamilyStore discardFlushResults;
+
public final Keyspace keyspace;
public final String name;
public final CFMetaData metadata;
@@ -867,14 +870,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param memtable
*/
- public Future<?> switchMemtableIfCurrent(Memtable memtable)
+ public ListenableFuture<ReplayPosition> switchMemtableIfCurrent(Memtable memtable)
{
synchronized (data)
{
if (data.getView().getCurrentMemtable() == memtable)
return switchMemtable();
}
- return Futures.immediateFuture(null);
+ return waitForFlushes();
}
/*
@@ -884,14 +887,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL
* marked clean up to the position owned by the Memtable.
*/
- public ListenableFuture<?> switchMemtable()
+ public ListenableFuture<ReplayPosition> switchMemtable()
{
synchronized (data)
{
logFlush();
Flush flush = new Flush(false);
flushExecutor.execute(flush);
- ListenableFutureTask<?> task = ListenableFutureTask.create(flush.postFlush, null);
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(flush.postFlush);
postFlushExecutor.submit(task);
return task;
}
@@ -926,77 +929,93 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
- public ListenableFuture<?> forceFlush()
+ /**
+ * Flush if there is unflushed data in the memtables
+ *
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ public ListenableFuture<ReplayPosition> forceFlush()
{
- return forceFlush(null);
+ Memtable current = data.getView().getCurrentMemtable();
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ if (!cfs.data.getView().getCurrentMemtable().isClean())
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
}
/**
* Flush if there is unflushed data that was written to the CommitLog before @param flushIfDirtyBefore
- * (inclusive). If @param flushIfDirtyBefore is null, flush if there is any unflushed data.
+ * (inclusive).
*
- * @return a Future such that when the future completes, all data inserted before forceFlush was called,
- * will be flushed.
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
*/
- public ListenableFuture<?> forceFlush(ReplayPosition flushIfDirtyBefore)
+ public ListenableFuture<ReplayPosition> forceFlush(ReplayPosition flushIfDirtyBefore)
{
- // we synchronize on the data tracker to ensure we don't race against other calls to switchMemtable(),
- // unnecessarily queueing memtables that are about to be made clean
- synchronized (data)
- {
- // during index build, 2ary index memtables can be dirty even if parent is not. if so,
- // we want to flush the 2ary index ones too.
- boolean clean = true;
- for (ColumnFamilyStore cfs : concatWithIndexes())
- clean &= cfs.data.getView().getCurrentMemtable().isCleanAfter(flushIfDirtyBefore);
+ // we don't loop through the remaining memtables since here we only care about commit log dirtiness
+ // and this does not vary between a table and its table-backed indexes
+ Memtable current = data.getView().getCurrentMemtable();
+ if (current.mayContainDataBefore(flushIfDirtyBefore))
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
+ }
- if (clean)
+ /**
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ private ListenableFuture<ReplayPosition> waitForFlushes()
+ {
+ // we grab the current memtable; once any preceding memtables have flushed, we know its
+ // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable)
+ final Memtable current = data.getView().getCurrentMemtable();
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(new Callable<ReplayPosition>()
+ {
+ public ReplayPosition call()
{
- // We could have a memtable for this column family that is being
- // flushed. Make sure the future returned wait for that so callers can
- // assume that any data inserted prior to the call are fully flushed
- // when the future returns (see #5241).
- ListenableFutureTask<?> task = ListenableFutureTask.create(new Runnable()
- {
- public void run()
- {
- logger.trace("forceFlush requested but everything is clean in {}", name);
- }
- }, null);
- postFlushExecutor.execute(task);
- return task;
+ logger.debug("forceFlush requested but everything is clean in {}", name);
+ return current.getCommitLogLowerBound();
}
-
- return switchMemtable();
- }
+ });
+ postFlushExecutor.execute(task);
+ return task;
}
- public void forceBlockingFlush()
+ public ReplayPosition forceBlockingFlush()
{
- FBUtilities.waitOnFuture(forceFlush());
+ return FBUtilities.waitOnFuture(forceFlush());
}
/**
* Both synchronises custom secondary indexes and provides ordering guarantees for futures on switchMemtable/flush
* etc, which expect to be able to wait until the flush (and all prior flushes) requested have completed.
*/
- private final class PostFlush implements Runnable
+ private final class PostFlush implements Callable<ReplayPosition>
{
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<SSTableReader> readers;
volatile FSWriteError flushFailure = null;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<SSTableReader> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
+ this.memtables = memtables;
+ this.readers = readers;
}
- public void run()
+ public ReplayPosition call()
{
+ if (discardFlushResults == ColumnFamilyStore.this)
+ return commitLogUpperBound;
+
writeBarrier.await();
/**
@@ -1018,7 +1037,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
try
{
- // we wait on the latch for the lastReplayPosition to be set, and so that waiters
+ // we wait on the latch for the commitLogUpperBound to be set, and so that waiters
// on this task can rely on all prior flushes being complete
latch.await();
}
@@ -1027,18 +1046,23 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
+ SSTableReader reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
+ }
}
-
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
+ return commitLogUpperBound;
}
}
@@ -1053,7 +1077,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
+ final List<SSTableReader> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@ -1069,43 +1094,33 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* that all write operations register themselves with, and assigning this barrier to the memtables,
* after which we *.issue()* the barrier. This barrier is used to direct write operations started prior
* to the barrier.issue() into the memtable we have switched out, and any started after to its replacement.
- * In doing so it also tells the write operations to update the lastReplayPosition of the memtable, so
+ * In doing so it also tells the write operations to update the commitLogUpperBound of the memtable, so
* that we know the CL position we are dirty to, which can be marked clean when we complete.
*/
writeBarrier = keyspace.writeOrder.newBarrier();
- memtables = new ArrayList<>();
// submit flushes for the memtable for any indexed sub-cfses, and our own
- AtomicReference<ReplayPosition> lastReplayPositionHolder = new AtomicReference<>();
+ AtomicReference<ReplayPosition> commitLogUpperBound = new AtomicReference<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
// switch all memtables, regardless of their dirty status, setting the barrier
// so that we can reach a coordinated decision about cleanliness once they
// are no longer possible to be modified
- Memtable mt = cfs.data.switchMemtable(truncate);
- mt.setDiscarding(writeBarrier, lastReplayPositionHolder);
- memtables.add(mt);
+ Memtable newMemtable = new Memtable(commitLogUpperBound, cfs);
+ Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable);
+ oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound);
+ memtables.add(oldMemtable);
}
- // we now attempt to define the lastReplayPosition; we do this by grabbing the current limit from the CL
- // and attempting to set the holder to this value. at the same time all writes to the memtables are
- // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
- // so that we know all operations prior to the position have not reached it yet
- ReplayPosition lastReplayPosition;
- while (true)
- {
- lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
- ReplayPosition currentLast = lastReplayPositionHolder.get();
- if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
- && lastReplayPositionHolder.compareAndSet(currentLast, lastReplayPosition))
- break;
- }
+ // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log
+ // records owned by this memtable
+ setCommitLogUpperBound(commitLogUpperBound);
// we then issue the barrier; this lets us wait for all operations started prior to the barrier to complete;
- // since this happens after wiring up the lastReplayPosition, we also know all operations with earlier
+ // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier
// replay positions have also completed, i.e. the memtables are done and ready to flush
writeBarrier.issue();
- postFlush = new PostFlush(!truncate, writeBarrier, lastReplayPosition);
+ postFlush = new PostFlush(!truncate, writeBarrier, commitLogUpperBound.get(), memtables, readers);
}
public void run()
@@ -1124,7 +1139,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, null);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
reclaim(memtable);
iter.remove();
}
@@ -1143,8 +1159,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (Memtable memtable : memtables)
{
// flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+ SSTableReader reader = memtable.flush();
+ memtable.cfs.data.replaceFlushed(memtable, reader);
reclaim(memtable);
+ readers.add(reader);
}
}
catch (FSWriteError e)
@@ -1174,6 +1192,38 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ // atomically set the upper bound for the commit log
+ private static void setCommitLogUpperBound(AtomicReference<ReplayPosition> commitLogUpperBound)
+ {
+ // we attempt to set the holder to the current commit log context. at the same time all writes to the memtables are
+ // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
+ // so that we know all operations prior to the position have not reached it yet
+ ReplayPosition lastReplayPosition;
+ while (true)
+ {
+ lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
+ ReplayPosition currentLast = commitLogUpperBound.get();
+ if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
+ && commitLogUpperBound.compareAndSet(currentLast, lastReplayPosition))
+ break;
+ }
+ }
+
+ @VisibleForTesting
+ // this method should ONLY be used for testing commit log behaviour; it discards the current memtable
+ // contents without marking the commit log clean, and prevents any proceeding flushes from marking
+ // the commit log as done, however they *will* terminate (unlike under typical failures) to ensure progress is made
+ public void simulateFailedFlush()
+ {
+ discardFlushResults = this;
+ data.markFlushing(data.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), this)));
+ }
+
+ public void resumeFlushing()
+ {
+ discardFlushResults = null;
+ }
+
/**
* Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
* queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
@@ -1589,11 +1639,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
maybeFail(data.dropSSTables(Predicates.in(sstables), compactionType, null));
}
- void replaceFlushed(Memtable memtable, SSTableReader sstable)
- {
- compactionStrategyWrapper.replaceFlushed(memtable, sstable);
- }
-
public boolean isValid()
{
return valid;
@@ -1615,6 +1660,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return data.getSSTables();
}
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
public Set<SSTableReader> getUncompactingSSTables()
{
return data.getUncompacting();
@@ -2670,6 +2720,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Void call()
{
cfs.data.reset();
+ cfs.getCompactionStrategy().shutdown();
+ cfs.getCompactionStrategy().startup();
return null;
}
}, true);
@@ -2707,39 +2759,42 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- // just nuke the memtable data w/o writing to disk first
- synchronized (data)
+ if (keyspace.getMetadata().durableWrites || takeSnapshot)
{
+ replayAfter = forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
+ replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
}
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
if (takeSnapshot)
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
for (SecondaryIndex index : indexManager.getIndexes())
index.truncateBlocking(truncatedAt);
@@ -2807,7 +2862,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public LifecycleTransaction call() throws Exception
{
assert data.getCompacting().isEmpty() : data.getCompacting();
- Collection<SSTableReader> sstables = Lists.newArrayList(AbstractCompactionStrategy.filterSuspectSSTables(getSSTables()));
+ Iterable<SSTableReader> sstables = getPermittedToCompactSSTables();
+ sstables = AbstractCompactionStrategy.filterSuspectSSTables(sstables);
+ sstables = ImmutableList.copyOf(sstables);
LifecycleTransaction modifier = data.tryModify(sstables, operationType);
assert modifier != null: "something marked things compacting while compactions are disabled";
return modifier;
@@ -3025,10 +3082,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param truncatedAt The timestamp of the truncation
* (all SSTables before that timestamp are going be marked as compacted)
- *
- * @return the most recent replay position of the truncated data
*/
- public ReplayPosition discardSSTables(long truncatedAt)
+ public void discardSSTables(long truncatedAt)
{
assert data.getCompacting().isEmpty() : data.getCompacting();
@@ -3040,11 +3095,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
truncatedSSTables.add(sstable);
}
- if (truncatedSSTables.isEmpty())
- return ReplayPosition.NONE;
-
- markObsolete(truncatedSSTables, OperationType.UNKNOWN);
- return ReplayPosition.getReplayPosition(truncatedSSTables);
+ if (!truncatedSSTables.isEmpty())
+ markObsolete(truncatedSSTables, OperationType.UNKNOWN);
}
public double getDroppableTombstoneRatio()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index c2901a4..2b3662f 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -346,7 +346,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- return null;
+ throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index fb4da72..b4ada09 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -29,21 +29,20 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -62,14 +61,17 @@ public class Memtable implements Comparable<Memtable>
// the write barrier for directing writes to this memtable during a switch
private volatile OpOrder.Barrier writeBarrier;
- // the last ReplayPosition owned by this Memtable; all ReplayPositions lower are owned by this or an earlier Memtable
- private volatile AtomicReference<ReplayPosition> lastReplayPosition;
- // the "first" ReplayPosition owned by this Memtable; this is inaccurate, and only used as a convenience to prevent CLSM flushing wantonly
- private final ReplayPosition minReplayPosition = CommitLog.instance.getContext();
+ // the precise upper bound of ReplayPosition owned by this memtable
+ private volatile AtomicReference<ReplayPosition> commitLogUpperBound;
+ // the precise lower bound of ReplayPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
+ private AtomicReference<ReplayPosition> commitLogLowerBound;
+ // the approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
+ // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
+ private final ReplayPosition approximateCommitLogLowerBound = CommitLog.instance.getContext();
public int compareTo(Memtable that)
{
- return this.minReplayPosition.compareTo(that.minReplayPosition);
+ return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound);
}
public static final class LastReplayPosition extends ReplayPosition
@@ -84,7 +86,6 @@ public class Memtable implements Comparable<Memtable>
// actually only store DecoratedKey.
private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@ -95,9 +96,10 @@ public class Memtable implements Comparable<Memtable>
// memtable was created with the new or old comparator.
public final CellNameType initialComparator;
- public Memtable(ColumnFamilyStore cfs)
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@ -131,7 +133,7 @@ public class Memtable implements Comparable<Memtable>
public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference<ReplayPosition> lastReplayPosition)
{
assert this.writeBarrier == null;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = lastReplayPosition;
this.writeBarrier = writeBarrier;
allocator.setDiscarding();
}
@@ -161,16 +163,21 @@ public class Memtable implements Comparable<Memtable>
// its current value and ours; if it HAS been finalised, we simply accept its judgement
// this permits us to coordinate a safe boundary, as the boundary choice is made
// atomically wrt our max() maintenance, so an operation cannot sneak into the past
- ReplayPosition currentLast = lastReplayPosition.get();
+ ReplayPosition currentLast = commitLogUpperBound.get();
if (currentLast instanceof LastReplayPosition)
return currentLast.compareTo(replayPosition) >= 0;
if (currentLast != null && currentLast.compareTo(replayPosition) >= 0)
return true;
- if (lastReplayPosition.compareAndSet(currentLast, replayPosition))
+ if (commitLogUpperBound.compareAndSet(currentLast, replayPosition))
return true;
}
}
+ public ReplayPosition getCommitLogLowerBound()
+ {
+ return commitLogLowerBound.get();
+ }
+
public boolean isLive()
{
return allocator.isLive();
@@ -181,9 +188,9 @@ public class Memtable implements Comparable<Memtable>
return rows.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@ -252,11 +259,6 @@ public class Memtable implements Comparable<Memtable>
return rows.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@ -285,20 +287,21 @@ public class Memtable implements Comparable<Memtable>
public Map.Entry<DecoratedKey, ColumnFamily> next()
{
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entry = iter.next();
+ Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
// Actual stored key should be true DecoratedKey
- assert entry.getKey() instanceof DecoratedKey;
+ assert entryRowPosition.getKey() instanceof DecoratedKey;
+ @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
+ Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
if (MEMORY_POOL.needToCopyOnHeap())
{
- DecoratedKey key = (DecoratedKey) entry.getKey();
+ DecoratedKey key = entry.getKey();
key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
}
// Store the reference to the current entry so that remove() can update the current size.
currentEntry = entry;
- // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- return (Map.Entry<DecoratedKey, ColumnFamily>) entry;
+ return entry;
}
public void remove()
@@ -315,9 +318,13 @@ public class Memtable implements Comparable<Memtable>
return rows.get(key);
}
- public long creationTime()
+ public SSTableReader flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
+ Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
+ File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@ -325,115 +332,88 @@ public class Memtable implements Comparable<Memtable>
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- FlushRunnable(ReplayPosition context)
- {
- this.context = context;
-
- long keySize = 0;
- for (RowPosition key : rows.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
- }
-
- protected void runMayThrow() throws Exception
+ long keySize = 0;
+ for (RowPosition key : rows.keySet())
{
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- SSTableReader sstable = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstable);
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected Directories getDirectories()
- {
- return cfs.directories;
- }
+ private SSTableReader writeSortedContents(File sstableDirectory)
+ {
+ logger.info("Writing {}", Memtable.this.toString());
- private SSTableReader writeSortedContents(ReplayPosition context, File sstableDirectory)
+ SSTableReader ssTable;
+ // errors when creating the writer that may leave empty temp files.
+ try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
+ for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
+ AtomicBTreeColumns cf = entry.getValue();
+
+ if (cf.isMarkedForDelete() && cf.hasColumns())
{
- AtomicBTreeColumns cf = entry.getValue();
-
- if (cf.isMarkedForDelete() && cf.hasColumns())
- {
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
- continue;
- }
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ // When every node is up, there's no reason to write batchlog data out to sstables
+ // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
+ // and BL data is strictly local, so we don't need to preserve tombstones for repair.
+ // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
+ // See CASSANDRA-4667.
+ if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
+ continue;
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
- context));
+ if (trackContention && cf.usePessimisticLocking())
+ heavilyContendedRowCount++;
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTable = null;
- }
+ if (!cf.isEmpty())
+ writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
+ commitLogUpperBound));
- return ssTable;
+ // temp sstables should contain non-repaired data.
+ ssTable = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
+ ssTable = null;
}
- }
- public SSTableWriter createFlushWriter(String filename)
- {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
+ if (heavilyContendedRowCount > 0)
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ return ssTable;
}
}
+ private SSTableWriter createFlushWriter(String filename)
+ {
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
+ .commitLogLowerBound(commitLogLowerBound.get())
+ .commitLogUpperBound(commitLogUpperBound.get());
+ return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ }
+
private static int estimateRowOverhead(final int count)
{
// calculate row overhead
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 98fb556..a58aeb4 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -33,7 +33,6 @@ import com.google.common.base.Predicate;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +69,7 @@ public class CommitLogReplayer
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final ICRC32 checksum;
private byte[] buffer;
@@ -79,7 +78,7 @@ public class CommitLogReplayer
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
this.futures = new ArrayList<Future<?>>();
@@ -89,7 +88,7 @@ public class CommitLogReplayer
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
this.checksum = CRC32Factory.instance.create();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@ -98,17 +97,12 @@ public class CommitLogReplayer
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
-
- // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@ -125,19 +119,21 @@ public class CommitLogReplayer
cfs.metadata.ksName,
cfs.metadata.cfName);
SystemKeyspace.removeTruncationRecord(cfs.metadata.cfId);
+ truncatedAt = null;
}
}
- else
- {
- rp = replayPositionOrdering.max(Arrays.asList(rp, truncatedAt));
- }
}
- cfPositions.put(cfs.metadata.cfId, rp);
+ ReplayPosition.ReplayFilter filter = new ReplayPosition.ReplayFilter(cfs.getSSTables(), truncatedAt);
+ if (!filter.isEmpty())
+ cfPersisted.put(cfs.metadata.cfId, filter);
+ else
+ globalPosition = ReplayPosition.NONE; // if we have no ranges for this CF, we must replay everything and filter
}
- ReplayPosition globalPosition = replayPositionOrdering.min(cfPositions.values());
- logger.trace("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPositions));
- return new CommitLogReplayer(commitLog, globalPosition, cfPositions, replayFilter);
+ if (globalPosition == null)
+ globalPosition = ReplayPosition.firstNotCovered(cfPersisted.values());
+ logger.debug("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPersisted));
+ return new CommitLogReplayer(commitLog, globalPosition, cfPersisted, replayFilter);
}
public void recover(File[] clogs) throws IOException
@@ -273,6 +269,18 @@ public class CommitLogReplayer
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
@SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
@@ -495,7 +503,7 @@ public class CommitLogReplayer
mutationStart, errorContext);
continue;
}
- replayMutation(buffer, serializedSize, reader.getFilePointer(), desc);
+ replayMutation(buffer, serializedSize, (int) reader.getFilePointer(), desc);
}
return true;
}
@@ -504,7 +512,7 @@ public class CommitLogReplayer
* Deserializes and replays a commit log entry.
*/
void replayMutation(byte[] inputBuffer, int size,
- final long entryLocation, final CommitLogDescriptor desc) throws IOException
+ final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
final Mutation mutation;
@@ -577,11 +585,7 @@ public class CommitLogReplayer
if (Schema.instance.getCF(columnFamily.id()) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(columnFamily.id());
-
- // replay if current segment is newer than last flushed one or,
- // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
+ if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index ca1969f..17802ad 100644
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@ -19,10 +19,10 @@ package org.apache.cassandra.db.commitlog;
import java.io.DataInput;
import java.io.IOException;
-import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
@@ -35,46 +35,78 @@ public class ReplayPosition implements Comparable<ReplayPosition>
public static final ReplayPositionSerializer serializer = new ReplayPositionSerializer();
// NONE is used for SSTables that are streamed from other nodes and thus have no relationship
- // with our local commitlog. The values satisfy the critera that
+ // with our local commitlog. The values satisfy the criteria that
// - no real commitlog segment will have the given id
// - it will sort before any real replayposition, so it will be effectively ignored by getReplayPosition
public static final ReplayPosition NONE = new ReplayPosition(-1, 0);
+ public final long segment;
+ public final int position;
+
/**
- * Convenience method to compute the replay position for a group of SSTables.
- * @param sstables
- * @return the most recent (highest) replay position
+ * A filter of known safe-to-discard commit log replay positions, based on
+ * the range covered by on disk sstables and those prior to the most recent truncation record
*/
- public static ReplayPosition getReplayPosition(Iterable<? extends SSTableReader> sstables)
+ public static class ReplayFilter
{
- if (Iterables.isEmpty(sstables))
- return NONE;
-
- Function<SSTableReader, ReplayPosition> f = new Function<SSTableReader, ReplayPosition>()
+ final NavigableMap<ReplayPosition, ReplayPosition> persisted = new TreeMap<>();
+ public ReplayFilter(Iterable<SSTableReader> onDisk, ReplayPosition truncatedAt)
{
- public ReplayPosition apply(SSTableReader sstable)
+ for (SSTableReader reader : onDisk)
{
- return sstable.getReplayPosition();
+ ReplayPosition start = reader.getSSTableMetadata().commitLogLowerBound;
+ ReplayPosition end = reader.getSSTableMetadata().commitLogUpperBound;
+ add(persisted, start, end);
}
- };
- Ordering<ReplayPosition> ordering = Ordering.from(ReplayPosition.comparator);
- return ordering.max(Iterables.transform(sstables, f));
- }
+ if (truncatedAt != null)
+ add(persisted, ReplayPosition.NONE, truncatedAt);
+ }
+ private static void add(NavigableMap<ReplayPosition, ReplayPosition> ranges, ReplayPosition start, ReplayPosition end)
+ {
+ // extend ourselves to cover any ranges we overlap
+ // record directly preceding our end may extend past us, so take the max of our end and its
+ Map.Entry<ReplayPosition, ReplayPosition> extend = ranges.floorEntry(end);
+ if (extend != null && extend.getValue().compareTo(end) > 0)
+ end = extend.getValue();
+
+ // record directly preceding our start may extend into us; if it does, we take it as our start
+ extend = ranges.lowerEntry(start);
+ if (extend != null && extend.getValue().compareTo(start) >= 0)
+ start = extend.getKey();
+
+ ranges.subMap(start, end).clear();
+ ranges.put(start, end);
+ }
- public final long segment;
- public final int position;
+ public boolean shouldReplay(ReplayPosition position)
+ {
+ // replay ranges are start exclusive, end inclusive
+ Map.Entry<ReplayPosition, ReplayPosition> range = persisted.lowerEntry(position);
+ return range == null || position.compareTo(range.getValue()) > 0;
+ }
- public static final Comparator<ReplayPosition> comparator = new Comparator<ReplayPosition>()
- {
- public int compare(ReplayPosition o1, ReplayPosition o2)
+ public boolean isEmpty()
{
- if (o1.segment != o2.segment)
- return Long.valueOf(o1.segment).compareTo(o2.segment);
+ return persisted.isEmpty();
+ }
+ }
- return Integer.valueOf(o1.position).compareTo(o2.position);
+ public static ReplayPosition firstNotCovered(Iterable<ReplayFilter> ranges)
+ {
+ ReplayPosition min = null;
+ for (ReplayFilter map : ranges)
+ {
+ ReplayPosition first = map.persisted.firstEntry().getValue();
+ if (min == null)
+ min = first;
+ else
+ min = Ordering.natural().min(min, first);
}
- };
+ if (min == null)
+ return NONE;
+ return min;
+ }
public ReplayPosition(long segment, int position)
{
@@ -83,9 +115,12 @@ public class ReplayPosition implements Comparable<ReplayPosition>
this.position = position;
}
- public int compareTo(ReplayPosition other)
+ public int compareTo(ReplayPosition that)
{
- return comparator.compare(this, other);
+ if (this.segment != that.segment)
+ return Long.compare(this.segment, that.segment);
+
+ return Integer.compare(this.position, that.position);
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 02497df..575aa51 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -232,9 +232,6 @@ public abstract class AbstractCompactionStrategy
*/
public void replaceFlushed(Memtable memtable, SSTableReader sstable)
{
- cfs.getTracker().replaceFlushed(memtable, sstable);
- if (sstable != null)
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e9137e2..d1fe702 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -98,8 +98,6 @@ public class Scrubber implements Closeable
// Calculate the expected compacted filesize
this.destination = cfs.directories.getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(toScrub, OperationType.SCRUB));
- if (destination == null)
- throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
this.controller = transaction.isOffline()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index a074216..5d5701f 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -31,11 +31,13 @@ import com.google.common.collect.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -45,6 +47,8 @@ import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.OpOrder;
import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.in;
+import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.Iterables.filter;
import static java.util.Collections.singleton;
@@ -195,10 +199,12 @@ public class Tracker
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@ -294,9 +300,8 @@ public class Tracker
*
* @return the previously active memtable
*/
- public Memtable switchMemtable(boolean truncating)
+ public Memtable switchMemtable(boolean truncating, Memtable newMemtable)
{
- Memtable newMemtable = new Memtable(cfstore);
Pair<View, View> result = apply(View.switchMemtable(newMemtable));
if (truncating)
notifyRenewed(newMemtable);
@@ -328,15 +333,35 @@ public class Tracker
Throwable fail;
fail = updateSizeTracking(emptySet(), singleton(sstable), null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstable, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
+ public void permitCompactionOfFlushed(SSTableReader sstable)
+ {
+ if (sstable == null)
+ return;
+
+ apply(View.permitCompactionOfFlushed(sstable));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
+ notifyAdded(sstable);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
+
// MISCELLANEOUS public utility calls
@@ -346,6 +371,12 @@ public class Tracker
return view.get().sstables;
}
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index 73ba131..fba1627 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.db.lifecycle;
import java.util.*;
+import javax.annotation.Nullable;
+
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
@@ -66,6 +68,7 @@ public class View
public final List<Memtable> flushingMemtables;
public final Set<SSTableReader> compacting;
public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
@@ -74,7 +77,7 @@ public class View
public final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, SSTableIntervalTree intervalTree)
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@ -88,6 +91,7 @@ public class View
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
this.compacting = compacting;
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@ -155,7 +159,7 @@ public class View
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
replace(view.compacting, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@ -169,7 +173,7 @@ public class View
public boolean apply(View view)
{
for (SSTableReader reader : readers)
- if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted())
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
return false;
return true;
}
@@ -186,7 +190,7 @@ public class View
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting,
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@ -201,7 +205,7 @@ public class View
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -220,7 +224,7 @@ public class View
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -237,15 +241,33 @@ public class View
if (flushed == null)
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compacting, view.intervalTree);
+ view.compacting, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compacting,
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e81e4e9..c303975 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1864,11 +1864,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstableMetadata.compressionRatio;
}
- public ReplayPosition getReplayPosition()
- {
- return sstableMetadata.replayPosition;
- }
-
public long getMinTimestamp()
{
return sstableMetadata.minTimestamp;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index faaa89e..41a83e1 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -52,6 +52,8 @@ public abstract class Version
public abstract boolean hasNewFileName();
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index a1e32cf..9244bbb 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -125,7 +125,7 @@ public class BigFormat implements SSTableFormat
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "la";
+ public static final String current_version = "lb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@ -135,6 +135,7 @@ public class BigFormat implements SSTableFormat
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@ -143,6 +144,7 @@ public class BigFormat implements SSTableFormat
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ private final boolean hasCommitLogLowerBound;
public BigVersion(String version)
{
@@ -155,6 +157,7 @@ public class BigFormat implements SSTableFormat
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("la") >= 0;
+ hasCommitLogLowerBound = version.compareTo("lb") >= 0;
}
@Override
@@ -199,6 +202,11 @@ public class BigFormat implements SSTableFormat
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
public boolean isCompatible()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 505bac0..3a01f87 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -432,7 +432,7 @@ public class BigTableWriter extends SSTableWriter
File file = new File(desc.filenameFor(Component.STATS));
try (SequentialWriter out = SequentialWriter.open(file);)
{
- desc.getMetadataSerializer().serialize(components, out.stream);
+ desc.getMetadataSerializer().serialize(components, desc.version, out.stream);
out.setDescriptor(desc).finish();
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index ed1f327..c8e6ee8 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -72,7 +72,7 @@ public class CompactionMetadata extends MetadataComponent
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component) throws IOException
+ public int serializedSize(CompactionMetadata component, Version version) throws IOException
{
int size = 0;
size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
@@ -83,7 +83,7 @@ public class CompactionMetadata extends MetadataComponent
return size;
}
- public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeInt(component.ancestors.size());
for (int g : component.ancestors)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
index dc8fbdf..e3d867f 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
@@ -35,7 +35,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @return serialized size of this component
* @throws IOException
*/
- int serializedSize(T component) throws IOException;
+ int serializedSize(T component, Version version) throws IOException;
/**
* Serialize metadata component to given output.
@@ -45,7 +45,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @param out serialize destination
* @throws IOException
*/
- void serialize(T component, DataOutputPlus out) throws IOException;
+ void serialize(T component, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize metadata component from given input.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index df577df..a7d23f4 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -22,6 +22,7 @@ import java.util.EnumSet;
import java.util.Map;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -37,7 +38,7 @@ public interface IMetadataSerializer
* @param out
* @throws IOException
*/
- void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException;
+ void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize specified metadata components from given descriptor.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 4bd060e..bfeb930 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Maps;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -42,7 +43,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
* Legacy serialization is only used for SSTable level reset.
*/
@Override
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
@@ -52,7 +53,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
@@ -70,6 +71,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
out.writeInt(stats.maxColumnNames.size());
for (ByteBuffer columnName : stats.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@ -91,7 +94,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -116,6 +120,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
@@ -124,7 +130,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
components.put(MetadataType.STATS,
new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 5962a46..579ff7a 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -28,6 +28,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -69,6 +70,7 @@ public class MetadataCollector
return new StatsMetadata(defaultRowSizeHistogram(),
defaultColumnCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@ -83,7 +85,8 @@ public class MetadataCollector
protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
protected long minTimestamp = Long.MAX_VALUE;
protected long maxTimestamp = Long.MIN_VALUE;
protected int maxLocalDeletionTime = Integer.MIN_VALUE;
@@ -113,7 +116,23 @@ public class MetadataCollector
{
this(columnNameComparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
// Get the max timestamp of the precompacted sstables
// and adds generation of live ancestors
@@ -199,9 +218,15 @@ public class MetadataCollector
return this;
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@ -257,7 +282,8 @@ public class MetadataCollector
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
[05/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 30ed85b..ca7fe82 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.FileDataInput;
@@ -49,7 +50,7 @@ public class MetadataSerializer implements IMetadataSerializer
{
private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
// sort components by type
List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
@@ -66,12 +67,12 @@ public class MetadataSerializer implements IMetadataSerializer
out.writeInt(type.ordinal());
// serialize position
out.writeInt(lastPosition);
- lastPosition += type.serializer.serializedSize(component);
+ lastPosition += type.serializer.serializedSize(component, version);
}
// serialize components
for (MetadataComponent component : sortedComponents)
{
- component.getType().serializer.serialize(component, out);
+ component.getType().serializer.serialize(component, version, out);
}
}
@@ -153,7 +154,7 @@ public class MetadataSerializer implements IMetadataSerializer
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(tmpDescriptor.filenameFor(Component.STATS))))
{
- serialize(currentComponents, out);
+ serialize(currentComponents, descriptor.version, out);
out.flush();
}
// we cant move a file on top of another file in windows:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index f2eb1af..3d48e34 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -43,7 +43,8 @@ public class StatsMetadata extends MetadataComponent
public final EstimatedHistogram estimatedRowSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
public final int maxLocalDeletionTime;
@@ -57,7 +58,8 @@ public class StatsMetadata extends MetadataComponent
public StatsMetadata(EstimatedHistogram estimatedRowSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
int maxLocalDeletionTime,
@@ -71,7 +73,8 @@ public class StatsMetadata extends MetadataComponent
{
this.estimatedRowSize = estimatedRowSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
this.maxLocalDeletionTime = maxLocalDeletionTime;
@@ -117,7 +120,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -134,7 +138,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -157,7 +162,8 @@ public class StatsMetadata extends MetadataComponent
return new EqualsBuilder()
.append(estimatedRowSize, that.estimatedRowSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
@@ -177,7 +183,8 @@ public class StatsMetadata extends MetadataComponent
return new HashCodeBuilder()
.append(estimatedRowSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
.append(maxLocalDeletionTime)
@@ -193,12 +200,12 @@ public class StatsMetadata extends MetadataComponent
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component) throws IOException
+ public int serializedSize(StatsMetadata component, Version version) throws IOException
{
int size = 0;
size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition, TypeSizes.NATIVE);
+ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
@@ -211,14 +218,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
size += 2 + columnName.remaining(); // with short length
size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
return size;
}
- public void serialize(StatsMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
{
EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
out.writeInt(component.maxLocalDeletionTime);
@@ -233,13 +242,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
out.writeBoolean(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
public StatsMetadata deserialize(Version version, DataInput in) throws IOException
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -264,9 +276,12 @@ public class StatsMetadata extends MetadataComponent
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
return new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index 603732b..4ca078b 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -71,12 +71,12 @@ public class ValidationMetadata extends MetadataComponent
public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
{
- public int serializedSize(ValidationMetadata component) throws IOException
+ public int serializedSize(ValidationMetadata component, Version version) throws IOException
{
return TypeSizes.NATIVE.sizeof(component.partitioner) + 8;
}
- public void serialize(ValidationMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(ValidationMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeUTF(component.partitioner);
out.writeDouble(component.bloomFilterFPChance);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
deleted file mode 100644
index 1a15d6f..0000000
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public abstract class DiskAwareRunnable extends WrappedRunnable
-{
- protected Directories.DataDirectory getWriteDirectory(long writeSize)
- {
- Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
- if (directory == null)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
-
- return directory;
- }
-
- /**
- * Get sstable directories for the CF.
- * @return Directories instance for the CF.
- */
- protected abstract Directories getDirectories();
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index dd56b8b..6911ec6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -122,14 +122,11 @@ public class StreamReceiveTask extends StreamTask
for (SSTableWriter writer : task.sstables)
writer.abort();
task.sstables.clear();
- task.session.taskCompleted(task);
return;
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
File lockfiledir = cfs.directories.getWriteableLocationAsFile(task.sstables.size() * 256L);
- if (lockfiledir == null)
- throw new IOError(new IOException("All disks full"));
StreamLockfile lockfile = new StreamLockfile(lockfiledir, UUID.randomUUID());
lockfile.create(task.sstables);
List<SSTableReader> readers = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index 2665f40..8319014 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -70,7 +70,8 @@ public class SSTableMetadataViewer
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
out.println("Estimated tombstone drop times:");
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 1b4edee..f9b4156 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -452,7 +452,7 @@ public class CommitLogStressTest
int cells = 0;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index c377a21..0c46061 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -59,7 +59,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
Mutation mutation;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 93b88b4..adeb778 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.MockSchema;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
@@ -263,15 +264,15 @@ public class TrackerTest
Tracker tracker = cfs.getTracker();
tracker.subscribe(listener);
- Memtable prev1 = tracker.switchMemtable(true);
+ Memtable prev1 = tracker.switchMemtable(true, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write1 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier1 = cfs.keyspace.writeOrder.newBarrier();
- prev1.setDiscarding(barrier1, new AtomicReference<ReplayPosition>());
+ prev1.setDiscarding(barrier1, new AtomicReference<>(CommitLog.instance.getContext()));
barrier1.issue();
- Memtable prev2 = tracker.switchMemtable(false);
+ Memtable prev2 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write2 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier2 = cfs.keyspace.writeOrder.newBarrier();
- prev2.setDiscarding(barrier2, new AtomicReference<ReplayPosition>());
+ prev2.setDiscarding(barrier2, new AtomicReference<>(CommitLog.instance.getContext()));
barrier2.issue();
Memtable cur = tracker.getView().getCurrentMemtable();
OpOrder.Group writecur = cfs.keyspace.writeOrder.getCurrent();
@@ -297,6 +298,9 @@ public class TrackerTest
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
tracker.replaceFlushed(prev2, reader);
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(reader);
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
@@ -307,16 +311,17 @@ public class TrackerTest
tracker = cfs.getTracker();
listener = new MockListener(false);
tracker.subscribe(listener);
- prev1 = tracker.switchMemtable(false);
+ prev1 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
tracker.replaceFlushed(prev1, reader);
+ tracker.permitCompactionOfFlushed(reader);
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
+ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 32a81e2..5706598 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -208,6 +208,6 @@ public class ViewTest
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
+ Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 931422b..3bef89e 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -113,7 +113,7 @@ public class CompressedRandomAccessReaderTest
ChannelProxy channel = new ChannelProxy(f);
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
SequentialWriter writer = compressed
? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
: SequentialWriter.open(f);
@@ -166,7 +166,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
@@ -251,7 +251,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 27b866d..43c44fd 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -85,7 +85,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index eda4f17..19fa7c4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.EnumSet;
@@ -27,12 +28,16 @@ import java.util.Set;
import com.google.common.collect.Sets;
import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
@@ -45,18 +50,51 @@ public class MetadataSerializerTest
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
+
+ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
+
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
+ // Serialize to tmp file
+ File statsFile = File.createTempFile(Component.STATS.name, null);
+ try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
+ {
+ serializer.serialize(metadata, version, out);
+ }
+ return statsFile;
+ }
+
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
new long[] { 3L, 4L, 5L });
EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
new long[] { 8L, 9L, 10L });
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ ReplayPosition start = new ReplayPosition(11L, 12);
+ ReplayPosition end = new ReplayPosition(15L, 9);
long minTimestamp = 2162517136L;
long maxTimestamp = 4162517136L;
MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
.estimatedRowSize(rowSizes)
.estimatedColumnCount(columnCounts)
- .replayPosition(rp);
+ .commitLogLowerBound(start)
+ .commitLogUpperBound(end);
collector.updateMinTimestamp(minTimestamp);
collector.updateMaxTimestamp(maxTimestamp);
@@ -67,23 +105,35 @@ public class MetadataSerializerTest
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
+ return originalMetadata;
+ }
+
+ @Test
+ public void testLaReadsLb() throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
MetadataSerializer serializer = new MetadataSerializer();
- // Serialize to tmp file
- File statsFile = File.createTempFile(Component.STATS.name, null);
- try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
- {
- serializer.serialize(originalMetadata, out);
- }
+ // Write metadata in two minor formats.
+ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
+ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ // Reading both as earlier version should yield identical results.
+ Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
- assertEquals(originalMetadata.get(type), deserialized.get(type));
+ assertEquals(deserializedLa.get(type), deserializedLb.get(type));
+ if (!originalMetadata.get(type).equals(deserializedLb.get(type)))
+ {
+ // Currently only STATS can be different. Change if no longer the case
+ assertEquals(MetadataType.STATS, type);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
index 8409a26..1f66fb7 100644
--- a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
@@ -157,7 +157,7 @@ public class IntervalTreeTest
public String deserialize(DataInput in) throws IOException { return in.readUTF(); }
public long serializedSize(String v, TypeSizes s) { return v.length(); }
},
- (Constructor<Interval<Integer, String>>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class)
+ (Constructor<Interval<Integer, String>>) (Constructor<?>) Interval.class.getConstructor(Object.class, Object.class, Object.class)
);
DataOutputBuffer out = new DataOutputBuffer();
[15/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-CompressionInfo.db
index 0000000,0000000..288663f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Data.db
index 0000000,0000000..6a2f28e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
index 0000000,0000000..fb255bd
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++655951031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Index.db
index 0000000,0000000..b3094bf
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Statistics.db
index 0000000,0000000..55dcdce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..adb7fc4
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Data.db
index 0000000,0000000..5cd0481
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
index 0000000,0000000..7d2bdbe
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3883019031
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Index.db
index 0000000,0000000..56f29df
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Statistics.db
index 0000000,0000000..dff24fe
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..19d25c9
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Data.db
index 0000000,0000000..77a1394
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
index 0000000,0000000..172b695
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++2876949266
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Index.db
index 0000000,0000000..59e65ca
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Statistics.db
index 0000000,0000000..4c67715
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..56c95a8
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Data.db
index 0000000,0000000..00a7a39
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..f9f2fff
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++1214766167
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Index.db
index 0000000,0000000..d094f73
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..33c33f2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_simple_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index fea83c1,0c46061..e690785
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@@ -59,9 -59,9 +59,9 @@@ public class CommitLogTestReplayer exte
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
- FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
+ RebufferingInputStream bufIn = new DataInputBuffer(inputBuffer, 0, size);
Mutation mutation;
try
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 7b9b19c,adeb778..b8de711
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@@ -38,8 -38,10 +38,9 @@@ import org.apache.cassandra.MockSchema
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+ import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.notifications.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@@ -292,17 -291,19 +293,20 @@@ public class TrackerTes
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev1));
Assert.assertEquals(2, tracker.getView().flushingMemtables.size());
-- tracker.replaceFlushed(prev1, null);
++ tracker.replaceFlushed(prev1, Collections.emptyList());
Assert.assertEquals(1, tracker.getView().flushingMemtables.size());
Assert.assertTrue(tracker.getView().flushingMemtables.contains(prev2));
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
- tracker.replaceFlushed(prev2, reader);
+ tracker.replaceFlushed(prev2, Collections.singleton(reader));
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
- tracker.permitCompactionOfFlushed(reader);
++ tracker.permitCompactionOfFlushed(singleton(reader));
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
+ Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
+ Assert.assertTrue(reader.isKeyCacheSetup());
Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount());
// test invalidated CFS
@@@ -314,14 -315,13 +318,13 @@@
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
- tracker.replaceFlushed(prev1, reader);
- tracker.permitCompactionOfFlushed(reader);
+ tracker.replaceFlushed(prev1, Collections.singleton(reader));
++ tracker.permitCompactionOfFlushed(Collections.singleton(reader));
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(3, listener.received.size());
- Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertTrue(listener.received.get(1) instanceof SSTableDeletingNotification);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
- Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
++ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
++ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 523c203,5706598..8a5e00e
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@@ -212,6 -208,6 +212,6 @@@ public class ViewTes
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader, SSTableReader>emptyMap(), SSTableIntervalTree.build(sstables));
- Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
++ Collections.<SSTableReader, SSTableReader>emptyMap(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 74b5c74,3bef89e..802d9c8
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@@ -115,56 -107,40 +115,56 @@@ public class CompressedRandomAccessRead
}
}
- private void testResetAndTruncate(File f, boolean compressed, int junkSize) throws IOException
+ private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException
{
final String filename = f.getAbsolutePath();
- ChannelProxy channel = new ChannelProxy(f);
- try
+ try(ChannelProxy channel = new ChannelProxy(f))
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
- : SequentialWriter.open(f);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = compressed
+ ? new CompressedSequentialWriter(f, filename + ".metadata", CompressionParams.snappy(), sstableMetadataCollector)
+ : SequentialWriter.open(f))
+ {
+ writer.write("The quick ".getBytes());
+ DataPosition mark = writer.mark();
+ writer.write("blue fox jumps over the lazy dog".getBytes());
+
+ // write enough to be sure to change chunk
+ for (int i = 0; i < junkSize; ++i)
+ {
+ writer.write((byte) 1);
+ }
+
+ writer.resetAndTruncate(mark);
+ writer.write("brown fox jumps over the lazy dog".getBytes());
+ writer.finish();
+ }
+ assert f.exists();
- writer.write("The quick ".getBytes());
- FileMark mark = writer.mark();
- writer.write("blue fox jumps over the lazy dog".getBytes());
+ CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null;
+ RandomAccessReader.Builder builder = compressed
+ ? new CompressedRandomAccessReader.Builder(channel, compressionMetadata)
+ : new RandomAccessReader.Builder(channel);
- // write enough to be sure to change chunk
- for (int i = 0; i < junkSize; ++i)
+ if (usemmap)
{
- writer.write((byte)1);
+ if (compressed)
+ builder.regions(MmappedRegions.map(channel, compressionMetadata));
+ else
+ builder.regions(MmappedRegions.map(channel, f.length()));
}
- writer.resetAndTruncate(mark);
- writer.write("brown fox jumps over the lazy dog".getBytes());
- writer.finish();
+ try(RandomAccessReader reader = builder.build())
+ {
+ String expected = "The quick brown fox jumps over the lazy dog";
+ assertEquals(expected.length(), reader.length());
+ byte[] b = new byte[expected.length()];
+ reader.readFully(b);
+ assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + '\'';
+ }
- assert f.exists();
- RandomAccessReader reader = compressed
- ? CompressedRandomAccessReader.open(channel, new CompressionMetadata(filename + ".metadata", f.length()))
- : RandomAccessReader.open(f);
- String expected = "The quick brown fox jumps over the lazy dog";
- assertEquals(expected.length(), reader.length());
- byte[] b = new byte[expected.length()];
- reader.readFully(b);
- assert new String(b).equals(expected) : "Expecting '" + expected + "', got '" + new String(b) + "'";
+ if (usemmap)
+ builder.regions.close();
}
finally
{
@@@ -187,11 -166,8 +187,11 @@@
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
- try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
+ assertTrue(file.createNewFile());
+ assertTrue(metadata.createNewFile());
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)).replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), CompressionParams.snappy(), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
writer.finish();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 9b09f0b,43c44fd..e045aad
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@@ -88,13 -85,12 +88,13 @@@ public class CompressedSequentialWriter
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance))).replayPosition(null);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(Arrays.<AbstractType<?>>asList(BytesType.instance)));
+
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
- try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
+ try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", compressionParameters, sstableMetadataCollector);)
{
- Random r = new Random();
+ Random r = new Random(42);
// Test both write with byte[] and ByteBuffer
r.nextBytes(dataPre);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 4b9a769,f4b9617..62228e3
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@@ -60,33 -58,13 +60,33 @@@ import org.apache.cassandra.utils.FBUti
*/
public class LegacySSTableTest
{
+ private static final Logger logger = LoggerFactory.getLogger(LegacySSTableTest.class);
+
public static final String LEGACY_SSTABLE_PROP = "legacy-sstable-root";
- public static final String KSNAME = "Keyspace1";
- public static final String CFNAME = "Standard1";
- public static Set<String> TEST_DATA;
public static File LEGACY_SSTABLE_ROOT;
+ /**
+ * When adding a new sstable version, add that one here.
+ * See {@link #testGenerateSstables()} to generate sstables.
+ * Take care on commit as you need to add the sstable files using {@code git add -f}
+ */
- public static final String[] legacyVersions = {"ma", "la", "ka", "jb"};
++ public static final String[] legacyVersions = {"mb", "ma", "la", "ka", "jb"};
+
+ // 1200 chars
+ static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +
+ "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789";
+
@BeforeClass
public static void defineSchema() throws ConfigurationException
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 4c10882,19fa7c4..93365ef
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@@ -19,25 -18,30 +19,28 @@@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.Map;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
import org.junit.Test;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
-import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
-import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
+import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.EstimatedHistogram;
import static org.junit.Assert.assertEquals;
@@@ -46,29 -50,81 +49,78 @@@ public class MetadataSerializerTes
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
- CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
-
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
- MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp);
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
++ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
- String partitioner = RandomPartitioner.class.getCanonicalName();
- double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST));
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
- MetadataSerializer serializer = new MetadataSerializer();
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
// Serialize to tmp file
File statsFile = File.createTempFile(Component.STATS.name, null);
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
{
- serializer.serialize(originalMetadata, out, BigFormat.latestVersion);
- serializer.serialize(metadata, version, out);
++ serializer.serialize(metadata, out, version);
}
+ return statsFile;
+ }
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
- EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
- new long[] { 3L, 4L, 5L });
- EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
- new long[] { 8L, 9L, 10L });
- ReplayPosition start = new ReplayPosition(11L, 12);
- ReplayPosition end = new ReplayPosition(15L, 9);
- long minTimestamp = 2162517136L;
- long maxTimestamp = 4162517136L;
-
- MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
- .estimatedRowSize(rowSizes)
- .estimatedColumnCount(columnCounts)
- .commitLogLowerBound(start)
- .commitLogUpperBound(end);
- collector.updateMinTimestamp(minTimestamp);
- collector.updateMaxTimestamp(maxTimestamp);
-
- Set<Integer> ancestors = Sets.newHashSet(1, 2, 3, 4);
- for (int i : ancestors)
- collector.addAncestor(i);
++ ReplayPosition club = new ReplayPosition(11L, 12);
++ ReplayPosition cllb = new ReplayPosition(9L, 12);
++
++ CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
++ MetadataCollector collector = new MetadataCollector(cfm.comparator)
++ .commitLogLowerBound(cllb)
++ .commitLogUpperBound(club);
+
+ String partitioner = RandomPartitioner.class.getCanonicalName();
+ double bfFpChance = 0.1;
- Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
++ Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.emptyList()));
+ return originalMetadata;
+ }
+
+ @Test
- public void testLaReadsLb() throws IOException
++ public void testLaReadLb() throws IOException
++ {
++ testOldReadsNew("la", "lb");
++ }
++
++ @Test
++ public void testMaReadMb() throws IOException
++ {
++ testOldReadsNew("ma", "mb");
++ }
++
++ public void testOldReadsNew(String oldV, String newV) throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ // Write metadata in two minor formats.
- File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
- File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
-
++ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV));
++ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
+ // Reading both as earlier version should yield identical results.
- Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
++ Descriptor desc = new Descriptor(oldV, statsFileLb.getParentFile(), "", "", 0, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
index 9df3fed,0000000..7cf7bd3
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java
@@@ -1,375 -1,0 +1,374 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import com.google.common.primitives.Ints;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.CompressionParams;
+import org.apache.cassandra.utils.ChecksumType;
+
+import static junit.framework.Assert.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class MmappedRegionsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class);
+
+ private static ByteBuffer allocateBuffer(int size)
+ {
+ ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size));
+ long seed = System.nanoTime();
+ //seed = 365238103404423L;
+ logger.info("Seed {}", seed);
+
+ new Random(seed).nextBytes(ret.array());
+ return ret;
+ }
+
+ private static File writeFile(String fileName, ByteBuffer buffer) throws IOException
+ {
+ File ret = File.createTempFile(fileName, "1");
+ ret.deleteOnExit();
+
+ try (SequentialWriter writer = SequentialWriter.open(ret))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ assert ret.exists();
+ assert ret.length() >= buffer.capacity();
+ return ret;
+
+ }
+
+ @Test
+ public void testEmpty() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ assertTrue(regions.isEmpty());
+ assertTrue(regions.isValid(channel));
+ }
+ }
+
+ @Test
+ public void testTwoSegments() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(2048);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ for (int i = 0; i < 1024; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+
+ regions.extend(2048);
+ for (int i = 0; i < 2048; i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ if (i < 1024)
+ {
+ assertEquals(0, region.bottom());
+ assertEquals(1024, region.top());
+ }
+ else
+ {
+ assertEquals(1024, region.bottom());
+ assertEquals(2048, region.top());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSmallSegmentSize() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(1024);
+ regions.extend(2048);
+ regions.extend(4096);
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testAllocRegions() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3);
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity());
+
+ final int SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ }
+ }
+
+ @Test
+ public void testCopy() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4))
+ {
+ // create 3 more segments, one per quater capacity
+ regions.extend(buffer.capacity() / 2);
+ regions.extend(3 * buffer.capacity() / 4);
+ regions.extend(buffer.capacity());
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ assertFalse(snapshot.isCleanedUp());
+
+ final int SIZE = buffer.capacity() / 4;
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = snapshot.floor(i);
+ assertNotNull(region);
+ assertEquals(SIZE * (i / SIZE), region.bottom());
+ assertEquals(SIZE + (SIZE * (i / SIZE)), region.top());
+
+ // check we can access the buffer
+ assertNotNull(region.buffer.duplicate().getInt());
+ }
+
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ assertTrue(snapshot.isCleanedUp());
+ }
+
+ @Test(expected = AssertionError.class)
+ public void testCopyCannotExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+
+ MmappedRegions snapshot;
+ ChannelProxy channelCopy;
+
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(buffer.capacity() / 2);
+
+ // make a snapshot
+ snapshot = regions.sharedCopy();
+
+ // keep the channel open
+ channelCopy = channel.sharedCopy();
+ }
+
+ try
+ {
+ snapshot.extend(buffer.capacity());
+ }
+ finally
+ {
+ assertNull(snapshot.close(null));
+ assertNull(channelCopy.close(null));
+ }
+ }
+
+ @Test
+ public void testExtendOutOfOrder() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(4096);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(4096);
+ regions.extend(1024);
+ regions.extend(2048);
+
+ for (int i = 0; i < buffer.capacity(); i++)
+ {
+ MmappedRegions.Region region = regions.floor(i);
+ assertNotNull(region);
+ assertEquals(0, region.bottom());
+ assertEquals(4096, region.top());
+ }
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNegativeExtend() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer));
+ MmappedRegions regions = MmappedRegions.empty(channel))
+ {
+ regions.extend(-1);
+ }
+ }
+
+ @Test
+ public void testMapForCompressionMetadata() throws Exception
+ {
+ int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
+ MmappedRegions.MAX_SEGMENT_SIZE = 1024;
+
+ ByteBuffer buffer = allocateBuffer(128 * 1024);
+ File f = File.createTempFile("testMapForCompressionMetadata", "1");
+ f.deleteOnExit();
+
+ File cf = File.createTempFile(f.getName() + ".metadata", "1");
+ cf.deleteOnExit();
+
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance))
- .replayPosition(null);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = new CompressedSequentialWriter(f,
+ cf.getAbsolutePath(),
+ CompressionParams.snappy(),
+ sstableMetadataCollector))
+ {
+ writer.write(buffer);
+ writer.finish();
+ }
+
+ CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
+ try(ChannelProxy channel = new ChannelProxy(f);
+ MmappedRegions regions = MmappedRegions.map(channel, metadata))
+ {
+
+ assertFalse(regions.isEmpty());
+ int i = 0;
+ while(i < buffer.capacity())
+ {
+ CompressionMetadata.Chunk chunk = metadata.chunkFor(i);
+
+ MmappedRegions.Region region = regions.floor(chunk.offset);
+ assertNotNull(region);
+
+ ByteBuffer compressedChunk = region.buffer.duplicate();
+ assertNotNull(compressedChunk);
+ assertEquals(chunk.length + 4, compressedChunk.capacity());
+
+ assertEquals(chunk.offset, region.bottom());
+ assertEquals(chunk.offset + chunk.length + 4, region.top());
+
+ i += metadata.chunkLength();
+ }
+ }
+ finally
+ {
+ MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
+ metadata.close();
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap1() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, 0))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap2() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, -1L))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIllegalArgForMap3() throws Exception
+ {
+ ByteBuffer buffer = allocateBuffer(1024);
+ try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer));
+ MmappedRegions regions = MmappedRegions.map(channel, null))
+ {
+ assertTrue(regions.isEmpty());
+ }
+ }
+
+}
[03/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
index 30ed85b..ca7fe82 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.FileDataInput;
@@ -49,7 +50,7 @@ public class MetadataSerializer implements IMetadataSerializer
{
private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class);
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
// sort components by type
List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values());
@@ -66,12 +67,12 @@ public class MetadataSerializer implements IMetadataSerializer
out.writeInt(type.ordinal());
// serialize position
out.writeInt(lastPosition);
- lastPosition += type.serializer.serializedSize(component);
+ lastPosition += type.serializer.serializedSize(component, version);
}
// serialize components
for (MetadataComponent component : sortedComponents)
{
- component.getType().serializer.serialize(component, out);
+ component.getType().serializer.serialize(component, version, out);
}
}
@@ -153,7 +154,7 @@ public class MetadataSerializer implements IMetadataSerializer
try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(tmpDescriptor.filenameFor(Component.STATS))))
{
- serialize(currentComponents, out);
+ serialize(currentComponents, descriptor.version, out);
out.flush();
}
// we cant move a file on top of another file in windows:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index f2eb1af..3d48e34 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -43,7 +43,8 @@ public class StatsMetadata extends MetadataComponent
public final EstimatedHistogram estimatedRowSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
public final int maxLocalDeletionTime;
@@ -57,7 +58,8 @@ public class StatsMetadata extends MetadataComponent
public StatsMetadata(EstimatedHistogram estimatedRowSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
int maxLocalDeletionTime,
@@ -71,7 +73,8 @@ public class StatsMetadata extends MetadataComponent
{
this.estimatedRowSize = estimatedRowSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
this.maxLocalDeletionTime = maxLocalDeletionTime;
@@ -117,7 +120,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -134,7 +138,8 @@ public class StatsMetadata extends MetadataComponent
{
return new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
@@ -157,7 +162,8 @@ public class StatsMetadata extends MetadataComponent
return new EqualsBuilder()
.append(estimatedRowSize, that.estimatedRowSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
@@ -177,7 +183,8 @@ public class StatsMetadata extends MetadataComponent
return new HashCodeBuilder()
.append(estimatedRowSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
.append(maxLocalDeletionTime)
@@ -193,12 +200,12 @@ public class StatsMetadata extends MetadataComponent
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component) throws IOException
+ public int serializedSize(StatsMetadata component, Version version) throws IOException
{
int size = 0;
size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition, TypeSizes.NATIVE);
+ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
@@ -211,14 +218,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
size += 2 + columnName.remaining(); // with short length
size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
return size;
}
- public void serialize(StatsMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
{
EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
out.writeInt(component.maxLocalDeletionTime);
@@ -233,13 +242,16 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer columnName : component.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
out.writeBoolean(component.hasLegacyCounterShards);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
public StatsMetadata deserialize(Version version, DataInput in) throws IOException
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -264,9 +276,12 @@ public class StatsMetadata extends MetadataComponent
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
return new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index 603732b..4ca078b 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -71,12 +71,12 @@ public class ValidationMetadata extends MetadataComponent
public static class ValidationMetadataSerializer implements IMetadataComponentSerializer<ValidationMetadata>
{
- public int serializedSize(ValidationMetadata component) throws IOException
+ public int serializedSize(ValidationMetadata component, Version version) throws IOException
{
return TypeSizes.NATIVE.sizeof(component.partitioner) + 8;
}
- public void serialize(ValidationMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(ValidationMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeUTF(component.partitioner);
out.writeDouble(component.bloomFilterFPChance);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java b/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
deleted file mode 100644
index 1a15d6f..0000000
--- a/src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.util;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.utils.WrappedRunnable;
-
-public abstract class DiskAwareRunnable extends WrappedRunnable
-{
- protected Directories.DataDirectory getWriteDirectory(long writeSize)
- {
- Directories.DataDirectory directory = getDirectories().getWriteableLocation(writeSize);
- if (directory == null)
- throw new FSWriteError(new IOException("Insufficient disk space to write " + writeSize + " bytes"), "");
-
- return directory;
- }
-
- /**
- * Get sstable directories for the CF.
- * @return Directories instance for the CF.
- */
- protected abstract Directories getDirectories();
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index dd56b8b..6911ec6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -122,14 +122,11 @@ public class StreamReceiveTask extends StreamTask
for (SSTableWriter writer : task.sstables)
writer.abort();
task.sstables.clear();
- task.session.taskCompleted(task);
return;
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
File lockfiledir = cfs.directories.getWriteableLocationAsFile(task.sstables.size() * 256L);
- if (lockfiledir == null)
- throw new IOError(new IOException("All disks full"));
StreamLockfile lockfile = new StreamLockfile(lockfiledir, UUID.randomUUID());
lockfile.create(task.sstables);
List<SSTableReader> readers = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index 2665f40..8319014 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -70,7 +70,8 @@ public class SSTableMetadataViewer
out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
out.printf("SSTable Level: %d%n", stats.sstableLevel);
out.printf("Repaired at: %d%n", stats.repairedAt);
- out.println(stats.replayPosition);
+ out.printf("Minimum replay position: %s\n", stats.commitLogLowerBound);
+ out.printf("Maximum replay position: %s\n", stats.commitLogUpperBound);
out.println("Estimated tombstone drop times:");
for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index 1b4edee..f9b4156 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -452,7 +452,7 @@ public class CommitLogStressTest
int cells = 0;
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
if (desc.id < discardedPos.segment)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
index c377a21..0c46061 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTestReplayer.java
@@ -59,7 +59,7 @@ public class CommitLogTestReplayer extends CommitLogReplayer
}
@Override
- void replayMutation(byte[] inputBuffer, int size, final long entryLocation, final CommitLogDescriptor desc)
+ void replayMutation(byte[] inputBuffer, int size, final int entryLocation, final CommitLogDescriptor desc)
{
FastByteArrayInputStream bufIn = new FastByteArrayInputStream(inputBuffer, 0, size);
Mutation mutation;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index 93b88b4..adeb778 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.MockSchema;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
@@ -263,15 +264,15 @@ public class TrackerTest
Tracker tracker = cfs.getTracker();
tracker.subscribe(listener);
- Memtable prev1 = tracker.switchMemtable(true);
+ Memtable prev1 = tracker.switchMemtable(true, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write1 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier1 = cfs.keyspace.writeOrder.newBarrier();
- prev1.setDiscarding(barrier1, new AtomicReference<ReplayPosition>());
+ prev1.setDiscarding(barrier1, new AtomicReference<>(CommitLog.instance.getContext()));
barrier1.issue();
- Memtable prev2 = tracker.switchMemtable(false);
+ Memtable prev2 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
OpOrder.Group write2 = cfs.keyspace.writeOrder.getCurrent();
OpOrder.Barrier barrier2 = cfs.keyspace.writeOrder.newBarrier();
- prev2.setDiscarding(barrier2, new AtomicReference<ReplayPosition>());
+ prev2.setDiscarding(barrier2, new AtomicReference<>(CommitLog.instance.getContext()));
barrier2.issue();
Memtable cur = tracker.getView().getCurrentMemtable();
OpOrder.Group writecur = cfs.keyspace.writeOrder.getCurrent();
@@ -297,6 +298,9 @@ public class TrackerTest
SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
tracker.replaceFlushed(prev2, reader);
Assert.assertEquals(1, tracker.getView().sstables.size());
+ Assert.assertEquals(1, tracker.getView().premature.size());
+ tracker.permitCompactionOfFlushed(reader);
+ Assert.assertEquals(0, tracker.getView().premature.size());
Assert.assertEquals(1, listener.received.size());
Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
listener.received.clear();
@@ -307,16 +311,17 @@ public class TrackerTest
tracker = cfs.getTracker();
listener = new MockListener(false);
tracker.subscribe(listener);
- prev1 = tracker.switchMemtable(false);
+ prev1 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfs));
tracker.markFlushing(prev1);
reader = MockSchema.sstable(0, 10, true, cfs);
cfs.invalidate(false);
tracker.replaceFlushed(prev1, reader);
+ tracker.permitCompactionOfFlushed(reader);
Assert.assertEquals(0, tracker.getView().sstables.size());
Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
- Assert.assertEquals(reader, ((SSTableAddedNotification) listener.received.get(0)).added);
- Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+ Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(0)).removed.size());
+ Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(1)).deleting));
DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index 32a81e2..5706598 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -208,6 +208,6 @@ public class ViewTest
for (int i = 0 ; i < sstableCount ; i++)
sstables.add(MockSchema.sstable(i, cfs));
return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
- Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
+ Collections.<SSTableReader>emptySet(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 931422b..3bef89e 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@ -113,7 +113,7 @@ public class CompressedRandomAccessReaderTest
ChannelProxy channel = new ChannelProxy(f);
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
SequentialWriter writer = compressed
? new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector)
: SequentialWriter.open(f);
@@ -166,7 +166,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
@@ -251,7 +251,7 @@ public class CompressedRandomAccessReaderTest
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(), new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector))
{
writer.write(CONTENT.getBytes());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
index 27b866d..43c44fd 100644
--- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java
@@ -85,7 +85,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest
try
{
- MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance)).replayPosition(null);
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance));
byte[] dataPre = new byte[bytesToTest];
byte[] rawPost = new byte[bytesToTest];
try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", new CompressionParameters(compressor), sstableMetadataCollector);)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index eda4f17..19fa7c4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable.metadata;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.EnumSet;
@@ -27,12 +28,16 @@ import java.util.Set;
import com.google.common.collect.Sets;
import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataOutputStreamPlus;
import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus;
import org.apache.cassandra.io.util.RandomAccessReader;
@@ -45,18 +50,51 @@ public class MetadataSerializerTest
@Test
public void testSerialization() throws IOException
{
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
+
+ MetadataSerializer serializer = new MetadataSerializer();
+ File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
+
+ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
+ try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ {
+ Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+
+ for (MetadataType type : MetadataType.values())
+ {
+ assertEquals(originalMetadata.get(type), deserialized.get(type));
+ }
+ }
+ }
+
+ public File serialize(Map<MetadataType, MetadataComponent> metadata, MetadataSerializer serializer, Version version)
+ throws IOException, FileNotFoundException
+ {
+ // Serialize to tmp file
+ File statsFile = File.createTempFile(Component.STATS.name, null);
+ try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
+ {
+ serializer.serialize(metadata, version, out);
+ }
+ return statsFile;
+ }
+
+ public Map<MetadataType, MetadataComponent> constructMetadata()
+ {
EstimatedHistogram rowSizes = new EstimatedHistogram(new long[] { 1L, 2L },
new long[] { 3L, 4L, 5L });
EstimatedHistogram columnCounts = new EstimatedHistogram(new long[] { 6L, 7L },
new long[] { 8L, 9L, 10L });
- ReplayPosition rp = new ReplayPosition(11L, 12);
+ ReplayPosition start = new ReplayPosition(11L, 12);
+ ReplayPosition end = new ReplayPosition(15L, 9);
long minTimestamp = 2162517136L;
long maxTimestamp = 4162517136L;
MetadataCollector collector = new MetadataCollector(new SimpleDenseCellNameType(BytesType.instance))
.estimatedRowSize(rowSizes)
.estimatedColumnCount(columnCounts)
- .replayPosition(rp);
+ .commitLogLowerBound(start)
+ .commitLogUpperBound(end);
collector.updateMinTimestamp(minTimestamp);
collector.updateMaxTimestamp(maxTimestamp);
@@ -67,23 +105,35 @@ public class MetadataSerializerTest
String partitioner = RandomPartitioner.class.getCanonicalName();
double bfFpChance = 0.1;
Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0);
+ return originalMetadata;
+ }
+
+ @Test
+ public void testLaReadsLb() throws IOException
+ {
+ Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();
MetadataSerializer serializer = new MetadataSerializer();
- // Serialize to tmp file
- File statsFile = File.createTempFile(Component.STATS.name, null);
- try (DataOutputStreamPlus out = new BufferedDataOutputStreamPlus(new FileOutputStream(statsFile)))
- {
- serializer.serialize(originalMetadata, out);
- }
+ // Write metadata in two minor formats.
+ File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("lb"));
+ File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion("la"));
- Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
- try (RandomAccessReader in = RandomAccessReader.open(statsFile))
+ // Reading both as earlier version should yield identical results.
+ Descriptor desc = new Descriptor("la", statsFileLb.getParentFile(), "", "", 0, Descriptor.Type.FINAL, DatabaseDescriptor.getSSTableFormat());
+ try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
+ RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{
- Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLb = serializer.deserialize(desc, inLb, EnumSet.allOf(MetadataType.class));
+ Map<MetadataType, MetadataComponent> deserializedLa = serializer.deserialize(desc, inLa, EnumSet.allOf(MetadataType.class));
for (MetadataType type : MetadataType.values())
{
- assertEquals(originalMetadata.get(type), deserialized.get(type));
+ assertEquals(deserializedLa.get(type), deserializedLb.get(type));
+ if (!originalMetadata.get(type).equals(deserializedLb.get(type)))
+ {
+ // Currently only STATS can be different. Change if no longer the case
+ assertEquals(MetadataType.STATS, type);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
index 8409a26..1f66fb7 100644
--- a/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
+++ b/test/unit/org/apache/cassandra/utils/IntervalTreeTest.java
@@ -157,7 +157,7 @@ public class IntervalTreeTest
public String deserialize(DataInput in) throws IOException { return in.readUTF(); }
public long serializedSize(String v, TypeSizes s) { return v.length(); }
},
- (Constructor<Interval<Integer, String>>) (Object) Interval.class.getConstructor(Object.class, Object.class, Object.class)
+ (Constructor<Interval<Integer, String>>) (Constructor<?>) Interval.class.getConstructor(Object.class, Object.class, Object.class)
);
DataOutputBuffer out = new DataOutputBuffer();
[02/20] cassandra git commit: Fix commit log replay after
out-of-order flush completion.
Posted by sl...@apache.org.
Fix commit log replay after out-of-order flush completion.
Patch by Benedict Elliott Smith; reviewed by Branimir Lambov for
CASSANDRA-9669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/849a4386
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/849a4386
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/849a4386
Branch: refs/heads/cassandra-2.2
Commit: 849a438690aa97a361227781108cc90355dcbcd9
Parents: e614433
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Wed Sep 9 15:25:59 2015 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:17:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/db/ColumnFamilyStore.java | 258 +++++++++++--------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 218 +++++++---------
.../db/commitlog/CommitLogReplayer.java | 58 +++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++---
.../compaction/AbstractCompactionStrategy.java | 3 -
.../cassandra/db/compaction/Scrubber.java | 2 -
.../apache/cassandra/db/lifecycle/Tracker.java | 47 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 38 ++-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 10 +-
.../io/sstable/format/big/BigTableWriter.java | 2 +-
.../io/sstable/metadata/CompactionMetadata.java | 4 +-
.../metadata/IMetadataComponentSerializer.java | 4 +-
.../sstable/metadata/IMetadataSerializer.java | 3 +-
.../metadata/LegacyMetadataSerializer.java | 15 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/MetadataSerializer.java | 9 +-
.../io/sstable/metadata/StatsMetadata.java | 41 ++-
.../io/sstable/metadata/ValidationMetadata.java | 4 +-
.../cassandra/io/util/DiskAwareRunnable.java | 42 ---
.../cassandra/streaming/StreamReceiveTask.java | 3 -
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 19 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 6 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 74 +++++-
.../cassandra/utils/IntervalTreeTest.java | 2 +-
33 files changed, 600 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2bb3518..dfcad10 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,5 @@
+ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+
2.2.7
* Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 513a138..88e22c0 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -146,6 +146,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ @VisibleForTesting
+ public static volatile ColumnFamilyStore discardFlushResults;
+
public final Keyspace keyspace;
public final String name;
public final CFMetaData metadata;
@@ -867,14 +870,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param memtable
*/
- public Future<?> switchMemtableIfCurrent(Memtable memtable)
+ public ListenableFuture<ReplayPosition> switchMemtableIfCurrent(Memtable memtable)
{
synchronized (data)
{
if (data.getView().getCurrentMemtable() == memtable)
return switchMemtable();
}
- return Futures.immediateFuture(null);
+ return waitForFlushes();
}
/*
@@ -884,14 +887,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL
* marked clean up to the position owned by the Memtable.
*/
- public ListenableFuture<?> switchMemtable()
+ public ListenableFuture<ReplayPosition> switchMemtable()
{
synchronized (data)
{
logFlush();
Flush flush = new Flush(false);
flushExecutor.execute(flush);
- ListenableFutureTask<?> task = ListenableFutureTask.create(flush.postFlush, null);
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(flush.postFlush);
postFlushExecutor.submit(task);
return task;
}
@@ -926,77 +929,93 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
- public ListenableFuture<?> forceFlush()
+ /**
+ * Flush if there is unflushed data in the memtables
+ *
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ public ListenableFuture<ReplayPosition> forceFlush()
{
- return forceFlush(null);
+ Memtable current = data.getView().getCurrentMemtable();
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ if (!cfs.data.getView().getCurrentMemtable().isClean())
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
}
/**
* Flush if there is unflushed data that was written to the CommitLog before @param flushIfDirtyBefore
- * (inclusive). If @param flushIfDirtyBefore is null, flush if there is any unflushed data.
+ * (inclusive).
*
- * @return a Future such that when the future completes, all data inserted before forceFlush was called,
- * will be flushed.
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
*/
- public ListenableFuture<?> forceFlush(ReplayPosition flushIfDirtyBefore)
+ public ListenableFuture<ReplayPosition> forceFlush(ReplayPosition flushIfDirtyBefore)
{
- // we synchronize on the data tracker to ensure we don't race against other calls to switchMemtable(),
- // unnecessarily queueing memtables that are about to be made clean
- synchronized (data)
- {
- // during index build, 2ary index memtables can be dirty even if parent is not. if so,
- // we want to flush the 2ary index ones too.
- boolean clean = true;
- for (ColumnFamilyStore cfs : concatWithIndexes())
- clean &= cfs.data.getView().getCurrentMemtable().isCleanAfter(flushIfDirtyBefore);
+ // we don't loop through the remaining memtables since here we only care about commit log dirtiness
+ // and this does not vary between a table and its table-backed indexes
+ Memtable current = data.getView().getCurrentMemtable();
+ if (current.mayContainDataBefore(flushIfDirtyBefore))
+ return switchMemtableIfCurrent(current);
+ return waitForFlushes();
+ }
- if (clean)
+ /**
+ * @return a Future yielding the commit log position that can be guaranteed to have been successfully written
+ * to sstables for this table once the future completes
+ */
+ private ListenableFuture<ReplayPosition> waitForFlushes()
+ {
+ // we grab the current memtable; once any preceding memtables have flushed, we know its
+ // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable)
+ final Memtable current = data.getView().getCurrentMemtable();
+ ListenableFutureTask<ReplayPosition> task = ListenableFutureTask.create(new Callable<ReplayPosition>()
+ {
+ public ReplayPosition call()
{
- // We could have a memtable for this column family that is being
- // flushed. Make sure the future returned wait for that so callers can
- // assume that any data inserted prior to the call are fully flushed
- // when the future returns (see #5241).
- ListenableFutureTask<?> task = ListenableFutureTask.create(new Runnable()
- {
- public void run()
- {
- logger.trace("forceFlush requested but everything is clean in {}", name);
- }
- }, null);
- postFlushExecutor.execute(task);
- return task;
+ logger.debug("forceFlush requested but everything is clean in {}", name);
+ return current.getCommitLogLowerBound();
}
-
- return switchMemtable();
- }
+ });
+ postFlushExecutor.execute(task);
+ return task;
}
- public void forceBlockingFlush()
+ public ReplayPosition forceBlockingFlush()
{
- FBUtilities.waitOnFuture(forceFlush());
+ return FBUtilities.waitOnFuture(forceFlush());
}
/**
* Both synchronises custom secondary indexes and provides ordering guarantees for futures on switchMemtable/flush
* etc, which expect to be able to wait until the flush (and all prior flushes) requested have completed.
*/
- private final class PostFlush implements Runnable
+ private final class PostFlush implements Callable<ReplayPosition>
{
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
+ final List<SSTableReader> readers;
volatile FSWriteError flushFailure = null;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
+ List<Memtable> memtables, List<SSTableReader> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
+ this.memtables = memtables;
+ this.readers = readers;
}
- public void run()
+ public ReplayPosition call()
{
+ if (discardFlushResults == ColumnFamilyStore.this)
+ return commitLogUpperBound;
+
writeBarrier.await();
/**
@@ -1018,7 +1037,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
try
{
- // we wait on the latch for the lastReplayPosition to be set, and so that waiters
+ // we wait on the latch for the commitLogUpperBound to be set, and so that waiters
// on this task can rely on all prior flushes being complete
latch.await();
}
@@ -1027,18 +1046,23 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
+ SSTableReader reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
+ }
}
-
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
+ return commitLogUpperBound;
}
}
@@ -1053,7 +1077,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
+ final List<SSTableReader> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@ -1069,43 +1094,33 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* that all write operations register themselves with, and assigning this barrier to the memtables,
* after which we *.issue()* the barrier. This barrier is used to direct write operations started prior
* to the barrier.issue() into the memtable we have switched out, and any started after to its replacement.
- * In doing so it also tells the write operations to update the lastReplayPosition of the memtable, so
+ * In doing so it also tells the write operations to update the commitLogUpperBound of the memtable, so
* that we know the CL position we are dirty to, which can be marked clean when we complete.
*/
writeBarrier = keyspace.writeOrder.newBarrier();
- memtables = new ArrayList<>();
// submit flushes for the memtable for any indexed sub-cfses, and our own
- AtomicReference<ReplayPosition> lastReplayPositionHolder = new AtomicReference<>();
+ AtomicReference<ReplayPosition> commitLogUpperBound = new AtomicReference<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
// switch all memtables, regardless of their dirty status, setting the barrier
// so that we can reach a coordinated decision about cleanliness once they
// are no longer possible to be modified
- Memtable mt = cfs.data.switchMemtable(truncate);
- mt.setDiscarding(writeBarrier, lastReplayPositionHolder);
- memtables.add(mt);
+ Memtable newMemtable = new Memtable(commitLogUpperBound, cfs);
+ Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable);
+ oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound);
+ memtables.add(oldMemtable);
}
- // we now attempt to define the lastReplayPosition; we do this by grabbing the current limit from the CL
- // and attempting to set the holder to this value. at the same time all writes to the memtables are
- // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
- // so that we know all operations prior to the position have not reached it yet
- ReplayPosition lastReplayPosition;
- while (true)
- {
- lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
- ReplayPosition currentLast = lastReplayPositionHolder.get();
- if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
- && lastReplayPositionHolder.compareAndSet(currentLast, lastReplayPosition))
- break;
- }
+ // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log
+ // records owned by this memtable
+ setCommitLogUpperBound(commitLogUpperBound);
// we then issue the barrier; this lets us wait for all operations started prior to the barrier to complete;
- // since this happens after wiring up the lastReplayPosition, we also know all operations with earlier
+ // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier
// replay positions have also completed, i.e. the memtables are done and ready to flush
writeBarrier.issue();
- postFlush = new PostFlush(!truncate, writeBarrier, lastReplayPosition);
+ postFlush = new PostFlush(!truncate, writeBarrier, commitLogUpperBound.get(), memtables, readers);
}
public void run()
@@ -1124,7 +1139,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
+ memtable.cfs.data.replaceFlushed(memtable, null);
+ memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
reclaim(memtable);
iter.remove();
}
@@ -1143,8 +1159,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (Memtable memtable : memtables)
{
// flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
+ SSTableReader reader = memtable.flush();
+ memtable.cfs.data.replaceFlushed(memtable, reader);
reclaim(memtable);
+ readers.add(reader);
}
}
catch (FSWriteError e)
@@ -1174,6 +1192,38 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ // atomically set the upper bound for the commit log
+ private static void setCommitLogUpperBound(AtomicReference<ReplayPosition> commitLogUpperBound)
+ {
+ // we attempt to set the holder to the current commit log context. at the same time all writes to the memtables are
+ // also maintaining this value, so if somebody sneaks ahead of us somehow (should be rare) we simply retry,
+ // so that we know all operations prior to the position have not reached it yet
+ ReplayPosition lastReplayPosition;
+ while (true)
+ {
+ lastReplayPosition = new Memtable.LastReplayPosition(CommitLog.instance.getContext());
+ ReplayPosition currentLast = commitLogUpperBound.get();
+ if ((currentLast == null || currentLast.compareTo(lastReplayPosition) <= 0)
+ && commitLogUpperBound.compareAndSet(currentLast, lastReplayPosition))
+ break;
+ }
+ }
+
+ @VisibleForTesting
+ // this method should ONLY be used for testing commit log behaviour; it discards the current memtable
+ // contents without marking the commit log clean, and prevents any proceeding flushes from marking
+ // the commit log as done, however they *will* terminate (unlike under typical failures) to ensure progress is made
+ public void simulateFailedFlush()
+ {
+ discardFlushResults = this;
+ data.markFlushing(data.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), this)));
+ }
+
+ public void resumeFlushing()
+ {
+ discardFlushResults = null;
+ }
+
/**
* Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
* queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
@@ -1589,11 +1639,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
maybeFail(data.dropSSTables(Predicates.in(sstables), compactionType, null));
}
- void replaceFlushed(Memtable memtable, SSTableReader sstable)
- {
- compactionStrategyWrapper.replaceFlushed(memtable, sstable);
- }
-
public boolean isValid()
{
return valid;
@@ -1615,6 +1660,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return data.getSSTables();
}
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
public Set<SSTableReader> getUncompactingSSTables()
{
return data.getUncompacting();
@@ -2670,6 +2720,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Void call()
{
cfs.data.reset();
+ cfs.getCompactionStrategy().shutdown();
+ cfs.getCompactionStrategy().startup();
return null;
}
}, true);
@@ -2707,39 +2759,42 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- // just nuke the memtable data w/o writing to disk first
- synchronized (data)
+ if (keyspace.getMetadata().durableWrites || takeSnapshot)
{
+ replayAfter = forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
+ replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
}
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
if (takeSnapshot)
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
for (SecondaryIndex index : indexManager.getIndexes())
index.truncateBlocking(truncatedAt);
@@ -2807,7 +2862,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public LifecycleTransaction call() throws Exception
{
assert data.getCompacting().isEmpty() : data.getCompacting();
- Collection<SSTableReader> sstables = Lists.newArrayList(AbstractCompactionStrategy.filterSuspectSSTables(getSSTables()));
+ Iterable<SSTableReader> sstables = getPermittedToCompactSSTables();
+ sstables = AbstractCompactionStrategy.filterSuspectSSTables(sstables);
+ sstables = ImmutableList.copyOf(sstables);
LifecycleTransaction modifier = data.tryModify(sstables, operationType);
assert modifier != null: "something marked things compacting while compactions are disabled";
return modifier;
@@ -3025,10 +3082,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
*
* @param truncatedAt The timestamp of the truncation
* (all SSTables before that timestamp are going be marked as compacted)
- *
- * @return the most recent replay position of the truncated data
*/
- public ReplayPosition discardSSTables(long truncatedAt)
+ public void discardSSTables(long truncatedAt)
{
assert data.getCompacting().isEmpty() : data.getCompacting();
@@ -3040,11 +3095,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
truncatedSSTables.add(sstable);
}
- if (truncatedSSTables.isEmpty())
- return ReplayPosition.NONE;
-
- markObsolete(truncatedSSTables, OperationType.UNKNOWN);
- return ReplayPosition.getReplayPosition(truncatedSSTables);
+ if (!truncatedSSTables.isEmpty())
+ markObsolete(truncatedSSTables, OperationType.UNKNOWN);
}
public double getDroppableTombstoneRatio()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index c2901a4..2b3662f 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -346,7 +346,7 @@ public class Directories
if (candidates.isEmpty())
if (tooBig)
- return null;
+ throw new RuntimeException("Insufficient disk space to write " + writeSize + " bytes");
else
throw new FSWriteError(new IOException("All configured data directories have been blacklisted as unwritable for erroring out"), "");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index fb4da72..b4ada09 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -29,21 +29,20 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.concurrent.OpOrder;
@@ -62,14 +61,17 @@ public class Memtable implements Comparable<Memtable>
// the write barrier for directing writes to this memtable during a switch
private volatile OpOrder.Barrier writeBarrier;
- // the last ReplayPosition owned by this Memtable; all ReplayPositions lower are owned by this or an earlier Memtable
- private volatile AtomicReference<ReplayPosition> lastReplayPosition;
- // the "first" ReplayPosition owned by this Memtable; this is inaccurate, and only used as a convenience to prevent CLSM flushing wantonly
- private final ReplayPosition minReplayPosition = CommitLog.instance.getContext();
+ // the precise upper bound of ReplayPosition owned by this memtable
+ private volatile AtomicReference<ReplayPosition> commitLogUpperBound;
+ // the precise lower bound of ReplayPosition owned by this memtable; equal to its predecessor's commitLogUpperBound
+ private AtomicReference<ReplayPosition> commitLogLowerBound;
+ // the approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor
+ // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound
+ private final ReplayPosition approximateCommitLogLowerBound = CommitLog.instance.getContext();
public int compareTo(Memtable that)
{
- return this.minReplayPosition.compareTo(that.minReplayPosition);
+ return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound);
}
public static final class LastReplayPosition extends ReplayPosition
@@ -84,7 +86,6 @@ public class Memtable implements Comparable<Memtable>
// actually only store DecoratedKey.
private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@ -95,9 +96,10 @@ public class Memtable implements Comparable<Memtable>
// memtable was created with the new or old comparator.
public final CellNameType initialComparator;
- public Memtable(ColumnFamilyStore cfs)
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@ -131,7 +133,7 @@ public class Memtable implements Comparable<Memtable>
public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference<ReplayPosition> lastReplayPosition)
{
assert this.writeBarrier == null;
- this.lastReplayPosition = lastReplayPosition;
+ this.commitLogUpperBound = lastReplayPosition;
this.writeBarrier = writeBarrier;
allocator.setDiscarding();
}
@@ -161,16 +163,21 @@ public class Memtable implements Comparable<Memtable>
// its current value and ours; if it HAS been finalised, we simply accept its judgement
// this permits us to coordinate a safe boundary, as the boundary choice is made
// atomically wrt our max() maintenance, so an operation cannot sneak into the past
- ReplayPosition currentLast = lastReplayPosition.get();
+ ReplayPosition currentLast = commitLogUpperBound.get();
if (currentLast instanceof LastReplayPosition)
return currentLast.compareTo(replayPosition) >= 0;
if (currentLast != null && currentLast.compareTo(replayPosition) >= 0)
return true;
- if (lastReplayPosition.compareAndSet(currentLast, replayPosition))
+ if (commitLogUpperBound.compareAndSet(currentLast, replayPosition))
return true;
}
}
+ public ReplayPosition getCommitLogLowerBound()
+ {
+ return commitLogLowerBound.get();
+ }
+
public boolean isLive()
{
return allocator.isLive();
@@ -181,9 +188,9 @@ public class Memtable implements Comparable<Memtable>
return rows.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@ -252,11 +259,6 @@ public class Memtable implements Comparable<Memtable>
return rows.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@ -285,20 +287,21 @@ public class Memtable implements Comparable<Memtable>
public Map.Entry<DecoratedKey, ColumnFamily> next()
{
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entry = iter.next();
+ Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
// Actual stored key should be true DecoratedKey
- assert entry.getKey() instanceof DecoratedKey;
+ assert entryRowPosition.getKey() instanceof DecoratedKey;
+ @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
+ Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
if (MEMORY_POOL.needToCopyOnHeap())
{
- DecoratedKey key = (DecoratedKey) entry.getKey();
+ DecoratedKey key = entry.getKey();
key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
}
// Store the reference to the current entry so that remove() can update the current size.
currentEntry = entry;
- // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- return (Map.Entry<DecoratedKey, ColumnFamily>) entry;
+ return entry;
}
public void remove()
@@ -315,9 +318,13 @@ public class Memtable implements Comparable<Memtable>
return rows.get(key);
}
- public long creationTime()
+ public SSTableReader flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
+ Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
+ File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@ -325,115 +332,88 @@ public class Memtable implements Comparable<Memtable>
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- FlushRunnable(ReplayPosition context)
- {
- this.context = context;
-
- long keySize = 0;
- for (RowPosition key : rows.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
- }
-
- protected void runMayThrow() throws Exception
+ long keySize = 0;
+ for (RowPosition key : rows.keySet())
{
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- SSTableReader sstable = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstable);
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected Directories getDirectories()
- {
- return cfs.directories;
- }
+ private SSTableReader writeSortedContents(File sstableDirectory)
+ {
+ logger.info("Writing {}", Memtable.this.toString());
- private SSTableReader writeSortedContents(ReplayPosition context, File sstableDirectory)
+ SSTableReader ssTable;
+ // errors when creating the writer that may leave empty temp files.
+ try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
+ for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
+ AtomicBTreeColumns cf = entry.getValue();
+
+ if (cf.isMarkedForDelete() && cf.hasColumns())
{
- AtomicBTreeColumns cf = entry.getValue();
-
- if (cf.isMarkedForDelete() && cf.hasColumns())
- {
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
- continue;
- }
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ // When every node is up, there's no reason to write batchlog data out to sstables
+ // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
+ // and BL data is strictly local, so we don't need to preserve tombstones for repair.
+ // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
+ // See CASSANDRA-4667.
+ if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
+ continue;
}
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
- context));
+ if (trackContention && cf.usePessimisticLocking())
+ heavilyContendedRowCount++;
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTable = null;
- }
+ if (!cf.isEmpty())
+ writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
+ FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
+ commitLogUpperBound));
- return ssTable;
+ // temp sstables should contain non-repaired data.
+ ssTable = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
+ ssTable = null;
}
- }
- public SSTableWriter createFlushWriter(String filename)
- {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
+ if (heavilyContendedRowCount > 0)
+ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ return ssTable;
}
}
+ private SSTableWriter createFlushWriter(String filename)
+ {
+ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
+ .commitLogLowerBound(commitLogLowerBound.get())
+ .commitLogUpperBound(commitLogUpperBound.get());
+ return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ }
+
private static int estimateRowOverhead(final int count)
{
// calculate row overhead
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 98fb556..a58aeb4 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@ -33,7 +33,6 @@ import com.google.common.base.Predicate;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +69,7 @@ public class CommitLogReplayer
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
private final ICRC32 checksum;
private byte[] buffer;
@@ -79,7 +78,7 @@ public class CommitLogReplayer
private final ReplayFilter replayFilter;
private final CommitLogArchiver archiver;
- CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition> cfPositions, ReplayFilter replayFilter)
+ CommitLogReplayer(CommitLog commitLog, ReplayPosition globalPosition, Map<UUID, ReplayPosition.ReplayFilter> cfPersisted, ReplayFilter replayFilter)
{
this.keyspacesRecovered = new NonBlockingHashSet<Keyspace>();
this.futures = new ArrayList<Future<?>>();
@@ -89,7 +88,7 @@ public class CommitLogReplayer
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
this.checksum = CRC32Factory.instance.create();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@ -98,17 +97,12 @@ public class CommitLogReplayer
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
-
- // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@ -125,19 +119,21 @@ public class CommitLogReplayer
cfs.metadata.ksName,
cfs.metadata.cfName);
SystemKeyspace.removeTruncationRecord(cfs.metadata.cfId);
+ truncatedAt = null;
}
}
- else
- {
- rp = replayPositionOrdering.max(Arrays.asList(rp, truncatedAt));
- }
}
- cfPositions.put(cfs.metadata.cfId, rp);
+ ReplayPosition.ReplayFilter filter = new ReplayPosition.ReplayFilter(cfs.getSSTables(), truncatedAt);
+ if (!filter.isEmpty())
+ cfPersisted.put(cfs.metadata.cfId, filter);
+ else
+ globalPosition = ReplayPosition.NONE; // if we have no ranges for this CF, we must replay everything and filter
}
- ReplayPosition globalPosition = replayPositionOrdering.min(cfPositions.values());
- logger.trace("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPositions));
- return new CommitLogReplayer(commitLog, globalPosition, cfPositions, replayFilter);
+ if (globalPosition == null)
+ globalPosition = ReplayPosition.firstNotCovered(cfPersisted.values());
+ logger.debug("Global replay position is {} from columnfamilies {}", globalPosition, FBUtilities.toString(cfPersisted));
+ return new CommitLogReplayer(commitLog, globalPosition, cfPersisted, replayFilter);
}
public void recover(File[] clogs) throws IOException
@@ -273,6 +269,18 @@ public class CommitLogReplayer
}
}
+ /**
+ * consult the known-persisted ranges for our sstables;
+ * if the position is covered by one of them it does not need to be replayed
+ *
+ * @return true iff replay is necessary
+ */
+ private boolean shouldReplay(UUID cfId, ReplayPosition position)
+ {
+ ReplayPosition.ReplayFilter filter = cfPersisted.get(cfId);
+ return filter == null || filter.shouldReplay(position);
+ }
+
@SuppressWarnings("resource")
public void recover(File file, boolean tolerateTruncation) throws IOException
{
@@ -495,7 +503,7 @@ public class CommitLogReplayer
mutationStart, errorContext);
continue;
}
- replayMutation(buffer, serializedSize, reader.getFilePointer(), desc);
+ replayMutation(buffer, serializedSize, (int) reader.getFilePointer(), desc);
}
return true;
}
@@ -504,7 +512,7 @@ public class CommitLogReplayer
* Deserializes and replays a commit log entry.
*/
void replayMutation(byte[] inputBuffer, int size,
- final long entryLocation, final CommitLogDescriptor desc) throws IOException
+ final int entryLocation, final CommitLogDescriptor desc) throws IOException
{
final Mutation mutation;
@@ -577,11 +585,7 @@ public class CommitLogReplayer
if (Schema.instance.getCF(columnFamily.id()) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(columnFamily.id());
-
- // replay if current segment is newer than last flushed one or,
- // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
+ if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index ca1969f..17802ad 100644
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@ -19,10 +19,10 @@ package org.apache.cassandra.db.commitlog;
import java.io.DataInput;
import java.io.IOException;
-import java.util.Comparator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
@@ -35,46 +35,78 @@ public class ReplayPosition implements Comparable<ReplayPosition>
public static final ReplayPositionSerializer serializer = new ReplayPositionSerializer();
// NONE is used for SSTables that are streamed from other nodes and thus have no relationship
- // with our local commitlog. The values satisfy the critera that
+ // with our local commitlog. The values satisfy the criteria that
// - no real commitlog segment will have the given id
// - it will sort before any real replayposition, so it will be effectively ignored by getReplayPosition
public static final ReplayPosition NONE = new ReplayPosition(-1, 0);
+ public final long segment;
+ public final int position;
+
/**
- * Convenience method to compute the replay position for a group of SSTables.
- * @param sstables
- * @return the most recent (highest) replay position
+ * A filter of known safe-to-discard commit log replay positions, based on
+ * the range covered by on disk sstables and those prior to the most recent truncation record
*/
- public static ReplayPosition getReplayPosition(Iterable<? extends SSTableReader> sstables)
+ public static class ReplayFilter
{
- if (Iterables.isEmpty(sstables))
- return NONE;
-
- Function<SSTableReader, ReplayPosition> f = new Function<SSTableReader, ReplayPosition>()
+ final NavigableMap<ReplayPosition, ReplayPosition> persisted = new TreeMap<>();
+ public ReplayFilter(Iterable<SSTableReader> onDisk, ReplayPosition truncatedAt)
{
- public ReplayPosition apply(SSTableReader sstable)
+ for (SSTableReader reader : onDisk)
{
- return sstable.getReplayPosition();
+ ReplayPosition start = reader.getSSTableMetadata().commitLogLowerBound;
+ ReplayPosition end = reader.getSSTableMetadata().commitLogUpperBound;
+ add(persisted, start, end);
}
- };
- Ordering<ReplayPosition> ordering = Ordering.from(ReplayPosition.comparator);
- return ordering.max(Iterables.transform(sstables, f));
- }
+ if (truncatedAt != null)
+ add(persisted, ReplayPosition.NONE, truncatedAt);
+ }
+ private static void add(NavigableMap<ReplayPosition, ReplayPosition> ranges, ReplayPosition start, ReplayPosition end)
+ {
+ // extend ourselves to cover any ranges we overlap
+ // record directly preceding our end may extend past us, so take the max of our end and its
+ Map.Entry<ReplayPosition, ReplayPosition> extend = ranges.floorEntry(end);
+ if (extend != null && extend.getValue().compareTo(end) > 0)
+ end = extend.getValue();
+
+ // record directly preceding our start may extend into us; if it does, we take it as our start
+ extend = ranges.lowerEntry(start);
+ if (extend != null && extend.getValue().compareTo(start) >= 0)
+ start = extend.getKey();
+
+ ranges.subMap(start, end).clear();
+ ranges.put(start, end);
+ }
- public final long segment;
- public final int position;
+ public boolean shouldReplay(ReplayPosition position)
+ {
+ // replay ranges are start exclusive, end inclusive
+ Map.Entry<ReplayPosition, ReplayPosition> range = persisted.lowerEntry(position);
+ return range == null || position.compareTo(range.getValue()) > 0;
+ }
- public static final Comparator<ReplayPosition> comparator = new Comparator<ReplayPosition>()
- {
- public int compare(ReplayPosition o1, ReplayPosition o2)
+ public boolean isEmpty()
{
- if (o1.segment != o2.segment)
- return Long.valueOf(o1.segment).compareTo(o2.segment);
+ return persisted.isEmpty();
+ }
+ }
- return Integer.valueOf(o1.position).compareTo(o2.position);
+ public static ReplayPosition firstNotCovered(Iterable<ReplayFilter> ranges)
+ {
+ ReplayPosition min = null;
+ for (ReplayFilter map : ranges)
+ {
+ ReplayPosition first = map.persisted.firstEntry().getValue();
+ if (min == null)
+ min = first;
+ else
+ min = Ordering.natural().min(min, first);
}
- };
+ if (min == null)
+ return NONE;
+ return min;
+ }
public ReplayPosition(long segment, int position)
{
@@ -83,9 +115,12 @@ public class ReplayPosition implements Comparable<ReplayPosition>
this.position = position;
}
- public int compareTo(ReplayPosition other)
+ public int compareTo(ReplayPosition that)
{
- return comparator.compare(this, other);
+ if (this.segment != that.segment)
+ return Long.compare(this.segment, that.segment);
+
+ return Integer.compare(this.position, that.position);
}
@Override
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 02497df..575aa51 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -232,9 +232,6 @@ public abstract class AbstractCompactionStrategy
*/
public void replaceFlushed(Memtable memtable, SSTableReader sstable)
{
- cfs.getTracker().replaceFlushed(memtable, sstable);
- if (sstable != null)
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index e9137e2..d1fe702 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -98,8 +98,6 @@ public class Scrubber implements Closeable
// Calculate the expected compacted filesize
this.destination = cfs.directories.getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(toScrub, OperationType.SCRUB));
- if (destination == null)
- throw new IOException("disk full");
// If we run scrub offline, we should never purge tombstone, as we cannot know if other sstable have data that the tombstone deletes.
this.controller = transaction.isOffline()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index a074216..5d5701f 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -31,11 +31,13 @@ import com.google.common.collect.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
@@ -45,6 +47,8 @@ import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.OpOrder;
import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.in;
+import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.Iterables.filter;
import static java.util.Collections.singleton;
@@ -195,10 +199,12 @@ public class Tracker
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@ -294,9 +300,8 @@ public class Tracker
*
* @return the previously active memtable
*/
- public Memtable switchMemtable(boolean truncating)
+ public Memtable switchMemtable(boolean truncating, Memtable newMemtable)
{
- Memtable newMemtable = new Memtable(cfstore);
Pair<View, View> result = apply(View.switchMemtable(newMemtable));
if (truncating)
notifyRenewed(newMemtable);
@@ -328,15 +333,35 @@ public class Tracker
Throwable fail;
fail = updateSizeTracking(emptySet(), singleton(sstable), null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstable, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
+ public void permitCompactionOfFlushed(SSTableReader sstable)
+ {
+ if (sstable == null)
+ return;
+
+ apply(View.permitCompactionOfFlushed(sstable));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
+ notifyAdded(sstable);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
+
// MISCELLANEOUS public utility calls
@@ -346,6 +371,12 @@ public class Tracker
return view.get().sstables;
}
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java
index 73ba131..fba1627 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.db.lifecycle;
import java.util.*;
+import javax.annotation.Nullable;
+
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
@@ -66,6 +68,7 @@ public class View
public final List<Memtable> flushingMemtables;
public final Set<SSTableReader> compacting;
public final Set<SSTableReader> sstables;
+ public final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
@@ -74,7 +77,7 @@ public class View
public final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, SSTableIntervalTree intervalTree)
+ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@ -88,6 +91,7 @@ public class View
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
this.compacting = compacting;
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@ -155,7 +159,7 @@ public class View
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
replace(view.compacting, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@ -169,7 +173,7 @@ public class View
public boolean apply(View view)
{
for (SSTableReader reader : readers)
- if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted())
+ if (view.compacting.contains(reader) || view.sstablesMap.get(reader) != reader || reader.isMarkedCompacted() || view.premature.contains(reader))
return false;
return true;
}
@@ -186,7 +190,7 @@ public class View
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting,
+ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@ -201,7 +205,7 @@ public class View
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -220,7 +224,7 @@ public class View
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.intervalTree);
+ return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
}
};
}
@@ -237,15 +241,33 @@ public class View
if (flushed == null)
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compacting, view.intervalTree);
+ view.compacting, view.premature, view.intervalTree);
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compacting,
+ Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
+ Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
+ return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
+ static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
+ {
+ return new Function<View, View>()
+ {
+
+ @Nullable
+ public View apply(View view)
+ {
+ Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
+ Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
+ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e81e4e9..c303975 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1864,11 +1864,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstableMetadata.compressionRatio;
}
- public ReplayPosition getReplayPosition()
- {
- return sstableMetadata.replayPosition;
- }
-
public long getMinTimestamp()
{
return sstableMetadata.minTimestamp;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index faaa89e..41a83e1 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -52,6 +52,8 @@ public abstract class Version
public abstract boolean hasNewFileName();
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index a1e32cf..9244bbb 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -125,7 +125,7 @@ public class BigFormat implements SSTableFormat
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "la";
+ public static final String current_version = "lb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@ -135,6 +135,7 @@ public class BigFormat implements SSTableFormat
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@ -143,6 +144,7 @@ public class BigFormat implements SSTableFormat
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ private final boolean hasCommitLogLowerBound;
public BigVersion(String version)
{
@@ -155,6 +157,7 @@ public class BigFormat implements SSTableFormat
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("la") >= 0;
+ hasCommitLogLowerBound = version.compareTo("lb") >= 0;
}
@Override
@@ -199,6 +202,11 @@ public class BigFormat implements SSTableFormat
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
public boolean isCompatible()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 505bac0..3a01f87 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -432,7 +432,7 @@ public class BigTableWriter extends SSTableWriter
File file = new File(desc.filenameFor(Component.STATS));
try (SequentialWriter out = SequentialWriter.open(file);)
{
- desc.getMetadataSerializer().serialize(components, out.stream);
+ desc.getMetadataSerializer().serialize(components, desc.version, out.stream);
out.setDescriptor(desc).finish();
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index ed1f327..c8e6ee8 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -72,7 +72,7 @@ public class CompactionMetadata extends MetadataComponent
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component) throws IOException
+ public int serializedSize(CompactionMetadata component, Version version) throws IOException
{
int size = 0;
size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
@@ -83,7 +83,7 @@ public class CompactionMetadata extends MetadataComponent
return size;
}
- public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
+ public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
{
out.writeInt(component.ancestors.size());
for (int g : component.ancestors)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
index dc8fbdf..e3d867f 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
@@ -35,7 +35,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @return serialized size of this component
* @throws IOException
*/
- int serializedSize(T component) throws IOException;
+ int serializedSize(T component, Version version) throws IOException;
/**
* Serialize metadata component to given output.
@@ -45,7 +45,7 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @param out serialize destination
* @throws IOException
*/
- void serialize(T component, DataOutputPlus out) throws IOException;
+ void serialize(T component, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize metadata component from given input.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
index df577df..a7d23f4 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
@@ -22,6 +22,7 @@ import java.util.EnumSet;
import java.util.Map;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -37,7 +38,7 @@ public interface IMetadataSerializer
* @param out
* @throws IOException
*/
- void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException;
+ void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException;
/**
* Deserialize specified metadata components from given descriptor.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 4bd060e..bfeb930 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -26,6 +26,7 @@ import com.google.common.collect.Maps;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -42,7 +43,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
* Legacy serialization is only used for SSTable level reset.
*/
@Override
- public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out) throws IOException
+ public void serialize(Map<MetadataType, MetadataComponent> components, Version version, DataOutputPlus out) throws IOException
{
ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
@@ -52,7 +53,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
@@ -70,6 +71,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
out.writeInt(stats.maxColumnNames.size());
for (ByteBuffer columnName : stats.maxColumnNames)
ByteBufferUtil.writeWithShortLength(columnName, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@ -91,7 +94,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@ -116,6 +120,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
@@ -124,7 +130,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
components.put(MetadataType.STATS,
new StatsMetadata(rowSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/849a4386/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 5962a46..579ff7a 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -28,6 +28,7 @@ import java.util.Set;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -69,6 +70,7 @@ public class MetadataCollector
return new StatsMetadata(defaultRowSizeHistogram(),
defaultColumnCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@ -83,7 +85,8 @@ public class MetadataCollector
protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
protected long minTimestamp = Long.MAX_VALUE;
protected long maxTimestamp = Long.MIN_VALUE;
protected int maxLocalDeletionTime = Integer.MIN_VALUE;
@@ -113,7 +116,23 @@ public class MetadataCollector
{
this(columnNameComparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
// Get the max timestamp of the precompacted sstables
// and adds generation of live ancestors
@@ -199,9 +218,15 @@ public class MetadataCollector
return this;
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@ -257,7 +282,8 @@ public class MetadataCollector
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
maxLocalDeletionTime,
[14/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
Merge commit '849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/78a3d2bb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/78a3d2bb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/78a3d2bb
Branch: refs/heads/cassandra-3.0
Commit: 78a3d2bba95b9efcda152a157f822f4970f22636
Parents: e9657a4 849a438
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:17:51 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:19:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 282 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 243 ++++++++--------
.../db/commitlog/CommitLogReplayer.java | 56 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 37 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 23 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
92 files changed, 715 insertions(+), 381 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 306f46f,dfcad10..677ea11
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,7 +1,31 @@@
- * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+3.0.7
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
+ * Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395)
-2.2.7
- * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+3.0.6
+ * Disallow creating view with a static column (CASSANDRA-11602)
+ * Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593)
+ * Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618)
+ * Fix queries with filtering on counter columns (CASSANDRA-11629)
+ * Improve tombstone printing in sstabledump (CASSANDRA-11655)
+ * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669)
+ * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600)
+ * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654)
+ * Ignore all LocalStrategy keyspaces for streaming and other related
+ operations (CASSANDRA-11627)
+ * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523)
+ * Only open one sstable scanner per sstable (CASSANDRA-11412)
+ * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410)
+ * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485)
+ * LogAwareFileLister should only use OLD sstable files in current folder to determine disk consistency (CASSANDRA-11470)
+ * Notify indexers of expired rows during compaction (CASSANDRA-11329)
+ * Properly respond with ProtocolError when a v1/v2 native protocol
+ header is received (CASSANDRA-11464)
+ * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120)
+Merged from 2.2:
++ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
* Exit JVM if JMX server fails to startup (CASSANDRA-11540)
* Produce a heap dump when exiting on OOM (CASSANDRA-9861)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 5b8de8f,88e22c0..e9a2938
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -892,10 -996,13 +903,13 @@@ public class ColumnFamilyStore implemen
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ volatile FSWriteError flushFailure = null;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
- final List<SSTableReader> readers;
- volatile FSWriteError flushFailure = null;
++ final List<Collection<SSTableReader>> readers;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
- List<Memtable> memtables, List<SSTableReader> readers)
++ List<Memtable> memtables, List<Collection<SSTableReader>> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
@@@ -927,18 -1046,23 +946,25 @@@
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
- SSTableReader reader = readers.get(i);
++ Collection<SSTableReader> reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, reader);
+ }
}
+
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
++
+ return commitLogUpperBound;
}
}
@@@ -953,7 -1077,8 +979,8 @@@
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
- final List<SSTableReader> readers = new ArrayList<>();
++ final List<Collection<SSTableReader>> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@@ -1024,27 -1139,30 +1041,23 @@@
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
- memtable.cfs.data.replaceFlushed(memtable, null);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
++ memtable.cfs.data.replaceFlushed(memtable, Collections.emptyList());
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, Collections.emptyList());
reclaim(memtable);
iter.remove();
}
}
-- if (memtables.isEmpty())
-- {
-- postFlush.latch.countDown();
-- return;
-- }
--
metric.memtableSwitchCount.inc();
try
{
for (Memtable memtable : memtables)
{
-- // flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
- SSTableReader reader = memtable.flush();
- memtable.cfs.data.replaceFlushed(memtable, reader);
++ Collection<SSTableReader> readers = memtable.flush();
++ memtable.cfs.data.replaceFlushed(memtable, readers);
reclaim(memtable);
- readers.add(reader);
++ this.readers.add(readers);
}
}
catch (FSWriteError e)
@@@ -1406,22 -1655,172 +1451,32 @@@
return data;
}
+ public Collection<SSTableReader> getSSTables()
+ {
+ return data.getSSTables();
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
- public Set<SSTableReader> getUncompactingSSTables()
+ public Set<SSTableReader> getLiveSSTables()
{
- return data.getUncompacting();
+ return data.getView().liveSSTables();
}
- public ColumnFamily getColumnFamily(DecoratedKey key,
- Composite start,
- Composite finish,
- boolean reversed,
- int limit,
- long timestamp)
+ public Iterable<SSTableReader> getSSTables(SSTableSet sstableSet)
{
- return getColumnFamily(QueryFilter.getSliceFilter(key, name, start, finish, reversed, limit, timestamp));
+ return data.getView().sstables(sstableSet);
}
- /**
- * Fetch the row and columns given by filter.key if it is in the cache; if not, read it from disk and cache it
- *
- * If row is cached, and the filter given is within its bounds, we return from cache, otherwise from disk
- *
- * If row is not cached, we figure out what filter is "biggest", read that from disk, then
- * filter the result and either cache that or return it.
- *
- * @param cfId the column family to read the row from
- * @param filter the columns being queried.
- * @return the requested data for the filter provided
- */
- private ColumnFamily getThroughCache(UUID cfId, QueryFilter filter)
+ public Iterable<SSTableReader> getUncompactingSSTables()
{
- assert isRowCacheEnabled()
- : String.format("Row cache is not enabled on table [" + name + "]");
-
- RowCacheKey key = new RowCacheKey(metadata.ksAndCFName, filter.key);
-
- // attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
- // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
- // TODO: don't evict entire rows on writes (#2864)
- IRowCacheEntry cached = CacheService.instance.rowCache.get(key);
- if (cached != null)
- {
- if (cached instanceof RowCacheSentinel)
- {
- // Some other read is trying to cache the value, just do a normal non-caching read
- Tracing.trace("Row cache miss (race)");
- metric.rowCacheMiss.inc();
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- ColumnFamily cachedCf = (ColumnFamily)cached;
- if (isFilterFullyCoveredBy(filter.filter, cachedCf, filter.timestamp))
- {
- metric.rowCacheHit.inc();
- Tracing.trace("Row cache hit");
- ColumnFamily result = filterColumnFamily(cachedCf, filter);
- metric.updateSSTableIterated(0);
- return result;
- }
-
- metric.rowCacheHitOutOfRange.inc();
- Tracing.trace("Ignoring row cache as cached value could not satisfy query");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- metric.rowCacheMiss.inc();
- Tracing.trace("Row cache miss");
- RowCacheSentinel sentinel = new RowCacheSentinel();
- boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel);
- ColumnFamily data = null;
- ColumnFamily toCache = null;
- try
- {
- // If we are explicitely asked to fill the cache with full partitions, we go ahead and query the whole thing
- if (metadata.getCaching().rowCache.cacheFullPartitions())
- {
- data = getTopLevelColumns(QueryFilter.getIdentityFilter(filter.key, name, filter.timestamp), Integer.MIN_VALUE);
- toCache = data;
- Tracing.trace("Populating row cache with the whole partition");
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return filterColumnFamily(data, filter);
- }
-
- // Otherwise, if we want to cache the result of the query we're about to do, we must make sure this query
- // covers what needs to be cached. And if the user filter does not satisfy that, we sometimes extend said
- // filter so we can populate the cache but only if:
- // 1) we can guarantee it is a strict extension, i.e. that we will still fetch the data asked by the user.
- // 2) the extension does not make us query more than getRowsPerPartitionToCache() (as a mean to limit the
- // amount of extra work we'll do on a user query for the purpose of populating the cache).
- //
- // In practice, we can only guarantee those 2 points if the filter is one that queries the head of the
- // partition (and if that filter actually counts CQL3 rows since that's what we cache and it would be
- // bogus to compare the filter count to the 'rows to cache' otherwise).
- if (filter.filter.isHeadFilter() && filter.filter.countCQL3Rows(metadata.comparator))
- {
- SliceQueryFilter sliceFilter = (SliceQueryFilter)filter.filter;
- int rowsToCache = metadata.getCaching().rowCache.rowsToCache;
-
- SliceQueryFilter cacheSlice = readFilterForCache();
- QueryFilter cacheFilter = new QueryFilter(filter.key, name, cacheSlice, filter.timestamp);
-
- // If the filter count is less than the number of rows cached, we simply extend it to make sure we do cover the
- // number of rows to cache, and if that count is greater than the number of rows to cache, we simply filter what
- // needs to be cached afterwards.
- if (sliceFilter.count < rowsToCache)
- {
- toCache = getTopLevelColumns(cacheFilter, Integer.MIN_VALUE);
- if (toCache != null)
- {
- Tracing.trace("Populating row cache ({} rows cached)", cacheSlice.lastCounted());
- data = filterColumnFamily(toCache, filter);
- }
- }
- else
- {
- data = getTopLevelColumns(filter, Integer.MIN_VALUE);
- if (data != null)
- {
- // The filter limit was greater than the number of rows to cache. But, if the filter had a non-empty
- // finish bound, we may have gotten less than what needs to be cached, in which case we shouldn't cache it
- // (otherwise a cache hit would assume the whole partition is cached which is not the case).
- if (sliceFilter.finish().isEmpty() || sliceFilter.lastCounted() >= rowsToCache)
- {
- toCache = filterColumnFamily(data, cacheFilter);
- Tracing.trace("Caching {} rows (out of {} requested)", cacheSlice.lastCounted(), sliceFilter.count);
- }
- else
- {
- Tracing.trace("Not populating row cache, not enough rows fetched ({} fetched but {} required for the cache)", sliceFilter.lastCounted(), rowsToCache);
- }
- }
- }
-
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return data;
- }
- else
- {
- Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
- }
- finally
- {
- if (sentinelSuccess && toCache == null)
- invalidateCachedRow(key);
- }
- }
-
- public SliceQueryFilter readFilterForCache()
- {
- // We create a new filter everytime before for now SliceQueryFilter is unfortunatly mutable.
- return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, metadata.getCaching().rowCache.rowsToCache, metadata.clusteringColumns().size());
+ return data.getUncompacting();
}
- public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now)
+ public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec)
{
// We can use the cached value only if we know that no data it doesn't contain could be covered
// by the query filter, that is if:
@@@ -1910,40 -2759,45 +1965,52 @@@
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- viewManager.forceBlockingFlush();
-
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- dumpMemtable();
- viewManager.dumpMemtables();
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
++ if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
+ {
+ replayAfter = forceBlockingFlush();
++ viewManager.forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
- final Flush flush = new Flush(true);
- flushExecutor.execute(flush);
- replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
++ viewManager.dumpMemtables();
++ try
++ {
++ replayAfter = dumpMemtable().get();
++ }
++ catch (Exception e)
++ {
++ throw new RuntimeException(e);
++ }
+ }
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
- if (takeSnapshot)
+ if (DatabaseDescriptor.isAutoSnapshot())
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
- for (SecondaryIndex index : indexManager.getIndexes())
- index.truncateBlocking(truncatedAt);
+ indexManager.truncateAllIndexesBlocking(truncatedAt);
-
- viewManager.truncateBlocking(truncatedAt);
++ viewManager.truncateBlocking(replayAfter, truncatedAt);
SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
logger.trace("cleaning out row cache");
@@@ -1955,20 -2809,7 +2022,20 @@@
logger.trace("truncate complete");
}
- public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation)
+ /**
+ * Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable.
+ */
- public void dumpMemtable()
++ public Future<ReplayPosition> dumpMemtable()
+ {
+ synchronized (data)
+ {
+ final Flush flush = new Flush(true);
+ flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
++ return postFlushExecutor.submit(flush.postFlush);
+ }
+ }
+
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
{
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 5d5f7bf,b4ada09..93dc5af
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -87,12 -81,11 +90,11 @@@ public class Memtable implements Compar
}
}
- // We index the memtable by RowPosition only for the purpose of being able
+ // We index the memtable by PartitionPosition only for the purpose of being able
// to select key range using Token.KeyBound. However put() ensures that we
// actually only store DecoratedKey.
- private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
+ private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@@ -101,14 -94,12 +103,16 @@@
// Record the comparator of the CFS at the creation of the memtable. This
// is only used when a user update the CF comparator, to know if the
// memtable was created with the new or old comparator.
- public final CellNameType initialComparator;
+ public final ClusteringComparator initialComparator;
+ private final ColumnsCollector columnsCollector;
+ private final StatsCollector statsCollector = new StatsCollector();
+
- public Memtable(ColumnFamilyStore cfs)
++ // only to be used by init(), to setup the very first memtable for the cfs
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@@ -191,12 -185,12 +200,12 @@@
public boolean isClean()
{
- return rows.isEmpty();
+ return partitions.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@@ -251,14 -256,9 +260,9 @@@
public int partitionCount()
{
- return rows.size();
+ return partitions.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@@ -266,54 -266,65 +270,60 @@@
100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
}
- /**
- * @param startWith Include data in the result from and including this key and to the end of the memtable
- * @return An iterator of entries with the data from the start key
- */
- public Iterator<Map.Entry<DecoratedKey, ColumnFamily>> getEntryIterator(final RowPosition startWith, final RowPosition stopAt)
+ public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange, final boolean isForThrift)
{
- return new Iterator<Map.Entry<DecoratedKey, ColumnFamily>>()
- {
- private Iterator<? extends Map.Entry<? extends RowPosition, AtomicBTreeColumns>> iter = stopAt.isMinimum()
- ? rows.tailMap(startWith).entrySet().iterator()
- : rows.subMap(startWith, true, stopAt, true).entrySet().iterator();
+ AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
- private Map.Entry<? extends RowPosition, ? extends ColumnFamily> currentEntry;
+ boolean startIsMin = keyRange.left.isMinimum();
+ boolean stopIsMin = keyRange.right.isMinimum();
- public boolean hasNext()
- {
- return iter.hasNext();
- }
+ boolean isBound = keyRange instanceof Bounds;
+ boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+ boolean includeStop = isBound || keyRange instanceof Range;
+ Map<PartitionPosition, AtomicBTreePartition> subMap;
+ if (startIsMin)
+ subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop);
+ else
+ subMap = stopIsMin
+ ? partitions.tailMap(keyRange.left, includeStart)
+ : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop);
- public Map.Entry<DecoratedKey, ColumnFamily> next()
- {
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
- // Actual stored key should be true DecoratedKey
- assert entryRowPosition.getKey() instanceof DecoratedKey;
- @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
- if (MEMORY_POOL.needToCopyOnHeap())
- {
- DecoratedKey key = entry.getKey();
- key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
- ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
- entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
- }
- // Store the reference to the current entry so that remove() can update the current size.
- currentEntry = entry;
- return entry;
- }
+ int minLocalDeletionTime = Integer.MAX_VALUE;
- public void remove()
- {
- iter.remove();
- liveDataSize.addAndGet(-currentEntry.getValue().dataSize());
- currentEntry = null;
- }
- };
+ // avoid iterating over the memtable if we purge all tombstones
+ if (cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones())
+ minLocalDeletionTime = findMinLocalDeletionTime(subMap.entrySet().iterator());
+
+ final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter = subMap.entrySet().iterator();
+
+ return new MemtableUnfilteredPartitionIterator(cfs, iter, isForThrift, minLocalDeletionTime, columnFilter, dataRange);
+ }
+
+ private int findMinLocalDeletionTime(Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iterator)
+ {
+ int minLocalDeletionTime = Integer.MAX_VALUE;
+ while (iterator.hasNext())
+ {
+ Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iterator.next();
+ minLocalDeletionTime = Math.min(minLocalDeletionTime, entry.getValue().stats().minLocalDeletionTime);
+ }
+ return minLocalDeletionTime;
}
- public ColumnFamily getColumnFamily(DecoratedKey key)
+ public Partition getPartition(DecoratedKey key)
{
- return rows.get(key);
+ return partitions.get(key);
}
- public long creationTime()
- public SSTableReader flush()
++ public Collection<SSTableReader> flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
- Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
++ Directories.DataDirectory dataDirectory = cfs.getDirectories().getWriteableLocation(estimatedSize);
++ if (dataDirectory == null)
++ throw new RuntimeException("Insufficient disk space to write " + estimatedSize + " bytes");
++ File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@@ -321,137 -332,86 +331,110 @@@
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- private final boolean isBatchLogTable;
-
- FlushRunnable(ReplayPosition context)
+ long keySize = 0;
- for (RowPosition key : rows.keySet())
++ for (PartitionPosition key : partitions.keySet())
{
- this.context = context;
-
- long keySize = 0;
- for (PartitionPosition key : partitions.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
-
- this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected void runMayThrow() throws Exception
- {
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- Collection<SSTableReader> sstables = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstables);
- }
- private SSTableReader writeSortedContents(File sstableDirectory)
++ private Collection<SSTableReader> writeSortedContents(File sstableDirectory)
+ {
- logger.info("Writing {}", Memtable.this.toString());
++ boolean isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- protected Directories getDirectories()
- {
- return cfs.getDirectories();
- }
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
++ logger.debug("Writing {}", Memtable.this.toString());
+
- private Collection<SSTableReader> writeSortedContents(ReplayPosition context, File sstableDirectory)
++ Collection<SSTableReader> ssTables;
++ try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- Collection<SSTableReader> ssTables;
- try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
++ for (AtomicBTreePartition partition : partitions.values())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (AtomicBTreePartition partition : partitions.values())
- AtomicBTreeColumns cf = entry.getValue();
++ // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
++ // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
++ // we don't need to preserve tombstones for repair. So if both operation are in this
++ // memtable (which will almost always be the case if there is no ongoing failure), we can
++ // just skip the entry (CASSANDRA-4667).
++ if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
++ continue;
++
++ if (trackContention && partition.usePessimisticLocking())
++ heavilyContendedRowCount++;
+
- if (cf.isMarkedForDelete() && cf.hasColumns())
++ if (!partition.isEmpty())
{
- // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
- // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
- // we don't need to preserve tombstones for repair. So if both operation are in this
- // memtable (which will almost always be the case if there is no ongoing failure), we can
- // just skip the entry (CASSANDRA-4667).
- if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
-- continue;
-
- if (trackContention && partition.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!partition.isEmpty())
++ try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+ {
- try (UnfilteredRowIterator iter = partition.unfilteredIterator())
- {
- writer.append(iter);
- }
++ writer.append(iter);
+ }
}
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getFilePointer()),
- context));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
++ FBUtilities.prettyPrintMemory(writer.getFilePointer()),
+ commitLogUpperBound));
- // sstables should contain non-repaired data.
- ssTables = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTables = null;
- }
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
++ // sstables should contain non-repaired data.
++ ssTables = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
- ssTable = null;
++ ssTables = Collections.emptyList();
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
+ if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
++ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
- return ssTables;
- }
- return ssTable;
++ return ssTables;
}
+ }
- @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
- public SSTableTxnWriter createFlushWriter(String filename,
- PartitionColumns columns,
- EncodingStats stats)
- private SSTableWriter createFlushWriter(String filename)
++ @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
++ public SSTableTxnWriter createFlushWriter(String filename,
++ PartitionColumns columns,
++ EncodingStats stats)
+ {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
- .commitLogLowerBound(commitLogLowerBound.get())
- .commitLogUpperBound(commitLogUpperBound.get());
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
++ // we operate "offline" here, as we expose the resulting reader consciously when done
++ // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
++ LifecycleTransaction txn = null;
++ try
+ {
- // we operate "offline" here, as we expose the resulting reader consciously when done
- // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
- LifecycleTransaction txn = null;
- try
- {
- txn = LifecycleTransaction.offline(OperationType.FLUSH);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- return new SSTableTxnWriter(txn,
- cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
- (long) partitions.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats),
- txn));
- }
- catch (Throwable t)
- {
- if (txn != null)
- txn.close();
- throw t;
- }
++ txn = LifecycleTransaction.offline(OperationType.FLUSH);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
++ .commitLogLowerBound(commitLogLowerBound.get())
++ .commitLogUpperBound(commitLogUpperBound.get());
++
++ return new SSTableTxnWriter(txn,
++ cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
++ (long) partitions.size(),
++ ActiveRepairService.UNREPAIRED_SSTABLE,
++ sstableMetadataCollector,
++ new SerializationHeader(true, cfs.metadata, columns, stats),
++ txn));
++ }
++ catch (Throwable t)
++ {
++ if (txn != null)
++ txn.close();
++ throw t;
+ }
}
private static int estimateRowOverhead(final int count)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 7169b2f,a58aeb4..f45a47a
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@@ -35,9 -33,6 +35,8 @@@ import com.google.common.base.Throwable
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
- import com.google.common.collect.Ordering;
+import com.google.common.util.concurrent.Uninterruptibles;
+
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -46,18 -43,16 +45,17 @@@ import org.apache.cassandra.concurrent.
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.compress.CompressionParameters;
+import org.apache.cassandra.io.util.FileSegmentInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.io.util.ByteBufferDataInput;
-import org.apache.cassandra.io.util.FastByteArrayInputStream;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.CRC32Factory;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.WrappedRunnable;
@@@ -76,9 -69,9 +74,9 @@@ public class CommitLogReplaye
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
- private final ICRC32 checksum;
+ private final CRC32 checksum;
private byte[] buffer;
private byte[] uncompressedBuffer;
@@@ -94,8 -87,8 +92,8 @@@
this.invalidMutations = new HashMap<UUID, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
- this.checksum = CRC32Factory.instance.create();
+ this.checksum = new CRC32();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@@ -104,17 -97,12 +102,12 @@@
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables(SSTableSet.CANONICAL));
-
- // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@@ -584,16 -580,12 +592,14 @@@
// or c) are part of a cf that was dropped.
// Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
Mutation newMutation = null;
- for (ColumnFamily columnFamily : replayFilter.filter(mutation))
+ for (PartitionUpdate update : replayFilter.filter(mutation))
{
- if (Schema.instance.getCF(columnFamily.id()) == null)
+ if (Schema.instance.getCF(update.metadata().cfId) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(update.metadata().cfId);
-
- if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
+ // replay if current segment is newer than last flushed one or,
+ // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
++ if (shouldReplay(update.metadata().cfId, new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index 28416f3,17802ad..0b21763
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@@ -17,11 -17,12 +17,11 @@@
*/
package org.apache.cassandra.db.commitlog;
-import java.io.DataInput;
import java.io.IOException;
- import java.util.Comparator;
+ import java.util.Map;
+ import java.util.NavigableMap;
+ import java.util.TreeMap;
- import com.google.common.base.Function;
- import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index c205d5c,575aa51..8928db5
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@@ -237,13 -228,10 +237,10 @@@ public abstract class AbstractCompactio
* Handle a flushed memtable.
*
* @param memtable the flushed memtable
- * @param sstable the written sstable. can be null if the memtable was clean.
+ * @param sstables the written sstables. can be null or empty if the memtable was clean.
*/
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 82fd872,0000000..f1127c9
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@@ -1,504 -1,0 +1,501 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.compaction;
+
+
+import java.util.*;
+import java.util.concurrent.Callable;
+
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * Manages the compaction strategies.
+ *
+ * Currently has two instances of actual compaction strategies - one for repaired data and one for
+ * unrepaired data. This is done to be able to totally separate the different sets of sstables.
+ */
+public class CompactionStrategyManager implements INotificationConsumer
+{
+ private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManager.class);
+ private final ColumnFamilyStore cfs;
+ private volatile AbstractCompactionStrategy repaired;
+ private volatile AbstractCompactionStrategy unrepaired;
+ private volatile boolean enabled = true;
+ public boolean isActive = true;
+ private volatile CompactionParams params;
+ /*
+ We keep a copy of the schema compaction parameters here to be able to decide if we
+ should update the compaction strategy in maybeReloadCompactionStrategy() due to an ALTER.
+
+ If a user changes the local compaction strategy and then later ALTERs a compaction parameter,
+ we will use the new compaction parameters.
+ */
+ private CompactionParams schemaCompactionParams;
+
+ public CompactionStrategyManager(ColumnFamilyStore cfs)
+ {
+ cfs.getTracker().subscribe(this);
+ logger.trace("{} subscribed to the data tracker.", this);
+ this.cfs = cfs;
+ reload(cfs.metadata);
+ params = cfs.metadata.params.compaction;
+ enabled = params.isEnabled();
+ }
+
+ /**
+ * Return the next background task
+ *
+ * Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks)
+ *
+ */
+ public synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
+ {
+ if (!isEnabled())
+ return null;
+
+ maybeReload(cfs.metadata);
+
+ if (repaired.getEstimatedRemainingTasks() > unrepaired.getEstimatedRemainingTasks())
+ {
+ AbstractCompactionTask repairedTask = repaired.getNextBackgroundTask(gcBefore);
+ if (repairedTask != null)
+ return repairedTask;
+ return unrepaired.getNextBackgroundTask(gcBefore);
+ }
+ else
+ {
+ AbstractCompactionTask unrepairedTask = unrepaired.getNextBackgroundTask(gcBefore);
+ if (unrepairedTask != null)
+ return unrepairedTask;
+ return repaired.getNextBackgroundTask(gcBefore);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled && isActive;
+ }
+
+ public synchronized void resume()
+ {
+ isActive = true;
+ }
+
+ /**
+ * pause compaction while we cancel all ongoing compactions
+ *
+ * Separate call from enable/disable to not have to save the enabled-state externally
+ */
+ public synchronized void pause()
+ {
+ isActive = false;
+ }
+
+
+ private void startup()
+ {
+ for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+ {
+ if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+ getCompactionStrategyFor(sstable).addSSTable(sstable);
+ }
+ repaired.startup();
+ unrepaired.startup();
+ }
+
+ /**
+ * return the compaction strategy for the given sstable
+ *
+ * returns differently based on the repaired status
+ * @param sstable
+ * @return
+ */
+ private AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
+ {
+ if (sstable.isRepaired())
+ return repaired;
+ else
+ return unrepaired;
+ }
+
+ public void shutdown()
+ {
+ isActive = false;
+ repaired.shutdown();
+ unrepaired.shutdown();
+ }
+
+ public synchronized void maybeReload(CFMetaData metadata)
+ {
+ // compare the old schema configuration to the new one, ignore any locally set changes.
+ if (metadata.params.compaction.equals(schemaCompactionParams))
+ return;
+ reload(metadata);
+ }
+
+ /**
+ * Reload the compaction strategies
+ *
+ * Called after changing configuration and at startup.
+ * @param metadata
+ */
+ public synchronized void reload(CFMetaData metadata)
+ {
+ boolean disabledWithJMX = !enabled && shouldBeEnabled();
+ setStrategy(metadata.params.compaction);
+ schemaCompactionParams = metadata.params.compaction;
+
+ if (disabledWithJMX || !shouldBeEnabled())
+ disable();
+ else
+ enable();
+ startup();
+ }
+
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ {
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
+ }
+
+ public int getUnleveledSSTables()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int count = 0;
+ count += ((LeveledCompactionStrategy)repaired).getLevelSize(0);
+ count += ((LeveledCompactionStrategy)unrepaired).getLevelSize(0);
+ return count;
+ }
+ return 0;
+ }
+
+ public synchronized int[] getSSTableCountPerLevel()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int [] res = new int[LeveledManifest.MAX_LEVEL_COUNT];
+ int[] repairedCountPerLevel = ((LeveledCompactionStrategy) repaired).getAllLevelSize();
+ res = sumArrays(res, repairedCountPerLevel);
+ int[] unrepairedCountPerLevel = ((LeveledCompactionStrategy) unrepaired).getAllLevelSize();
+ res = sumArrays(res, unrepairedCountPerLevel);
+ return res;
+ }
+ return null;
+ }
+
+ private static int[] sumArrays(int[] a, int[] b)
+ {
+ int[] res = new int[Math.max(a.length, b.length)];
+ for (int i = 0; i < res.length; i++)
+ {
+ if (i < a.length && i < b.length)
+ res[i] = a[i] + b[i];
+ else if (i < a.length)
+ res[i] = a[i];
+ else
+ res[i] = b[i];
+ }
+ return res;
+ }
+
+ public boolean shouldDefragment()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.shouldDefragment();
+ }
+
+ public Directories getDirectories()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.getDirectories();
+ }
+
+ public synchronized void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableAddedNotification)
+ {
+ SSTableAddedNotification flushedNotification = (SSTableAddedNotification) notification;
+ for (SSTableReader sstable : flushedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repaired.addSSTable(sstable);
+ else
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableListChangedNotification)
+ {
+ SSTableListChangedNotification listChangedNotification = (SSTableListChangedNotification) notification;
+ Set<SSTableReader> repairedRemoved = new HashSet<>();
+ Set<SSTableReader> repairedAdded = new HashSet<>();
+ Set<SSTableReader> unrepairedRemoved = new HashSet<>();
+ Set<SSTableReader> unrepairedAdded = new HashSet<>();
+
+ for (SSTableReader sstable : listChangedNotification.removed)
+ {
+ if (sstable.isRepaired())
+ repairedRemoved.add(sstable);
+ else
+ unrepairedRemoved.add(sstable);
+ }
+ for (SSTableReader sstable : listChangedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repairedAdded.add(sstable);
+ else
+ unrepairedAdded.add(sstable);
+ }
+ if (!repairedRemoved.isEmpty())
+ {
+ repaired.replaceSSTables(repairedRemoved, repairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : repairedAdded)
+ repaired.addSSTable(sstable);
+ }
+
+ if (!unrepairedRemoved.isEmpty())
+ {
+ unrepaired.replaceSSTables(unrepairedRemoved, unrepairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : unrepairedAdded)
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableRepairStatusChanged)
+ {
+ for (SSTableReader sstable : ((SSTableRepairStatusChanged) notification).sstable)
+ {
+ if (sstable.isRepaired())
+ {
+ unrepaired.removeSSTable(sstable);
+ repaired.addSSTable(sstable);
+ }
+ else
+ {
+ repaired.removeSSTable(sstable);
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ }
+ else if (notification instanceof SSTableDeletingNotification)
+ {
+ SSTableReader sstable = ((SSTableDeletingNotification)notification).deleting;
+ if (sstable.isRepaired())
+ repaired.removeSSTable(sstable);
+ else
+ unrepaired.removeSSTable(sstable);
+ }
+ }
+
+ public void enable()
+ {
+ if (repaired != null)
+ repaired.enable();
+ if (unrepaired != null)
+ unrepaired.enable();
+ // enable this last to make sure the strategies are ready to get calls.
+ enabled = true;
+ }
+
+ public void disable()
+ {
+ // disable this first avoid asking disabled strategies for compaction tasks
+ enabled = false;
+ if (repaired != null)
+ repaired.disable();
+ if (unrepaired != null)
+ unrepaired.disable();
+ }
+
+ /**
+ * Create ISSTableScanner from the given sstables
+ *
+ * Delegates the call to the compaction strategies to allow LCS to create a scanner
+ * @param sstables
+ * @param ranges
+ * @return
+ */
+ @SuppressWarnings("resource")
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ {
+ List<SSTableReader> repairedSSTables = new ArrayList<>();
+ List<SSTableReader> unrepairedSSTables = new ArrayList<>();
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.isRepaired())
+ repairedSSTables.add(sstable);
+ else
+ unrepairedSSTables.add(sstable);
+ }
+
+ Set<ISSTableScanner> scanners = new HashSet<>(sstables.size());
+ AbstractCompactionStrategy.ScannerList repairedScanners = repaired.getScanners(repairedSSTables, ranges);
+ AbstractCompactionStrategy.ScannerList unrepairedScanners = unrepaired.getScanners(unrepairedSSTables, ranges);
+ scanners.addAll(repairedScanners.scanners);
+ scanners.addAll(unrepairedScanners.scanners);
+ return new AbstractCompactionStrategy.ScannerList(new ArrayList<>(scanners));
+ }
+
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables)
+ {
+ return getScanners(sstables, null);
+ }
+
+ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+ {
+ return unrepaired.groupSSTablesForAntiCompaction(sstablesToGroup);
+ }
+
+ public long getMaxSSTableBytes()
+ {
+ return unrepaired.getMaxSSTableBytes();
+ }
+
+ public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, int gcBefore, long maxSSTableBytes)
+ {
+ return getCompactionStrategyFor(txn.originals().iterator().next()).getCompactionTask(txn, gcBefore, maxSSTableBytes);
+ }
+
+ public Collection<AbstractCompactionTask> getMaximalTasks(final int gcBefore, final boolean splitOutput)
+ {
+ // runWithCompactionsDisabled cancels active compactions and disables them, then we are able
+ // to make the repaired/unrepaired strategies mark their own sstables as compacting. Once the
+ // sstables are marked the compactions are re-enabled
+ return cfs.runWithCompactionsDisabled(new Callable<Collection<AbstractCompactionTask>>()
+ {
+ @Override
+ public Collection<AbstractCompactionTask> call() throws Exception
+ {
+ synchronized (CompactionStrategyManager.this)
+ {
+ Collection<AbstractCompactionTask> repairedTasks = repaired.getMaximalTask(gcBefore, splitOutput);
+ Collection<AbstractCompactionTask> unrepairedTasks = unrepaired.getMaximalTask(gcBefore, splitOutput);
+
+ if (repairedTasks == null && unrepairedTasks == null)
+ return null;
+
+ if (repairedTasks == null)
+ return unrepairedTasks;
+ if (unrepairedTasks == null)
+ return repairedTasks;
+
+ List<AbstractCompactionTask> tasks = new ArrayList<>();
+ tasks.addAll(repairedTasks);
+ tasks.addAll(unrepairedTasks);
+ return tasks;
+ }
+ }
+ }, false, false);
+ }
+
+ public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
+ {
+ return getCompactionStrategyFor(sstables.iterator().next()).getUserDefinedTask(sstables, gcBefore);
+ }
+
+ public int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ tasks += repaired.getEstimatedRemainingTasks();
+ tasks += unrepaired.getEstimatedRemainingTasks();
+
+ return tasks;
+ }
+
+ public boolean shouldBeEnabled()
+ {
+ return params.isEnabled();
+ }
+
+ public String getName()
+ {
+ return unrepaired.getName();
+ }
+
+ public List<AbstractCompactionStrategy> getStrategies()
+ {
+ return Arrays.asList(repaired, unrepaired);
+ }
+
+ public synchronized void setNewLocalCompactionStrategy(CompactionParams params)
+ {
+ logger.info("Switching local compaction strategy from {} to {}}", this.params, params);
+ setStrategy(params);
+ if (shouldBeEnabled())
+ enable();
+ else
+ disable();
+ startup();
+ }
+
+ private void setStrategy(CompactionParams params)
+ {
+ if (repaired != null)
+ repaired.shutdown();
+ if (unrepaired != null)
+ unrepaired.shutdown();
+ repaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ unrepaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ this.params = params;
+ }
+
+ public CompactionParams getCompactionParams()
+ {
+ return params;
+ }
+
+ public boolean onlyPurgeRepairedTombstones()
+ {
+ return Boolean.parseBoolean(params.options().get(AbstractCompactionStrategy.ONLY_PURGE_REPAIRED_TOMBSTONES));
+ }
+
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn)
+ {
+ if (repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
+ {
+ return unrepaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ else
+ {
+ return repaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index c09d49c,5d5701f..16090a1
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@@ -195,10 -199,12 +199,12 @@@ public class Tracke
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
- Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader, SSTableReader>emptyMap(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@@ -327,10 -314,10 +332,10 @@@
apply(View.markFlushing(memtable));
}
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
assert !isDummy();
- if (sstables == null || sstables.isEmpty())
- if (sstable == null)
++ if (sstables.isEmpty())
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
// if it's null, we don't care what state the cfstore is in, we just replace it and continue
@@@ -338,27 -325,58 +343,57 @@@
return;
}
- sstable.setupKeyCache();
+ sstables.forEach(SSTableReader::setupOnline);
// back up before creating a new Snapshot (which makes the new one eligible for compaction)
- maybeIncrementallyBackup(sstable);
+ maybeIncrementallyBackup(sstables);
- apply(View.replaceFlushed(memtable, sstable));
+ apply(View.replaceFlushed(memtable, sstables));
Throwable fail;
- fail = updateSizeTracking(emptySet(), singleton(sstable), null);
+ fail = updateSizeTracking(emptySet(), sstables, null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
- public void permitCompactionOfFlushed(SSTableReader sstable)
++ public void permitCompactionOfFlushed(Collection<SSTableReader> sstables)
+ {
- if (sstable == null)
++ if (sstables.isEmpty())
+ return;
+
- apply(View.permitCompactionOfFlushed(sstable));
++ apply(View.permitCompactionOfFlushed(sstables));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
- notifyAdded(sstable);
++ notifyAdded(sstables);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
-
// MISCELLANEOUS public utility calls
+ public Set<SSTableReader> getSSTables()
+ {
+ return view.get().sstables;
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
[11/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
Merge commit '849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/78a3d2bb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/78a3d2bb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/78a3d2bb
Branch: refs/heads/cassandra-3.7
Commit: 78a3d2bba95b9efcda152a157f822f4970f22636
Parents: e9657a4 849a438
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:17:51 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:19:15 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 282 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 243 ++++++++--------
.../db/commitlog/CommitLogReplayer.java | 56 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 37 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 23 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
92 files changed, 715 insertions(+), 381 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 306f46f,dfcad10..677ea11
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,30 -1,7 +1,31 @@@
- * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
+3.0.7
+ * Refactor Materialized View code (CASSANDRA-11475)
+ * Update Java Driver (CASSANDRA-11615)
+Merged from 2.2:
+ * Prohibit Reversed Counter type as part of the PK (CASSANDRA-9395)
-2.2.7
- * Prohibit Reverse Counter type as part of the PK (CASSANDRA-9395)
+3.0.6
+ * Disallow creating view with a static column (CASSANDRA-11602)
+ * Reduce the amount of object allocations caused by the getFunctions methods (CASSANDRA-11593)
+ * Potential error replaying commitlog with smallint/tinyint/date/time types (CASSANDRA-11618)
+ * Fix queries with filtering on counter columns (CASSANDRA-11629)
+ * Improve tombstone printing in sstabledump (CASSANDRA-11655)
+ * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669)
+ * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600)
+ * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654)
+ * Ignore all LocalStrategy keyspaces for streaming and other related
+ operations (CASSANDRA-11627)
+ * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523)
+ * Only open one sstable scanner per sstable (CASSANDRA-11412)
+ * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410)
+ * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485)
+ * LogAwareFileLister should only use OLD sstable files in current folder to determine disk consistency (CASSANDRA-11470)
+ * Notify indexers of expired rows during compaction (CASSANDRA-11329)
+ * Properly respond with ProtocolError when a v1/v2 native protocol
+ header is received (CASSANDRA-11464)
+ * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120)
+Merged from 2.2:
++ * Fix commit log replay after out-of-order flush completion (CASSANDRA-9669)
* cqlsh: correctly handle non-ascii chars in error messages (CASSANDRA-11626)
* Exit JVM if JMX server fails to startup (CASSANDRA-11540)
* Produce a heap dump when exiting on OOM (CASSANDRA-9861)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 5b8de8f,88e22c0..e9a2938
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -892,10 -996,13 +903,13 @@@ public class ColumnFamilyStore implemen
final boolean flushSecondaryIndexes;
final OpOrder.Barrier writeBarrier;
final CountDownLatch latch = new CountDownLatch(1);
- final ReplayPosition lastReplayPosition;
+ volatile FSWriteError flushFailure = null;
+ final ReplayPosition commitLogUpperBound;
+ final List<Memtable> memtables;
- final List<SSTableReader> readers;
- volatile FSWriteError flushFailure = null;
++ final List<Collection<SSTableReader>> readers;
- private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition lastReplayPosition)
+ private PostFlush(boolean flushSecondaryIndexes, OpOrder.Barrier writeBarrier, ReplayPosition commitLogUpperBound,
- List<Memtable> memtables, List<SSTableReader> readers)
++ List<Memtable> memtables, List<Collection<SSTableReader>> readers)
{
this.writeBarrier = writeBarrier;
this.flushSecondaryIndexes = flushSecondaryIndexes;
@@@ -927,18 -1046,23 +946,25 @@@
throw new IllegalStateException();
}
- // must check lastReplayPosition != null because Flush may find that all memtables are clean
- // and so not set a lastReplayPosition
// If a flush errored out but the error was ignored, make sure we don't discard the commit log.
- if (lastReplayPosition != null && flushFailure == null)
+ if (flushFailure == null)
{
- CommitLog.instance.discardCompletedSegments(metadata.cfId, lastReplayPosition);
+ CommitLog.instance.discardCompletedSegments(metadata.cfId, commitLogUpperBound);
+ for (int i = 0 ; i < memtables.size() ; i++)
+ {
+ Memtable memtable = memtables.get(i);
- SSTableReader reader = readers.get(i);
++ Collection<SSTableReader> reader = readers.get(i);
+ memtable.cfs.data.permitCompactionOfFlushed(reader);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, reader);
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, reader);
+ }
}
+
metric.pendingFlushes.dec();
if (flushFailure != null)
throw flushFailure;
++
+ return commitLogUpperBound;
}
}
@@@ -953,7 -1077,8 +979,8 @@@
private final class Flush implements Runnable
{
final OpOrder.Barrier writeBarrier;
- final List<Memtable> memtables;
+ final List<Memtable> memtables = new ArrayList<>();
- final List<SSTableReader> readers = new ArrayList<>();
++ final List<Collection<SSTableReader>> readers = new ArrayList<>();
final PostFlush postFlush;
final boolean truncate;
@@@ -1024,27 -1139,30 +1041,23 @@@
memtable.cfs.data.markFlushing(memtable);
if (memtable.isClean() || truncate)
{
- memtable.cfs.replaceFlushed(memtable, null);
- memtable.cfs.data.replaceFlushed(memtable, null);
- memtable.cfs.compactionStrategyWrapper.replaceFlushed(memtable, null);
++ memtable.cfs.data.replaceFlushed(memtable, Collections.emptyList());
++ memtable.cfs.compactionStrategyManager.replaceFlushed(memtable, Collections.emptyList());
reclaim(memtable);
iter.remove();
}
}
-- if (memtables.isEmpty())
-- {
-- postFlush.latch.countDown();
-- return;
-- }
--
metric.memtableSwitchCount.inc();
try
{
for (Memtable memtable : memtables)
{
-- // flush the memtable
- MoreExecutors.sameThreadExecutor().execute(memtable.flushRunnable());
- SSTableReader reader = memtable.flush();
- memtable.cfs.data.replaceFlushed(memtable, reader);
++ Collection<SSTableReader> readers = memtable.flush();
++ memtable.cfs.data.replaceFlushed(memtable, readers);
reclaim(memtable);
- readers.add(reader);
++ this.readers.add(readers);
}
}
catch (FSWriteError e)
@@@ -1406,22 -1655,172 +1451,32 @@@
return data;
}
+ public Collection<SSTableReader> getSSTables()
+ {
+ return data.getSSTables();
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompactSSTables()
+ {
+ return data.getPermittedToCompact();
+ }
+
- public Set<SSTableReader> getUncompactingSSTables()
+ public Set<SSTableReader> getLiveSSTables()
{
- return data.getUncompacting();
+ return data.getView().liveSSTables();
}
- public ColumnFamily getColumnFamily(DecoratedKey key,
- Composite start,
- Composite finish,
- boolean reversed,
- int limit,
- long timestamp)
+ public Iterable<SSTableReader> getSSTables(SSTableSet sstableSet)
{
- return getColumnFamily(QueryFilter.getSliceFilter(key, name, start, finish, reversed, limit, timestamp));
+ return data.getView().sstables(sstableSet);
}
- /**
- * Fetch the row and columns given by filter.key if it is in the cache; if not, read it from disk and cache it
- *
- * If row is cached, and the filter given is within its bounds, we return from cache, otherwise from disk
- *
- * If row is not cached, we figure out what filter is "biggest", read that from disk, then
- * filter the result and either cache that or return it.
- *
- * @param cfId the column family to read the row from
- * @param filter the columns being queried.
- * @return the requested data for the filter provided
- */
- private ColumnFamily getThroughCache(UUID cfId, QueryFilter filter)
+ public Iterable<SSTableReader> getUncompactingSSTables()
{
- assert isRowCacheEnabled()
- : String.format("Row cache is not enabled on table [" + name + "]");
-
- RowCacheKey key = new RowCacheKey(metadata.ksAndCFName, filter.key);
-
- // attempt a sentinel-read-cache sequence. if a write invalidates our sentinel, we'll return our
- // (now potentially obsolete) data, but won't cache it. see CASSANDRA-3862
- // TODO: don't evict entire rows on writes (#2864)
- IRowCacheEntry cached = CacheService.instance.rowCache.get(key);
- if (cached != null)
- {
- if (cached instanceof RowCacheSentinel)
- {
- // Some other read is trying to cache the value, just do a normal non-caching read
- Tracing.trace("Row cache miss (race)");
- metric.rowCacheMiss.inc();
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- ColumnFamily cachedCf = (ColumnFamily)cached;
- if (isFilterFullyCoveredBy(filter.filter, cachedCf, filter.timestamp))
- {
- metric.rowCacheHit.inc();
- Tracing.trace("Row cache hit");
- ColumnFamily result = filterColumnFamily(cachedCf, filter);
- metric.updateSSTableIterated(0);
- return result;
- }
-
- metric.rowCacheHitOutOfRange.inc();
- Tracing.trace("Ignoring row cache as cached value could not satisfy query");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
-
- metric.rowCacheMiss.inc();
- Tracing.trace("Row cache miss");
- RowCacheSentinel sentinel = new RowCacheSentinel();
- boolean sentinelSuccess = CacheService.instance.rowCache.putIfAbsent(key, sentinel);
- ColumnFamily data = null;
- ColumnFamily toCache = null;
- try
- {
- // If we are explicitely asked to fill the cache with full partitions, we go ahead and query the whole thing
- if (metadata.getCaching().rowCache.cacheFullPartitions())
- {
- data = getTopLevelColumns(QueryFilter.getIdentityFilter(filter.key, name, filter.timestamp), Integer.MIN_VALUE);
- toCache = data;
- Tracing.trace("Populating row cache with the whole partition");
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return filterColumnFamily(data, filter);
- }
-
- // Otherwise, if we want to cache the result of the query we're about to do, we must make sure this query
- // covers what needs to be cached. And if the user filter does not satisfy that, we sometimes extend said
- // filter so we can populate the cache but only if:
- // 1) we can guarantee it is a strict extension, i.e. that we will still fetch the data asked by the user.
- // 2) the extension does not make us query more than getRowsPerPartitionToCache() (as a mean to limit the
- // amount of extra work we'll do on a user query for the purpose of populating the cache).
- //
- // In practice, we can only guarantee those 2 points if the filter is one that queries the head of the
- // partition (and if that filter actually counts CQL3 rows since that's what we cache and it would be
- // bogus to compare the filter count to the 'rows to cache' otherwise).
- if (filter.filter.isHeadFilter() && filter.filter.countCQL3Rows(metadata.comparator))
- {
- SliceQueryFilter sliceFilter = (SliceQueryFilter)filter.filter;
- int rowsToCache = metadata.getCaching().rowCache.rowsToCache;
-
- SliceQueryFilter cacheSlice = readFilterForCache();
- QueryFilter cacheFilter = new QueryFilter(filter.key, name, cacheSlice, filter.timestamp);
-
- // If the filter count is less than the number of rows cached, we simply extend it to make sure we do cover the
- // number of rows to cache, and if that count is greater than the number of rows to cache, we simply filter what
- // needs to be cached afterwards.
- if (sliceFilter.count < rowsToCache)
- {
- toCache = getTopLevelColumns(cacheFilter, Integer.MIN_VALUE);
- if (toCache != null)
- {
- Tracing.trace("Populating row cache ({} rows cached)", cacheSlice.lastCounted());
- data = filterColumnFamily(toCache, filter);
- }
- }
- else
- {
- data = getTopLevelColumns(filter, Integer.MIN_VALUE);
- if (data != null)
- {
- // The filter limit was greater than the number of rows to cache. But, if the filter had a non-empty
- // finish bound, we may have gotten less than what needs to be cached, in which case we shouldn't cache it
- // (otherwise a cache hit would assume the whole partition is cached which is not the case).
- if (sliceFilter.finish().isEmpty() || sliceFilter.lastCounted() >= rowsToCache)
- {
- toCache = filterColumnFamily(data, cacheFilter);
- Tracing.trace("Caching {} rows (out of {} requested)", cacheSlice.lastCounted(), sliceFilter.count);
- }
- else
- {
- Tracing.trace("Not populating row cache, not enough rows fetched ({} fetched but {} required for the cache)", sliceFilter.lastCounted(), rowsToCache);
- }
- }
- }
-
- if (sentinelSuccess && toCache != null)
- CacheService.instance.rowCache.replace(key, sentinel, toCache);
- return data;
- }
- else
- {
- Tracing.trace("Fetching data but not populating cache as query does not query from the start of the partition");
- return getTopLevelColumns(filter, Integer.MIN_VALUE);
- }
- }
- finally
- {
- if (sentinelSuccess && toCache == null)
- invalidateCachedRow(key);
- }
- }
-
- public SliceQueryFilter readFilterForCache()
- {
- // We create a new filter everytime before for now SliceQueryFilter is unfortunatly mutable.
- return new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, metadata.getCaching().rowCache.rowsToCache, metadata.clusteringColumns().size());
+ return data.getUncompacting();
}
- public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now)
+ public boolean isFilterFullyCoveredBy(ClusteringIndexFilter filter, DataLimits limits, CachedPartition cached, int nowInSec)
{
// We can use the cached value only if we know that no data it doesn't contain could be covered
// by the query filter, that is if:
@@@ -1910,40 -2759,45 +1965,52 @@@
// position in the System keyspace.
logger.trace("truncating {}", name);
- if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
+ final long truncatedAt;
+ final ReplayPosition replayAfter;
- viewManager.forceBlockingFlush();
-
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
- else
+ synchronized (data)
{
- dumpMemtable();
- viewManager.dumpMemtables();
- if (keyspace.getMetadata().durableWrites || takeSnapshot)
++ if (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())
+ {
+ replayAfter = forceBlockingFlush();
++ viewManager.forceBlockingFlush();
+ }
+ else
+ {
+ // just nuke the memtable data w/o writing to disk first
- final Flush flush = new Flush(true);
- flushExecutor.execute(flush);
- replayAfter = FBUtilities.waitOnFuture(postFlushExecutor.submit(flush.postFlush));
++ viewManager.dumpMemtables();
++ try
++ {
++ replayAfter = dumpMemtable().get();
++ }
++ catch (Exception e)
++ {
++ throw new RuntimeException(e);
++ }
+ }
+
+ long now = System.currentTimeMillis();
+ // make sure none of our sstables are somehow in the future (clock drift, perhaps)
+ for (ColumnFamilyStore cfs : concatWithIndexes())
+ for (SSTableReader sstable : cfs.data.getSSTables())
+ now = Math.max(now, sstable.maxDataAge);
+ truncatedAt = now;
}
Runnable truncateRunnable = new Runnable()
{
public void run()
{
- logger.trace("Discarding sstable data for truncated CF + indexes");
-
- final long truncatedAt = System.currentTimeMillis();
+ logger.debug("Discarding sstable data for truncated CF + indexes");
data.notifyTruncated(truncatedAt);
- if (takeSnapshot)
+ if (DatabaseDescriptor.isAutoSnapshot())
snapshot(Keyspace.getTimestampedSnapshotName(name));
- ReplayPosition replayAfter = discardSSTables(truncatedAt);
+ discardSSTables(truncatedAt);
- for (SecondaryIndex index : indexManager.getIndexes())
- index.truncateBlocking(truncatedAt);
+ indexManager.truncateAllIndexesBlocking(truncatedAt);
-
- viewManager.truncateBlocking(truncatedAt);
++ viewManager.truncateBlocking(replayAfter, truncatedAt);
SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
logger.trace("cleaning out row cache");
@@@ -1955,20 -2809,7 +2022,20 @@@
logger.trace("truncate complete");
}
- public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation)
+ /**
+ * Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable.
+ */
- public void dumpMemtable()
++ public Future<ReplayPosition> dumpMemtable()
+ {
+ synchronized (data)
+ {
+ final Flush flush = new Flush(true);
+ flushExecutor.execute(flush);
- postFlushExecutor.submit(flush.postFlush);
++ return postFlushExecutor.submit(flush.postFlush);
+ }
+ }
+
+ public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
{
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Memtable.java
index 5d5f7bf,b4ada09..93dc5af
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@@ -87,12 -81,11 +90,11 @@@ public class Memtable implements Compar
}
}
- // We index the memtable by RowPosition only for the purpose of being able
+ // We index the memtable by PartitionPosition only for the purpose of being able
// to select key range using Token.KeyBound. However put() ensures that we
// actually only store DecoratedKey.
- private final ConcurrentNavigableMap<RowPosition, AtomicBTreeColumns> rows = new ConcurrentSkipListMap<>();
+ private final ConcurrentNavigableMap<PartitionPosition, AtomicBTreePartition> partitions = new ConcurrentSkipListMap<>();
public final ColumnFamilyStore cfs;
- private final long creationTime = System.currentTimeMillis();
private final long creationNano = System.nanoTime();
// The smallest timestamp for all partitions stored in this memtable
@@@ -101,14 -94,12 +103,16 @@@
// Record the comparator of the CFS at the creation of the memtable. This
// is only used when a user update the CF comparator, to know if the
// memtable was created with the new or old comparator.
- public final CellNameType initialComparator;
+ public final ClusteringComparator initialComparator;
+ private final ColumnsCollector columnsCollector;
+ private final StatsCollector statsCollector = new StatsCollector();
+
- public Memtable(ColumnFamilyStore cfs)
++ // only to be used by init(), to setup the very first memtable for the cfs
+ public Memtable(AtomicReference<ReplayPosition> commitLogLowerBound, ColumnFamilyStore cfs)
{
this.cfs = cfs;
+ this.commitLogLowerBound = commitLogLowerBound;
this.allocator = MEMORY_POOL.newAllocator();
this.initialComparator = cfs.metadata.comparator;
this.cfs.scheduleFlush();
@@@ -191,12 -185,12 +200,12 @@@
public boolean isClean()
{
- return rows.isEmpty();
+ return partitions.isEmpty();
}
- public boolean isCleanAfter(ReplayPosition position)
+ public boolean mayContainDataBefore(ReplayPosition position)
{
- return isClean() || (position != null && minReplayPosition.compareTo(position) >= 0);
+ return approximateCommitLogLowerBound.compareTo(position) < 0;
}
/**
@@@ -251,14 -256,9 +260,9 @@@
public int partitionCount()
{
- return rows.size();
+ return partitions.size();
}
- public FlushRunnable flushRunnable()
- {
- return new FlushRunnable(lastReplayPosition.get());
- }
-
public String toString()
{
return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)",
@@@ -266,54 -266,65 +270,60 @@@
100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio());
}
- /**
- * @param startWith Include data in the result from and including this key and to the end of the memtable
- * @return An iterator of entries with the data from the start key
- */
- public Iterator<Map.Entry<DecoratedKey, ColumnFamily>> getEntryIterator(final RowPosition startWith, final RowPosition stopAt)
+ public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange, final boolean isForThrift)
{
- return new Iterator<Map.Entry<DecoratedKey, ColumnFamily>>()
- {
- private Iterator<? extends Map.Entry<? extends RowPosition, AtomicBTreeColumns>> iter = stopAt.isMinimum()
- ? rows.tailMap(startWith).entrySet().iterator()
- : rows.subMap(startWith, true, stopAt, true).entrySet().iterator();
+ AbstractBounds<PartitionPosition> keyRange = dataRange.keyRange();
- private Map.Entry<? extends RowPosition, ? extends ColumnFamily> currentEntry;
+ boolean startIsMin = keyRange.left.isMinimum();
+ boolean stopIsMin = keyRange.right.isMinimum();
- public boolean hasNext()
- {
- return iter.hasNext();
- }
+ boolean isBound = keyRange instanceof Bounds;
+ boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds;
+ boolean includeStop = isBound || keyRange instanceof Range;
+ Map<PartitionPosition, AtomicBTreePartition> subMap;
+ if (startIsMin)
+ subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop);
+ else
+ subMap = stopIsMin
+ ? partitions.tailMap(keyRange.left, includeStart)
+ : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop);
- public Map.Entry<DecoratedKey, ColumnFamily> next()
- {
- Map.Entry<? extends RowPosition, ? extends ColumnFamily> entryRowPosition = iter.next();
- // Actual stored key should be true DecoratedKey
- assert entryRowPosition.getKey() instanceof DecoratedKey;
- @SuppressWarnings("unchecked") // Object cast is required since otherwise we can't turn RowPosition into DecoratedKey
- Map.Entry<DecoratedKey, ColumnFamily> entry = (Map.Entry<DecoratedKey, ColumnFamily>) entryRowPosition;
- if (MEMORY_POOL.needToCopyOnHeap())
- {
- DecoratedKey key = entry.getKey();
- key = new BufferDecoratedKey(key.getToken(), HeapAllocator.instance.clone(key.getKey()));
- ColumnFamily cells = ArrayBackedSortedColumns.localCopy(entry.getValue(), HeapAllocator.instance);
- entry = new AbstractMap.SimpleImmutableEntry<>(key, cells);
- }
- // Store the reference to the current entry so that remove() can update the current size.
- currentEntry = entry;
- return entry;
- }
+ int minLocalDeletionTime = Integer.MAX_VALUE;
- public void remove()
- {
- iter.remove();
- liveDataSize.addAndGet(-currentEntry.getValue().dataSize());
- currentEntry = null;
- }
- };
+ // avoid iterating over the memtable if we purge all tombstones
+ if (cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones())
+ minLocalDeletionTime = findMinLocalDeletionTime(subMap.entrySet().iterator());
+
+ final Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iter = subMap.entrySet().iterator();
+
+ return new MemtableUnfilteredPartitionIterator(cfs, iter, isForThrift, minLocalDeletionTime, columnFilter, dataRange);
+ }
+
+ private int findMinLocalDeletionTime(Iterator<Map.Entry<PartitionPosition, AtomicBTreePartition>> iterator)
+ {
+ int minLocalDeletionTime = Integer.MAX_VALUE;
+ while (iterator.hasNext())
+ {
+ Map.Entry<PartitionPosition, AtomicBTreePartition> entry = iterator.next();
+ minLocalDeletionTime = Math.min(minLocalDeletionTime, entry.getValue().stats().minLocalDeletionTime);
+ }
+ return minLocalDeletionTime;
}
- public ColumnFamily getColumnFamily(DecoratedKey key)
+ public Partition getPartition(DecoratedKey key)
{
- return rows.get(key);
+ return partitions.get(key);
}
- public long creationTime()
- public SSTableReader flush()
++ public Collection<SSTableReader> flush()
{
- return creationTime;
+ long estimatedSize = estimatedSize();
- Directories.DataDirectory dataDirectory = cfs.directories.getWriteableLocation(estimatedSize);
- File sstableDirectory = cfs.directories.getLocationForDisk(dataDirectory);
++ Directories.DataDirectory dataDirectory = cfs.getDirectories().getWriteableLocation(estimatedSize);
++ if (dataDirectory == null)
++ throw new RuntimeException("Insufficient disk space to write " + estimatedSize + " bytes");
++ File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
+ assert sstableDirectory != null : "Flush task is not bound to any disk";
+ return writeSortedContents(sstableDirectory);
}
public long getMinTimestamp()
@@@ -321,137 -332,86 +331,110 @@@
return minTimestamp;
}
- class FlushRunnable extends DiskAwareRunnable
+ private long estimatedSize()
{
- private final ReplayPosition context;
- private final long estimatedSize;
-
- private final boolean isBatchLogTable;
-
- FlushRunnable(ReplayPosition context)
+ long keySize = 0;
- for (RowPosition key : rows.keySet())
++ for (PartitionPosition key : partitions.keySet())
{
- this.context = context;
-
- long keySize = 0;
- for (PartitionPosition key : partitions.keySet())
- {
- // make sure we don't write non-sensical keys
- assert key instanceof DecoratedKey;
- keySize += ((DecoratedKey)key).getKey().remaining();
- }
- estimatedSize = (long) ((keySize // index entries
- + keySize // keys in data file
- + liveDataSize.get()) // data
- * 1.2); // bloom filter and row index overhead
-
- this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- }
-
- public long getExpectedWriteSize()
- {
- return estimatedSize;
+ // make sure we don't write non-sensical keys
+ assert key instanceof DecoratedKey;
+ keySize += ((DecoratedKey)key).getKey().remaining();
}
+ return (long) ((keySize // index entries
+ + keySize // keys in data file
+ + liveDataSize.get()) // data
+ * 1.2); // bloom filter and row index overhead
+ }
- protected void runMayThrow() throws Exception
- {
- long writeSize = getExpectedWriteSize();
- Directories.DataDirectory dataDirectory = getWriteDirectory(writeSize);
- File sstableDirectory = cfs.getDirectories().getLocationForDisk(dataDirectory);
- assert sstableDirectory != null : "Flush task is not bound to any disk";
- Collection<SSTableReader> sstables = writeSortedContents(context, sstableDirectory);
- cfs.replaceFlushed(Memtable.this, sstables);
- }
- private SSTableReader writeSortedContents(File sstableDirectory)
++ private Collection<SSTableReader> writeSortedContents(File sstableDirectory)
+ {
- logger.info("Writing {}", Memtable.this.toString());
++ boolean isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SystemKeyspace.NAME);
- protected Directories getDirectories()
- {
- return cfs.getDirectories();
- }
- SSTableReader ssTable;
- // errors when creating the writer that may leave empty temp files.
- try (SSTableWriter writer = createFlushWriter(cfs.getTempSSTablePath(sstableDirectory)))
++ logger.debug("Writing {}", Memtable.this.toString());
+
- private Collection<SSTableReader> writeSortedContents(ReplayPosition context, File sstableDirectory)
++ Collection<SSTableReader> ssTables;
++ try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
{
- logger.debug("Writing {}", Memtable.this.toString());
-
- Collection<SSTableReader> ssTables;
- try (SSTableTxnWriter writer = createFlushWriter(cfs.getSSTablePath(sstableDirectory), columnsCollector.get(), statsCollector.get()))
+ boolean trackContention = logger.isTraceEnabled();
+ int heavilyContendedRowCount = 0;
+ // (we can't clear out the map as-we-go to free up memory,
+ // since the memtable is being used for queries in the "pending flush" category)
- for (Map.Entry<RowPosition, AtomicBTreeColumns> entry : rows.entrySet())
++ for (AtomicBTreePartition partition : partitions.values())
{
- boolean trackContention = logger.isTraceEnabled();
- int heavilyContendedRowCount = 0;
- // (we can't clear out the map as-we-go to free up memory,
- // since the memtable is being used for queries in the "pending flush" category)
- for (AtomicBTreePartition partition : partitions.values())
- AtomicBTreeColumns cf = entry.getValue();
++ // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
++ // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
++ // we don't need to preserve tombstones for repair. So if both operation are in this
++ // memtable (which will almost always be the case if there is no ongoing failure), we can
++ // just skip the entry (CASSANDRA-4667).
++ if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
++ continue;
++
++ if (trackContention && partition.usePessimisticLocking())
++ heavilyContendedRowCount++;
+
- if (cf.isMarkedForDelete() && cf.hasColumns())
++ if (!partition.isEmpty())
{
- // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2
- // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local,
- // we don't need to preserve tombstones for repair. So if both operation are in this
- // memtable (which will almost always be the case if there is no ongoing failure), we can
- // just skip the entry (CASSANDRA-4667).
- if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows())
- // When every node is up, there's no reason to write batchlog data out to sstables
- // (which in turn incurs cost like compaction) since the BL write + delete cancel each other out,
- // and BL data is strictly local, so we don't need to preserve tombstones for repair.
- // If we have a data row + row level tombstone, then writing it is effectively an expensive no-op so we skip it.
- // See CASSANDRA-4667.
- if (cfs.name.equals(SystemKeyspace.BATCHLOG) && cfs.keyspace.getName().equals(SystemKeyspace.NAME))
-- continue;
-
- if (trackContention && partition.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!partition.isEmpty())
++ try (UnfilteredRowIterator iter = partition.unfilteredIterator())
+ {
- try (UnfilteredRowIterator iter = partition.unfilteredIterator())
- {
- writer.append(iter);
- }
++ writer.append(iter);
+ }
}
-
- if (trackContention && cf.usePessimisticLocking())
- heavilyContendedRowCount++;
-
- if (!cf.isEmpty())
- writer.append((DecoratedKey)entry.getKey(), cf);
+ }
- if (writer.getFilePointer() > 0)
- {
- logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
- writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getFilePointer()),
- context));
+ if (writer.getFilePointer() > 0)
+ {
+ logger.debug(String.format("Completed flushing %s (%s) for commitlog position %s",
+ writer.getFilename(),
- FBUtilities.prettyPrintMemory(writer.getOnDiskFilePointer()),
++ FBUtilities.prettyPrintMemory(writer.getFilePointer()),
+ commitLogUpperBound));
- // sstables should contain non-repaired data.
- ssTables = writer.finish(true);
- }
- else
- {
- logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
- writer.getFilename(), context);
- writer.abort();
- ssTables = null;
- }
- // temp sstables should contain non-repaired data.
- ssTable = writer.finish(true);
++ // sstables should contain non-repaired data.
++ ssTables = writer.finish(true);
+ }
+ else
+ {
+ logger.debug("Completed flushing {}; nothing needed to be retained. Commitlog position was {}",
+ writer.getFilename(), commitLogUpperBound);
+ writer.abort();
- ssTable = null;
++ ssTables = Collections.emptyList();
+ }
- if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
+ if (heavilyContendedRowCount > 0)
- logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, rows.size(), Memtable.this.toString()));
++ logger.trace(String.format("High update contention in %d/%d partitions of %s ", heavilyContendedRowCount, partitions.size(), Memtable.this.toString()));
- return ssTables;
- }
- return ssTable;
++ return ssTables;
}
+ }
- @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
- public SSTableTxnWriter createFlushWriter(String filename,
- PartitionColumns columns,
- EncodingStats stats)
- private SSTableWriter createFlushWriter(String filename)
++ @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
++ public SSTableTxnWriter createFlushWriter(String filename,
++ PartitionColumns columns,
++ EncodingStats stats)
+ {
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
- .commitLogLowerBound(commitLogLowerBound.get())
- .commitLogUpperBound(commitLogUpperBound.get());
- return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
++ // we operate "offline" here, as we expose the resulting reader consciously when done
++ // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
++ LifecycleTransaction txn = null;
++ try
+ {
- // we operate "offline" here, as we expose the resulting reader consciously when done
- // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour in LifecycleTransaction)
- LifecycleTransaction txn = null;
- try
- {
- txn = LifecycleTransaction.offline(OperationType.FLUSH);
- MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- return new SSTableTxnWriter(txn,
- cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
- (long) partitions.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- sstableMetadataCollector,
- new SerializationHeader(true, cfs.metadata, columns, stats),
- txn));
- }
- catch (Throwable t)
- {
- if (txn != null)
- txn.close();
- throw t;
- }
++ txn = LifecycleTransaction.offline(OperationType.FLUSH);
++ MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
++ .commitLogLowerBound(commitLogLowerBound.get())
++ .commitLogUpperBound(commitLogUpperBound.get());
++
++ return new SSTableTxnWriter(txn,
++ cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
++ (long) partitions.size(),
++ ActiveRepairService.UNREPAIRED_SSTABLE,
++ sstableMetadataCollector,
++ new SerializationHeader(true, cfs.metadata, columns, stats),
++ txn));
++ }
++ catch (Throwable t)
++ {
++ if (txn != null)
++ txn.close();
++ throw t;
+ }
}
private static int estimateRowOverhead(final int count)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
index 7169b2f,a58aeb4..f45a47a
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java
@@@ -35,9 -33,6 +35,8 @@@ import com.google.common.base.Throwable
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
- import com.google.common.collect.Ordering;
+import com.google.common.util.concurrent.Uninterruptibles;
+
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -46,18 -43,16 +45,17 @@@ import org.apache.cassandra.concurrent.
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
- import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.compress.CompressionParameters;
+import org.apache.cassandra.io.util.FileSegmentInputStream;
+import org.apache.cassandra.io.util.RebufferingInputStream;
+import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.io.compress.ICompressor;
-import org.apache.cassandra.io.util.ByteBufferDataInput;
-import org.apache.cassandra.io.util.FastByteArrayInputStream;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.CRC32Factory;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.WrappedRunnable;
@@@ -76,9 -69,9 +74,9 @@@ public class CommitLogReplaye
private final List<Future<?>> futures;
private final Map<UUID, AtomicInteger> invalidMutations;
private final AtomicInteger replayedCount;
- private final Map<UUID, ReplayPosition> cfPositions;
+ private final Map<UUID, ReplayPosition.ReplayFilter> cfPersisted;
private final ReplayPosition globalPosition;
- private final ICRC32 checksum;
+ private final CRC32 checksum;
private byte[] buffer;
private byte[] uncompressedBuffer;
@@@ -94,8 -87,8 +92,8 @@@
this.invalidMutations = new HashMap<UUID, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it to be a reference.
this.replayedCount = new AtomicInteger();
- this.checksum = CRC32Factory.instance.create();
+ this.checksum = new CRC32();
- this.cfPositions = cfPositions;
+ this.cfPersisted = cfPersisted;
this.globalPosition = globalPosition;
this.replayFilter = replayFilter;
this.archiver = commitLog.archiver;
@@@ -104,17 -97,12 +102,12 @@@
public static CommitLogReplayer construct(CommitLog commitLog)
{
// compute per-CF and global replay positions
- Map<UUID, ReplayPosition> cfPositions = new HashMap<UUID, ReplayPosition>();
- Ordering<ReplayPosition> replayPositionOrdering = Ordering.from(ReplayPosition.comparator);
+ Map<UUID, ReplayPosition.ReplayFilter> cfPersisted = new HashMap<>();
ReplayFilter replayFilter = ReplayFilter.create();
+ ReplayPosition globalPosition = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
- // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the Ordering.min call
- // below: gRP will return NONE if there are no flushed sstables, which is important to have in the
- // list (otherwise we'll just start replay from the first flush position that we do have, which is not correct).
- ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables(SSTableSet.CANONICAL));
-
- // but, if we've truncted the cf in question, then we need to need to start replay after the truncation
+ // but, if we've truncated the cf in question, then we need to need to start replay after the truncation
ReplayPosition truncatedAt = SystemKeyspace.getTruncatedPosition(cfs.metadata.cfId);
if (truncatedAt != null)
{
@@@ -584,16 -580,12 +592,14 @@@
// or c) are part of a cf that was dropped.
// Keep in mind that the cf.name() is suspect. do every thing based on the cfid instead.
Mutation newMutation = null;
- for (ColumnFamily columnFamily : replayFilter.filter(mutation))
+ for (PartitionUpdate update : replayFilter.filter(mutation))
{
- if (Schema.instance.getCF(columnFamily.id()) == null)
+ if (Schema.instance.getCF(update.metadata().cfId) == null)
continue; // dropped
- ReplayPosition rp = cfPositions.get(update.metadata().cfId);
-
- if (shouldReplay(columnFamily.id(), new ReplayPosition(desc.id, entryLocation)))
+ // replay if current segment is newer than last flushed one or,
+ // if it is the last known segment, if we are after the replay position
- if (desc.id > rp.segment || (desc.id == rp.segment && entryLocation > rp.position))
++ if (shouldReplay(update.metadata().cfId, new ReplayPosition(desc.id, entryLocation)))
{
if (newMutation == null)
newMutation = new Mutation(mutation.getKeyspaceName(), mutation.key());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index 28416f3,17802ad..0b21763
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@@ -17,11 -17,12 +17,11 @@@
*/
package org.apache.cassandra.db.commitlog;
-import java.io.DataInput;
import java.io.IOException;
- import java.util.Comparator;
+ import java.util.Map;
+ import java.util.NavigableMap;
+ import java.util.TreeMap;
- import com.google.common.base.Function;
- import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index c205d5c,575aa51..8928db5
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@@ -237,13 -228,10 +237,10 @@@ public abstract class AbstractCompactio
* Handle a flushed memtable.
*
* @param memtable the flushed memtable
- * @param sstable the written sstable. can be null if the memtable was clean.
+ * @param sstables the written sstables. can be null or empty if the memtable was clean.
*/
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index 82fd872,0000000..f1127c9
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@@ -1,504 -1,0 +1,501 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.compaction;
+
+
+import java.util.*;
+import java.util.concurrent.Callable;
+
+import com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Memtable;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.notifications.*;
+import org.apache.cassandra.schema.CompactionParams;
+import org.apache.cassandra.service.ActiveRepairService;
+
+/**
+ * Manages the compaction strategies.
+ *
+ * Currently has two instances of actual compaction strategies - one for repaired data and one for
+ * unrepaired data. This is done to be able to totally separate the different sets of sstables.
+ */
+public class CompactionStrategyManager implements INotificationConsumer
+{
+ private static final Logger logger = LoggerFactory.getLogger(CompactionStrategyManager.class);
+ private final ColumnFamilyStore cfs;
+ private volatile AbstractCompactionStrategy repaired;
+ private volatile AbstractCompactionStrategy unrepaired;
+ private volatile boolean enabled = true;
+ public boolean isActive = true;
+ private volatile CompactionParams params;
+ /*
+ We keep a copy of the schema compaction parameters here to be able to decide if we
+ should update the compaction strategy in maybeReloadCompactionStrategy() due to an ALTER.
+
+ If a user changes the local compaction strategy and then later ALTERs a compaction parameter,
+ we will use the new compaction parameters.
+ */
+ private CompactionParams schemaCompactionParams;
+
+ public CompactionStrategyManager(ColumnFamilyStore cfs)
+ {
+ cfs.getTracker().subscribe(this);
+ logger.trace("{} subscribed to the data tracker.", this);
+ this.cfs = cfs;
+ reload(cfs.metadata);
+ params = cfs.metadata.params.compaction;
+ enabled = params.isEnabled();
+ }
+
+ /**
+ * Return the next background task
+ *
+ * Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks)
+ *
+ */
+ public synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
+ {
+ if (!isEnabled())
+ return null;
+
+ maybeReload(cfs.metadata);
+
+ if (repaired.getEstimatedRemainingTasks() > unrepaired.getEstimatedRemainingTasks())
+ {
+ AbstractCompactionTask repairedTask = repaired.getNextBackgroundTask(gcBefore);
+ if (repairedTask != null)
+ return repairedTask;
+ return unrepaired.getNextBackgroundTask(gcBefore);
+ }
+ else
+ {
+ AbstractCompactionTask unrepairedTask = unrepaired.getNextBackgroundTask(gcBefore);
+ if (unrepairedTask != null)
+ return unrepairedTask;
+ return repaired.getNextBackgroundTask(gcBefore);
+ }
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled && isActive;
+ }
+
+ public synchronized void resume()
+ {
+ isActive = true;
+ }
+
+ /**
+ * pause compaction while we cancel all ongoing compactions
+ *
+ * Separate call from enable/disable to not have to save the enabled-state externally
+ */
+ public synchronized void pause()
+ {
+ isActive = false;
+ }
+
+
+ private void startup()
+ {
+ for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
+ {
+ if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+ getCompactionStrategyFor(sstable).addSSTable(sstable);
+ }
+ repaired.startup();
+ unrepaired.startup();
+ }
+
+ /**
+ * return the compaction strategy for the given sstable
+ *
+ * returns differently based on the repaired status
+ * @param sstable
+ * @return
+ */
+ private AbstractCompactionStrategy getCompactionStrategyFor(SSTableReader sstable)
+ {
+ if (sstable.isRepaired())
+ return repaired;
+ else
+ return unrepaired;
+ }
+
+ public void shutdown()
+ {
+ isActive = false;
+ repaired.shutdown();
+ unrepaired.shutdown();
+ }
+
+ public synchronized void maybeReload(CFMetaData metadata)
+ {
+ // compare the old schema configuration to the new one, ignore any locally set changes.
+ if (metadata.params.compaction.equals(schemaCompactionParams))
+ return;
+ reload(metadata);
+ }
+
+ /**
+ * Reload the compaction strategies
+ *
+ * Called after changing configuration and at startup.
+ * @param metadata
+ */
+ public synchronized void reload(CFMetaData metadata)
+ {
+ boolean disabledWithJMX = !enabled && shouldBeEnabled();
+ setStrategy(metadata.params.compaction);
+ schemaCompactionParams = metadata.params.compaction;
+
+ if (disabledWithJMX || !shouldBeEnabled())
+ disable();
+ else
+ enable();
+ startup();
+ }
+
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
+ {
- cfs.getTracker().replaceFlushed(memtable, sstables);
- if (sstables != null && !sstables.isEmpty())
- CompactionManager.instance.submitBackground(cfs);
+ }
+
+ public int getUnleveledSSTables()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int count = 0;
+ count += ((LeveledCompactionStrategy)repaired).getLevelSize(0);
+ count += ((LeveledCompactionStrategy)unrepaired).getLevelSize(0);
+ return count;
+ }
+ return 0;
+ }
+
+ public synchronized int[] getSSTableCountPerLevel()
+ {
+ if (repaired instanceof LeveledCompactionStrategy && unrepaired instanceof LeveledCompactionStrategy)
+ {
+ int [] res = new int[LeveledManifest.MAX_LEVEL_COUNT];
+ int[] repairedCountPerLevel = ((LeveledCompactionStrategy) repaired).getAllLevelSize();
+ res = sumArrays(res, repairedCountPerLevel);
+ int[] unrepairedCountPerLevel = ((LeveledCompactionStrategy) unrepaired).getAllLevelSize();
+ res = sumArrays(res, unrepairedCountPerLevel);
+ return res;
+ }
+ return null;
+ }
+
+ private static int[] sumArrays(int[] a, int[] b)
+ {
+ int[] res = new int[Math.max(a.length, b.length)];
+ for (int i = 0; i < res.length; i++)
+ {
+ if (i < a.length && i < b.length)
+ res[i] = a[i] + b[i];
+ else if (i < a.length)
+ res[i] = a[i];
+ else
+ res[i] = b[i];
+ }
+ return res;
+ }
+
+ public boolean shouldDefragment()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.shouldDefragment();
+ }
+
+ public Directories getDirectories()
+ {
+ assert repaired.getClass().equals(unrepaired.getClass());
+ return repaired.getDirectories();
+ }
+
+ public synchronized void handleNotification(INotification notification, Object sender)
+ {
+ if (notification instanceof SSTableAddedNotification)
+ {
+ SSTableAddedNotification flushedNotification = (SSTableAddedNotification) notification;
+ for (SSTableReader sstable : flushedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repaired.addSSTable(sstable);
+ else
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableListChangedNotification)
+ {
+ SSTableListChangedNotification listChangedNotification = (SSTableListChangedNotification) notification;
+ Set<SSTableReader> repairedRemoved = new HashSet<>();
+ Set<SSTableReader> repairedAdded = new HashSet<>();
+ Set<SSTableReader> unrepairedRemoved = new HashSet<>();
+ Set<SSTableReader> unrepairedAdded = new HashSet<>();
+
+ for (SSTableReader sstable : listChangedNotification.removed)
+ {
+ if (sstable.isRepaired())
+ repairedRemoved.add(sstable);
+ else
+ unrepairedRemoved.add(sstable);
+ }
+ for (SSTableReader sstable : listChangedNotification.added)
+ {
+ if (sstable.isRepaired())
+ repairedAdded.add(sstable);
+ else
+ unrepairedAdded.add(sstable);
+ }
+ if (!repairedRemoved.isEmpty())
+ {
+ repaired.replaceSSTables(repairedRemoved, repairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : repairedAdded)
+ repaired.addSSTable(sstable);
+ }
+
+ if (!unrepairedRemoved.isEmpty())
+ {
+ unrepaired.replaceSSTables(unrepairedRemoved, unrepairedAdded);
+ }
+ else
+ {
+ for (SSTableReader sstable : unrepairedAdded)
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ else if (notification instanceof SSTableRepairStatusChanged)
+ {
+ for (SSTableReader sstable : ((SSTableRepairStatusChanged) notification).sstable)
+ {
+ if (sstable.isRepaired())
+ {
+ unrepaired.removeSSTable(sstable);
+ repaired.addSSTable(sstable);
+ }
+ else
+ {
+ repaired.removeSSTable(sstable);
+ unrepaired.addSSTable(sstable);
+ }
+ }
+ }
+ else if (notification instanceof SSTableDeletingNotification)
+ {
+ SSTableReader sstable = ((SSTableDeletingNotification)notification).deleting;
+ if (sstable.isRepaired())
+ repaired.removeSSTable(sstable);
+ else
+ unrepaired.removeSSTable(sstable);
+ }
+ }
+
+ public void enable()
+ {
+ if (repaired != null)
+ repaired.enable();
+ if (unrepaired != null)
+ unrepaired.enable();
+ // enable this last to make sure the strategies are ready to get calls.
+ enabled = true;
+ }
+
+ public void disable()
+ {
+ // disable this first avoid asking disabled strategies for compaction tasks
+ enabled = false;
+ if (repaired != null)
+ repaired.disable();
+ if (unrepaired != null)
+ unrepaired.disable();
+ }
+
+ /**
+ * Create ISSTableScanner from the given sstables
+ *
+ * Delegates the call to the compaction strategies to allow LCS to create a scanner
+ * @param sstables
+ * @param ranges
+ * @return
+ */
+ @SuppressWarnings("resource")
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
+ {
+ List<SSTableReader> repairedSSTables = new ArrayList<>();
+ List<SSTableReader> unrepairedSSTables = new ArrayList<>();
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable.isRepaired())
+ repairedSSTables.add(sstable);
+ else
+ unrepairedSSTables.add(sstable);
+ }
+
+ Set<ISSTableScanner> scanners = new HashSet<>(sstables.size());
+ AbstractCompactionStrategy.ScannerList repairedScanners = repaired.getScanners(repairedSSTables, ranges);
+ AbstractCompactionStrategy.ScannerList unrepairedScanners = unrepaired.getScanners(unrepairedSSTables, ranges);
+ scanners.addAll(repairedScanners.scanners);
+ scanners.addAll(unrepairedScanners.scanners);
+ return new AbstractCompactionStrategy.ScannerList(new ArrayList<>(scanners));
+ }
+
+ public synchronized AbstractCompactionStrategy.ScannerList getScanners(Collection<SSTableReader> sstables)
+ {
+ return getScanners(sstables, null);
+ }
+
+ public Collection<Collection<SSTableReader>> groupSSTablesForAntiCompaction(Collection<SSTableReader> sstablesToGroup)
+ {
+ return unrepaired.groupSSTablesForAntiCompaction(sstablesToGroup);
+ }
+
+ public long getMaxSSTableBytes()
+ {
+ return unrepaired.getMaxSSTableBytes();
+ }
+
+ public AbstractCompactionTask getCompactionTask(LifecycleTransaction txn, int gcBefore, long maxSSTableBytes)
+ {
+ return getCompactionStrategyFor(txn.originals().iterator().next()).getCompactionTask(txn, gcBefore, maxSSTableBytes);
+ }
+
+ public Collection<AbstractCompactionTask> getMaximalTasks(final int gcBefore, final boolean splitOutput)
+ {
+ // runWithCompactionsDisabled cancels active compactions and disables them, then we are able
+ // to make the repaired/unrepaired strategies mark their own sstables as compacting. Once the
+ // sstables are marked the compactions are re-enabled
+ return cfs.runWithCompactionsDisabled(new Callable<Collection<AbstractCompactionTask>>()
+ {
+ @Override
+ public Collection<AbstractCompactionTask> call() throws Exception
+ {
+ synchronized (CompactionStrategyManager.this)
+ {
+ Collection<AbstractCompactionTask> repairedTasks = repaired.getMaximalTask(gcBefore, splitOutput);
+ Collection<AbstractCompactionTask> unrepairedTasks = unrepaired.getMaximalTask(gcBefore, splitOutput);
+
+ if (repairedTasks == null && unrepairedTasks == null)
+ return null;
+
+ if (repairedTasks == null)
+ return unrepairedTasks;
+ if (unrepairedTasks == null)
+ return repairedTasks;
+
+ List<AbstractCompactionTask> tasks = new ArrayList<>();
+ tasks.addAll(repairedTasks);
+ tasks.addAll(unrepairedTasks);
+ return tasks;
+ }
+ }
+ }, false, false);
+ }
+
+ public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
+ {
+ return getCompactionStrategyFor(sstables.iterator().next()).getUserDefinedTask(sstables, gcBefore);
+ }
+
+ public int getEstimatedRemainingTasks()
+ {
+ int tasks = 0;
+ tasks += repaired.getEstimatedRemainingTasks();
+ tasks += unrepaired.getEstimatedRemainingTasks();
+
+ return tasks;
+ }
+
+ public boolean shouldBeEnabled()
+ {
+ return params.isEnabled();
+ }
+
+ public String getName()
+ {
+ return unrepaired.getName();
+ }
+
+ public List<AbstractCompactionStrategy> getStrategies()
+ {
+ return Arrays.asList(repaired, unrepaired);
+ }
+
+ public synchronized void setNewLocalCompactionStrategy(CompactionParams params)
+ {
+ logger.info("Switching local compaction strategy from {} to {}}", this.params, params);
+ setStrategy(params);
+ if (shouldBeEnabled())
+ enable();
+ else
+ disable();
+ startup();
+ }
+
+ private void setStrategy(CompactionParams params)
+ {
+ if (repaired != null)
+ repaired.shutdown();
+ if (unrepaired != null)
+ unrepaired.shutdown();
+ repaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ unrepaired = CFMetaData.createCompactionStrategyInstance(cfs, params);
+ this.params = params;
+ }
+
+ public CompactionParams getCompactionParams()
+ {
+ return params;
+ }
+
+ public boolean onlyPurgeRepairedTombstones()
+ {
+ return Boolean.parseBoolean(params.options().get(AbstractCompactionStrategy.ONLY_PURGE_REPAIRED_TOMBSTONES));
+ }
+
+ public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn)
+ {
+ if (repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
+ {
+ return unrepaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ else
+ {
+ return repaired.createSSTableMultiWriter(descriptor, keyCount, repairedAt, collector, header, txn);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index c09d49c,5d5701f..16090a1
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@@ -195,10 -199,12 +199,12 @@@ public class Tracke
public void reset()
{
view.set(new View(
- !isDummy() ? ImmutableList.of(new Memtable(cfstore)) : Collections.<Memtable>emptyList(),
+ !isDummy() ? ImmutableList.of(new Memtable(new AtomicReference<>(CommitLog.instance.getContext()), cfstore))
+ : ImmutableList.<Memtable>of(),
ImmutableList.<Memtable>of(),
Collections.<SSTableReader, SSTableReader>emptyMap(),
- Collections.<SSTableReader>emptySet(),
+ Collections.<SSTableReader, SSTableReader>emptyMap(),
+ Collections.<SSTableReader>emptySet(),
SSTableIntervalTree.empty()));
}
@@@ -327,10 -314,10 +332,10 @@@
apply(View.markFlushing(memtable));
}
- public void replaceFlushed(Memtable memtable, SSTableReader sstable)
+ public void replaceFlushed(Memtable memtable, Collection<SSTableReader> sstables)
{
assert !isDummy();
- if (sstables == null || sstables.isEmpty())
- if (sstable == null)
++ if (sstables.isEmpty())
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
// if it's null, we don't care what state the cfstore is in, we just replace it and continue
@@@ -338,27 -325,58 +343,57 @@@
return;
}
- sstable.setupKeyCache();
+ sstables.forEach(SSTableReader::setupOnline);
// back up before creating a new Snapshot (which makes the new one eligible for compaction)
- maybeIncrementallyBackup(sstable);
+ maybeIncrementallyBackup(sstables);
- apply(View.replaceFlushed(memtable, sstable));
+ apply(View.replaceFlushed(memtable, sstables));
Throwable fail;
- fail = updateSizeTracking(emptySet(), singleton(sstable), null);
+ fail = updateSizeTracking(emptySet(), sstables, null);
- // TODO: if we're invalidated, should we notifyadded AND removed, or just skip both?
- fail = notifyAdded(sstables, fail);
-
- if (!isDummy() && !cfstore.isValid())
- dropSSTables();
maybeFail(fail);
}
+ /**
+ * permit compaction of the provided sstable; this translates to notifying compaction
+ * strategies of its existence, and potentially submitting a background task
+ */
- public void permitCompactionOfFlushed(SSTableReader sstable)
++ public void permitCompactionOfFlushed(Collection<SSTableReader> sstables)
+ {
- if (sstable == null)
++ if (sstables.isEmpty())
+ return;
+
- apply(View.permitCompactionOfFlushed(sstable));
++ apply(View.permitCompactionOfFlushed(sstables));
+
+ if (isDummy())
+ return;
+
+ if (cfstore.isValid())
+ {
- notifyAdded(sstable);
++ notifyAdded(sstables);
+ CompactionManager.instance.submitBackground(cfstore);
+ }
+ else
+ {
+ dropSSTables();
+ }
+ }
-
// MISCELLANEOUS public utility calls
+ public Set<SSTableReader> getSSTables()
+ {
+ return view.get().sstables;
+ }
+
+ public Iterable<SSTableReader> getPermittedToCompact()
+ {
+ View view = this.view.get();
+ return filter(view.sstables, not(in(view.premature)));
+ }
+
public Set<SSTableReader> getCompacting()
{
return view.get().compacting;
[13/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index b62c7e3,fba1627..17062b4
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -33,16 -33,14 +33,16 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.utils.Interval;
import static com.google.common.base.Predicates.equalTo;
++import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableList.copyOf;
import static com.google.common.collect.ImmutableList.of;
import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.filter;
-import static java.util.Collections.singleton;
+import static com.google.common.collect.Iterables.transform;
- import static java.util.Collections.singleton;
import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.filterOut;
import static org.apache.cassandra.db.lifecycle.Helpers.replace;
/**
@@@ -68,18 -66,18 +68,19 @@@ public class Vie
* flushed. In chronologically ascending order.
*/
public final List<Memtable> flushingMemtables;
- public final Set<SSTableReader> compacting;
- public final Set<SSTableReader> sstables;
- public final Set<SSTableReader> premature;
+ final Set<SSTableReader> compacting;
+ final Set<SSTableReader> sstables;
++ final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
// the one we made our decision to compact against.
- public final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> compactingMap;
- public final SSTableIntervalTree intervalTree;
+ final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, SSTableIntervalTree intervalTree)
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
++ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@@ -92,8 -90,8 +93,9 @@@
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
- this.compacting = compacting;
+ this.compactingMap = compacting;
+ this.compacting = compactingMap.keySet();
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@@ -238,8 -158,8 +240,8 @@@
{
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
- replace(view.compacting, unmark, mark),
+ replace(view.compactingMap, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@@ -270,7 -190,7 +272,7 @@@
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap,
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
++ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@@ -285,7 -205,7 +287,7 @@@
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -304,7 -224,7 +306,7 @@@
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -319,17 -239,35 +321,34 @@@
List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
-- if (flushed == null)
++ if (flushed == null || flushed.isEmpty())
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compactingMap, view.intervalTree);
- view.compacting, view.premature, view.intervalTree);
++ view.compactingMap, view.premature, view.intervalTree);
- Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
- Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), flushed);
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compactingMap,
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, emptySet(), flushed);
++ Set<SSTableReader> premature = replace(view.premature, emptySet(), flushed);
++ return new View(view.liveMemtables, flushingMemtables, sstableMap, compactingMap, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
- static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
++ static Function<View, View> permitCompactionOfFlushed(final Collection<SSTableReader> readers)
+ {
++ Set<SSTableReader> expectAndRemove = ImmutableSet.copyOf(readers);
+ return new Function<View, View>()
+ {
-
- @Nullable
+ public View apply(View view)
+ {
- Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
- Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
- return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
++ Set<SSTableReader> premature = replace(view.premature, expectAndRemove, emptySet());
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, expectAndRemove, emptySet());
++ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compactingMap, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/view/TableViews.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/view/TableViews.java
index 893bdd5,0000000..7feb67c
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@@ -1,481 -1,0 +1,481 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.view;
+
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.commitlog.ReplayPosition;
+import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+
+/**
+ * Groups all the views for a given table.
+ */
+public class TableViews extends AbstractCollection<View>
+{
+ private final CFMetaData baseTableMetadata;
+
+ // We need this to be thread-safe, but the number of times this is changed (when a view is created in the keyspace)
+ // massively exceeds the number of time it's read (for every mutation on the keyspace), so a copy-on-write list is the best option.
+ private final List<View> views = new CopyOnWriteArrayList();
+
+ public TableViews(CFMetaData baseTableMetadata)
+ {
+ this.baseTableMetadata = baseTableMetadata;
+ }
+
+ public int size()
+ {
+ return views.size();
+ }
+
+ public Iterator<View> iterator()
+ {
+ return views.iterator();
+ }
+
+ public boolean contains(String viewName)
+ {
+ return Iterables.any(views, view -> view.name.equals(viewName));
+ }
+
+ public boolean add(View view)
+ {
+ // We should have validated that there is no existing view with this name at this point
+ assert !contains(view.name);
+ return views.add(view);
+ }
+
+ public Iterable<ColumnFamilyStore> allViewsCfs()
+ {
+ Keyspace keyspace = Keyspace.open(baseTableMetadata.ksName);
+ return Iterables.transform(views, view -> keyspace.getColumnFamilyStore(view.getDefinition().viewName));
+ }
+
+ public void forceBlockingFlush()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.forceBlockingFlush();
+ }
+
+ public void dumpMemtables()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.dumpMemtable();
+ }
+
- public void truncateBlocking(long truncatedAt)
++ public void truncateBlocking(ReplayPosition replayAfter, long truncatedAt)
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ {
- ReplayPosition replayAfter = viewCfs.discardSSTables(truncatedAt);
++ viewCfs.discardSSTables(truncatedAt);
+ SystemKeyspace.saveTruncationRecord(viewCfs, truncatedAt, replayAfter);
+ }
+ }
+
+ public void removeByName(String viewName)
+ {
+ views.removeIf(v -> v.name.equals(viewName));
+ }
+
+ /**
+ * Calculates and pushes updates to the views replicas. The replicas are determined by
+ * {@link ViewUtils#getViewNaturalEndpoint(String, Token, Token)}.
+ *
+ * @param update an update on the base table represented by this object.
+ * @param writeCommitLog whether we should write the commit log for the view updates.
+ * @param baseComplete time from epoch in ms that the local base mutation was (or will be) completed
+ */
+ public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLog, AtomicLong baseComplete)
+ {
+ assert update.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ Collection<View> views = updatedViews(update);
+ if (views.isEmpty())
+ return;
+
+ // Read modified rows
+ int nowInSec = FBUtilities.nowInSeconds();
+ SinglePartitionReadCommand command = readExistingRowsCommand(update, views, nowInSec);
+ if (command == null)
+ return;
+
+ ColumnFamilyStore cfs = Keyspace.openAndGetStore(update.metadata());
+ long start = System.nanoTime();
+ Collection<Mutation> mutations;
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ UnfilteredRowIterator existings = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command);
+ UnfilteredRowIterator updates = update.unfilteredIterator())
+ {
+ mutations = generateViewUpdates(views, updates, existings, nowInSec);
+ }
+ Keyspace.openAndGetStore(update.metadata()).metric.viewReadTime.update(System.nanoTime() - start, TimeUnit.NANOSECONDS);
+
+ if (!mutations.isEmpty())
+ StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete);
+ }
+
+ /**
+ * Given some updates on the base table of this object and the existing values for the rows affected by that update, generates the
+ * mutation to be applied to the provided views.
+ *
+ * @param views the views potentially affected by {@code updates}.
+ * @param updates the base table updates being applied.
+ * @param existings the existing values for the rows affected by {@code updates}. This is used to decide if a view is
+ * obsoleted by the update and should be removed, gather the values for columns that may not be part of the update if
+ * a new view entry needs to be created, and compute the minimal updates to be applied if the view entry isn't changed
+ * but has simply some updated values. This will be empty for view building as we want to assume anything we'll pass
+ * to {@code updates} is new.
+ * @param nowInSec the current time in seconds.
+ * @return the mutations to apply to the {@code views}. This can be empty.
+ */
+ public Collection<Mutation> generateViewUpdates(Collection<View> views, UnfilteredRowIterator updates, UnfilteredRowIterator existings, int nowInSec)
+ {
+ assert updates.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ List<ViewUpdateGenerator> generators = new ArrayList<>(views.size());
+ for (View view : views)
+ generators.add(new ViewUpdateGenerator(view, updates.partitionKey(), nowInSec));
+
+ DeletionTracker existingsDeletion = new DeletionTracker(existings.partitionLevelDeletion());
+ DeletionTracker updatesDeletion = new DeletionTracker(updates.partitionLevelDeletion());
+
+ /*
+ * We iterate through the updates and the existing rows in parallel. This allows us to know the consequence
+ * on the view of each update.
+ */
+ PeekingIterator<Unfiltered> existingsIter = Iterators.peekingIterator(existings);
+ PeekingIterator<Unfiltered> updatesIter = Iterators.peekingIterator(updates);
+
+ while (existingsIter.hasNext() && updatesIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.peek();
+ Unfiltered update = updatesIter.peek();
+
+ Row existingRow;
+ Row updateRow;
+ int cmp = baseTableMetadata.comparator.compare(update, existing);
+ if (cmp < 0)
+ {
+ // We have an update where there was nothing before
+ if (update.isRangeTombstoneMarker())
+ {
+ updatesDeletion.update(updatesIter.next());
+ continue;
+ }
+
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ existingRow = emptyRow(updateRow.clustering(), existingsDeletion.currentDeletion());
+ }
+ else if (cmp > 0)
+ {
+ // We have something existing but no update (which will happen either because it's a range tombstone marker in
+ // existing, or because we've fetched the existing row due to some partition/range deletion in the updates)
+ if (existing.isRangeTombstoneMarker())
+ {
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion());
+
+ // The way we build the read command used for existing rows, we should always have updatesDeletion.currentDeletion()
+ // that is not live, since we wouldn't have read the existing row otherwise. And we could assert that, but if we ever
+ // change the read method so that it can slightly over-read in some case, that would be an easily avoiding bug lurking,
+ // so we just handle the case.
+ if (updateRow == null)
+ continue;
+ }
+ else
+ {
+ // We're updating a row that had pre-existing data
+ if (update.isRangeTombstoneMarker())
+ {
+ assert existing.isRangeTombstoneMarker();
+ updatesDeletion.update(updatesIter.next());
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ assert !existing.isRangeTombstoneMarker();
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ }
+
+ addToViewUpdateGenerators(existingRow, updateRow, generators, nowInSec);
+ }
+
+ // We only care about more existing rows if the update deletion isn't live, i.e. if we had a partition deletion
+ if (!updatesDeletion.currentDeletion().isLive())
+ {
+ while (existingsIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.next();
+ // If it's a range tombstone, we don't care, we're only looking for existing entry that gets deleted by
+ // the new partition deletion
+ if (existing.isRangeTombstoneMarker())
+ continue;
+
+ Row existingRow = (Row)existing;
+ addToViewUpdateGenerators(existingRow, emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion()), generators, nowInSec);
+ }
+ }
+ while (updatesIter.hasNext())
+ {
+ Unfiltered update = updatesIter.next();
+ // If it's a range tombstone, it removes nothing pre-exisiting, so we can ignore it for view updates
+ if (update.isRangeTombstoneMarker())
+ continue;
+
+ Row updateRow = (Row)update;
+ addToViewUpdateGenerators(emptyRow(updateRow.clustering(), DeletionTime.LIVE), updateRow, generators, nowInSec);
+ }
+
+ return buildMutations(baseTableMetadata, generators);
+ }
+
+ /**
+ * Return the views that are potentially updated by the provided updates.
+ *
+ * @param updates the updates applied to the base table.
+ * @return the views affected by {@code updates}.
+ */
+ public Collection<View> updatedViews(PartitionUpdate updates)
+ {
+ List<View> matchingViews = new ArrayList<>(views.size());
+
+ for (View view : views)
+ {
+ ReadQuery selectQuery = view.getReadQuery();
+ if (!selectQuery.selectsKey(updates.partitionKey()))
+ continue;
+
+ matchingViews.add(view);
+ }
+ return matchingViews;
+ }
+
+ /**
+ * Returns the command to use to read the existing rows required to generate view updates for the provided base
+ * base updates.
+ *
+ * @param updates the base table updates being applied.
+ * @param views the views potentially affected by {@code updates}.
+ * @param nowInSec the current time in seconds.
+ * @return the command to use to read the base table rows required to generate view updates for {@code updates}.
+ */
+ private SinglePartitionReadCommand readExistingRowsCommand(PartitionUpdate updates, Collection<View> views, int nowInSec)
+ {
+ Slices.Builder sliceBuilder = null;
+ DeletionInfo deletionInfo = updates.deletionInfo();
+ CFMetaData metadata = updates.metadata();
+ DecoratedKey key = updates.partitionKey();
+ // TODO: This is subtle: we need to gather all the slices that we have to fetch between partition del, range tombstones and rows.
+ if (!deletionInfo.isLive())
+ {
+ sliceBuilder = new Slices.Builder(metadata.comparator);
+ // Everything covered by a deletion might invalidate an existing view entry, which means we must read it to know. In practice
+ // though, the views involved might filter some base table clustering columns, in which case we can restrict what we read
+ // using those restrictions.
+ // If there is a partition deletion, then we can simply take each slices from each view select filter. They may overlap but
+ // the Slices.Builder handles that for us. Note that in many case this will just involve reading everything (as soon as any
+ // view involved has no clustering restrictions for instance).
+ // For range tombstone, we should theoretically take the difference between the range tombstoned and the slices selected
+ // by every views, but as we don't an easy way to compute that right now, we keep it simple and just use the tombstoned
+ // range.
+ // TODO: we should improve that latter part.
+ if (!deletionInfo.getPartitionDeletion().isLive())
+ {
+ for (View view : views)
+ sliceBuilder.addAll(view.getSelectStatement().clusteringIndexFilterAsSlices());
+ }
+ else
+ {
+ assert deletionInfo.hasRanges();
+ Iterator<RangeTombstone> iter = deletionInfo.rangeIterator(false);
+ while (iter.hasNext())
+ sliceBuilder.add(iter.next().deletedSlice());
+ }
+ }
+
+ // We need to read every row that is updated, unless we can prove that it has no impact on any view entries.
+
+ // If we had some slices from the deletions above, we'll continue using that. Otherwise, it's more efficient to build
+ // a names query.
+ BTreeSet.Builder<Clustering> namesBuilder = sliceBuilder == null ? BTreeSet.builder(metadata.comparator) : null;
+ for (Row row : updates)
+ {
+ // Don't read the existing state if we can prove the update won't affect any views
+ if (!affectsAnyViews(key, row, views))
+ continue;
+
+ if (namesBuilder == null)
+ sliceBuilder.add(Slice.make(row.clustering()));
+ else
+ namesBuilder.add(row.clustering());
+ }
+
+ NavigableSet<Clustering> names = namesBuilder == null ? null : namesBuilder.build();
+ // If we have a slice builder, it means we had some deletions and we have to read. But if we had
+ // only row updates, it's possible none of them affected the views, in which case we have nothing
+ // to do.
+ if (names != null && names.isEmpty())
+ return null;
+
+ ClusteringIndexFilter clusteringFilter = names == null
+ ? new ClusteringIndexSliceFilter(sliceBuilder.build(), false)
+ : new ClusteringIndexNamesFilter(names, false);
+ // If we have more than one view, we should merge the queried columns by each views but to keep it simple we just
+ // include everything. We could change that in the future.
+ ColumnFilter queriedColumns = views.size() == 1
+ ? Iterables.getOnlyElement(views).getSelectStatement().queriedColumns()
+ : ColumnFilter.all(metadata);
+ // Note that the views could have restrictions on regular columns, but even if that's the case we shouldn't apply those
+ // when we read, because even if an existing row doesn't match the view filter, the update can change that in which
+ // case we'll need to know the existing content. There is also no easy way to merge those RowFilter when we have multiple views.
+ // TODO: we could still make sense to special case for when there is a single view and a small number of updates (and
+ // no deletions). Indeed, in that case we could check whether any of the update modify any of the restricted regular
+ // column, and if that's not the case we could use view filter. We keep it simple for now though.
+ RowFilter rowFilter = RowFilter.NONE;
+ return SinglePartitionReadCommand.create(metadata, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, clusteringFilter);
+ }
+
+ private boolean affectsAnyViews(DecoratedKey partitionKey, Row update, Collection<View> views)
+ {
+ for (View view : views)
+ {
+ if (view.mayBeAffectedBy(partitionKey, update))
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Given an existing base row and the update that we're going to apply to this row, generate the modifications
+ * to apply to MVs using the provided {@code ViewUpdateGenerator}s.
+ *
+ * @param existingBaseRow the base table row as it is before an update.
+ * @param updateBaseRow the newly updates made to {@code existingBaseRow}.
+ * @param generators the view update generators to add the new changes to.
+ * @param nowInSec the current time in seconds. Used to decide if data is live or not.
+ */
+ private static void addToViewUpdateGenerators(Row existingBaseRow, Row updateBaseRow, Collection<ViewUpdateGenerator> generators, int nowInSec)
+ {
+ // Having existing empty is useful, it just means we'll insert a brand new entry for updateBaseRow,
+ // but if we have no update at all, we shouldn't get there.
+ assert !updateBaseRow.isEmpty();
+
+ // We allow existingBaseRow to be null, which we treat the same as being empty as an small optimization
+ // to avoid allocating empty row objects when we know there was nothing existing.
+ Row mergedBaseRow = existingBaseRow == null ? updateBaseRow : Rows.merge(existingBaseRow, updateBaseRow, nowInSec);
+ for (ViewUpdateGenerator generator : generators)
+ generator.addBaseTableUpdate(existingBaseRow, mergedBaseRow);
+ }
+
+ private static Row emptyRow(Clustering clustering, DeletionTime deletion)
+ {
+ // Returning null for an empty row is slightly ugly, but the case where there is no pre-existing row is fairly common
+ // (especially when building the view), so we want to avoid a dummy allocation of an empty row every time.
+ // And MultiViewUpdateBuilder knows how to deal with that.
+ return deletion.isLive() ? null : BTreeRow.emptyDeletedRow(clustering, Row.Deletion.regular(deletion));
+ }
+
+ /**
+ * Extracts (and potentially groups) the mutations generated by the provided view update generator.
+ * Returns the mutation that needs to be done to the views given the base table updates
+ * passed to {@link #addBaseTableUpdate}.
+ *
+ * @param baseTableMetadata the metadata for the base table being updated.
+ * @param generators the generators from which to extract the view mutations from.
+ * @return the mutations created by all the generators in {@code generators}.
+ */
+ private Collection<Mutation> buildMutations(CFMetaData baseTableMetadata, List<ViewUpdateGenerator> generators)
+ {
+ // One view is probably common enough and we can optimize a bit easily
+ if (generators.size() == 1)
+ {
+ Collection<PartitionUpdate> updates = generators.get(0).generateViewUpdates();
+ List<Mutation> mutations = new ArrayList<>(updates.size());
+ for (PartitionUpdate update : updates)
+ mutations.add(new Mutation(update));
+ return mutations;
+ }
+
+ Map<DecoratedKey, Mutation> mutations = new HashMap<>();
+ for (ViewUpdateGenerator generator : generators)
+ {
+ for (PartitionUpdate update : generator.generateViewUpdates())
+ {
+ DecoratedKey key = update.partitionKey();
+ Mutation mutation = mutations.get(key);
+ if (mutation == null)
+ {
+ mutation = new Mutation(baseTableMetadata.ksName, key);
+ mutations.put(key, mutation);
+ }
+ mutation.add(update);
+ }
+ }
+ return mutations.values();
+ }
+
+ /**
+ * A simple helper that tracks for a given {@code UnfilteredRowIterator} what is the current deletion at any time of the
+ * iteration. It will be the currently open range tombstone deletion if there is one and the partition deletion otherwise.
+ */
+ private static class DeletionTracker
+ {
+ private final DeletionTime partitionDeletion;
+ private DeletionTime deletion;
+
+ public DeletionTracker(DeletionTime partitionDeletion)
+ {
+ this.partitionDeletion = partitionDeletion;
+ }
+
+ public void update(Unfiltered marker)
+ {
+ assert marker instanceof RangeTombstoneMarker;
+ RangeTombstoneMarker rtm = (RangeTombstoneMarker)marker;
+ this.deletion = rtm.isOpen(false)
+ ? rtm.openDeletionTime(false)
+ : null;
+ }
+
+ public DeletionTime currentDeletion()
+ {
+ return deletion == null ? partitionDeletion : deletion;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/Version.java
index 16829ab,41a83e1..d9e289c
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@@ -60,16 -52,8 +60,18 @@@ public abstract class Versio
public abstract boolean hasNewFileName();
+ public abstract boolean storeRows();
+
+ public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far
+
+ public abstract boolean hasOldBfHashOrder();
+
+ public abstract boolean hasCompactionAncestors();
+
+ public abstract boolean hasBoundaries();
+
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 1f2a98f,9244bbb..e0fb3b1
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@@ -111,7 -125,7 +111,7 @@@ public class BigFormat implements SSTab
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "ma";
- public static final String current_version = "lb";
++ public static final String current_version = "mb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@@ -121,10 -135,7 +121,12 @@@
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
+ // ma (3.0.0): swap bf hash order
+ // store rows natively
++ // mb (3.0.6): commit log lower bound included
+ //
+ // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@@ -134,53 -144,20 +136,56 @@@
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ public final boolean storeRows;
+ public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
+ public final boolean hasBoundaries;
+ /**
+ * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
+ * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
+ */
+ private final boolean hasOldBfHashOrder;
+ private final boolean hasCommitLogLowerBound;
- public BigVersion(String version)
+ /**
+ * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+ */
+ private final boolean hasCompactionAncestors;
+
+ BigVersion(String version)
{
- super(instance,version);
+ super(instance, version);
isLatestVersion = version.compareTo(current_version) == 0;
hasSamplingLevel = version.compareTo("ka") >= 0;
newStatsFile = version.compareTo("ka") >= 0;
- hasAllAdlerChecksums = version.compareTo("ka") >= 0;
+
+ //For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell
+ //PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684
+ ChecksumType checksumType = ChecksumType.CRC32;
+ if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.uncompressedChecksumType = checksumType;
+
+ checksumType = ChecksumType.CRC32;
+ if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.compressedChecksumType = checksumType;
+
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
+
newFileName = version.compareTo("la") >= 0;
- hasCommitLogLowerBound = version.compareTo("lb") >= 0;
+
+ hasOldBfHashOrder = version.compareTo("ma") < 0;
+ hasCompactionAncestors = version.compareTo("ma") < 0;
+ storeRows = version.compareTo("ma") >= 0;
+ correspondingMessagingVersion = storeRows
+ ? MessagingService.VERSION_30
+ : MessagingService.VERSION_21;
+
+ hasBoundaries = version.compareTo("ma") < 0;
++ hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0)
++ || version.compareTo("mb") >= 0;
}
@Override
@@@ -243,25 -202,12 +248,30 @@@
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
+ public boolean storeRows()
+ {
+ return storeRows;
+ }
+
+ @Override
+ public int correspondingMessagingVersion()
+ {
+ return correspondingMessagingVersion;
+ }
+
+ @Override
+ public boolean hasBoundaries()
+ {
+ return hasBoundaries;
+ }
+
+ @Override
public boolean isCompatible()
{
return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index 4f4c04b,c8e6ee8..ef3453a
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@@ -72,14 -72,22 +72,23 @@@ public class CompactionMetadata extend
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, CompactionMetadata component) throws IOException
{
- int size = 0;
- size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
- for (int g : component.ancestors)
- size += TypeSizes.NATIVE.sizeof(g);
++ int sz = 0;
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ sz = 4;
++ }
byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
- return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
- size += TypeSizes.NATIVE.sizeof(serializedCardinality.length) + serializedCardinality.length;
- return size;
++ return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length + sz;
}
- public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
{
- out.writeInt(component.ancestors.size());
- for (int g : component.ancestors)
- out.writeInt(g);
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ out.writeInt(0);
++ }
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 2ed845c,bfeb930..4561520
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@@ -53,24 -51,28 +53,26 @@@ public class LegacyMetadataSerializer e
assert validation != null && stats != null && compaction != null && validation.partitioner != null;
- EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(stats.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
out.writeDouble(validation.bloomFilterFPChance);
out.writeDouble(stats.compressionRatio);
out.writeUTF(validation.partitioner);
- out.writeInt(compaction.ancestors.size());
- for (Integer g : compaction.ancestors)
- out.writeInt(g);
+ out.writeInt(0); // compaction ancestors
StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
out.writeInt(stats.sstableLevel);
- out.writeInt(stats.minColumnNames.size());
- for (ByteBuffer columnName : stats.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(stats.maxColumnNames.size());
- for (ByteBuffer columnName : stats.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(stats.minClusteringValues.size());
+ for (ByteBuffer value : stats.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(stats.maxClusteringValues.size());
+ for (ByteBuffer value : stats.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@@ -88,11 -90,12 +90,12 @@@
}
else
{
- try (DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile))))
+ try (DataInputStreamPlus in = new DataInputStreamPlus(new BufferedInputStream(new FileInputStream(statsFile))))
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@@ -115,21 -120,21 +118,25 @@@
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
+
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
new ValidationMetadata(partitioner, bloomFilterFPChance));
if (types.contains(MetadataType.STATS))
components.put(MetadataType.STATS,
- new StatsMetadata(rowSizes,
+ new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ Integer.MAX_VALUE,
maxLocalDeletionTime,
+ 0,
+ Integer.MAX_VALUE,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index c2b0caf,579ff7a..53cf0b0
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -26,16 -26,17 +26,16 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+ import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.io.sstable.ColumnNameHelper;
-import org.apache.cassandra.io.sstable.ColumnStats;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
+import org.apache.cassandra.db.rows.Cell;
- import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
@@@ -67,9 -67,10 +67,10 @@@ public class MetadataCollector implemen
public static StatsMetadata defaultStatsMetadata()
{
- return new StatsMetadata(defaultRowSizeHistogram(),
- defaultColumnCountHistogram(),
+ return new StatsMetadata(defaultPartitionSizeHistogram(),
+ defaultCellPerPartitionCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@@ -82,26 -80,23 +83,27 @@@
Collections.<ByteBuffer>emptyList(),
Collections.<ByteBuffer>emptyList(),
true,
- ActiveRepairService.UNREPAIRED_SSTABLE);
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ -1,
+ -1);
}
- protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
- protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
+ protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
+ // TODO: cound the number of row per partition (either with the number of cells, or instead)
+ protected EstimatedHistogram estimatedCellPerPartitionCount = defaultCellPerPartitionCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
- protected long minTimestamp = Long.MAX_VALUE;
- protected long maxTimestamp = Long.MIN_VALUE;
- protected int maxLocalDeletionTime = Integer.MIN_VALUE;
+ protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker();
+ protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
+ protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
protected double compressionRatio = NO_COMPRESSION_RATIO;
- protected Set<Integer> ancestors = new HashSet<>();
protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram();
protected int sstableLevel;
- protected List<ByteBuffer> minColumnNames = Collections.emptyList();
- protected List<ByteBuffer> maxColumnNames = Collections.emptyList();
+ protected ByteBuffer[] minClusteringValues;
+ protected ByteBuffer[] maxClusteringValues;
protected boolean hasLegacyCounterShards = false;
+ protected long totalColumnsSet;
+ protected long totalRows;
/**
* Default cardinality estimation method is to use HyperLogLog++.
@@@ -110,22 -105,52 +112,38 @@@
* See CASSANDRA-5906 for detail.
*/
protected ICardinality cardinality = new HyperLogLogPlus(13, 25);
- private final CellNameType columnNameComparator;
+ private final ClusteringComparator comparator;
- public MetadataCollector(CellNameType columnNameComparator)
+ public MetadataCollector(ClusteringComparator comparator)
{
- this.columnNameComparator = columnNameComparator;
+ this.comparator = comparator;
+
+ this.minClusteringValues = new ByteBuffer[comparator.size()];
+ this.maxClusteringValues = new ByteBuffer[comparator.size()];
}
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level, boolean skipAncestors)
+ public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
{
- this(columnNameComparator);
+ this(comparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
- // Get the max timestamp of the precompacted sstables
- // and adds generation of live ancestors
- if (!skipAncestors)
- {
- for (SSTableReader sstable : sstables)
- {
- addAncestor(sstable.descriptor.generation);
- for (Integer i : sstable.getAncestors())
- if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
- addAncestor(i);
- }
- }
- }
-
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level)
- {
- this(sstables, columnNameComparator, level, false);
}
public MetadataCollector addKey(ByteBuffer key)
@@@ -163,57 -188,45 +181,63 @@@
return this;
}
- public MetadataCollector updateMinTimestamp(long potentialMin)
+ public void update(LivenessInfo newInfo)
{
- minTimestamp = Math.min(minTimestamp, potentialMin);
- return this;
+ if (newInfo.isEmpty())
+ return;
+
+ updateTimestamp(newInfo.timestamp());
+ updateTTL(newInfo.ttl());
+ updateLocalDeletionTime(newInfo.localExpirationTime());
}
- public MetadataCollector updateMaxTimestamp(long potentialMax)
+ public void update(Cell cell)
{
- maxTimestamp = Math.max(maxTimestamp, potentialMax);
- return this;
+ updateTimestamp(cell.timestamp());
+ updateTTL(cell.ttl());
+ updateLocalDeletionTime(cell.localDeletionTime());
}
- public MetadataCollector updateMaxLocalDeletionTime(int maxLocalDeletionTime)
+ public void update(DeletionTime dt)
{
- this.maxLocalDeletionTime = Math.max(this.maxLocalDeletionTime, maxLocalDeletionTime);
- return this;
+ if (!dt.isLive())
+ {
+ updateTimestamp(dt.markedForDeleteAt());
+ updateLocalDeletionTime(dt.localDeletionTime());
+ }
}
- public MetadataCollector estimatedRowSize(EstimatedHistogram estimatedRowSize)
+ public void updateColumnSetPerRow(long columnSetInRow)
{
- this.estimatedRowSize = estimatedRowSize;
- return this;
+ totalColumnsSet += columnSetInRow;
+ ++totalRows;
}
- public MetadataCollector estimatedColumnCount(EstimatedHistogram estimatedColumnCount)
+ private void updateTimestamp(long newTimestamp)
{
- this.estimatedColumnCount = estimatedColumnCount;
- return this;
+ timestampTracker.update(newTimestamp);
+ }
+
+ private void updateLocalDeletionTime(int newLocalDeletionTime)
+ {
+ localDeletionTimeTracker.update(newLocalDeletionTime);
+ estimatedTombstoneDropTime.update(newLocalDeletionTime);
+ }
+
+ private void updateTTL(int newTTL)
+ {
+ ttlTracker.update(newTTL);
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@@ -276,15 -280,13 +300,16 @@@
{
Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
- components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
- estimatedColumnCount,
+ components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
+ estimatedCellPerPartitionCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
- minTimestamp,
- maxTimestamp,
- maxLocalDeletionTime,
+ timestampTracker.min(),
+ timestampTracker.max(),
+ localDeletionTimeTracker.min(),
+ localDeletionTimeTracker.max(),
+ ttlTracker.min(),
+ ttlTracker.max(),
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index 824bf8b,3d48e34..07e35bb
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@@ -40,53 -41,43 +40,56 @@@ public class StatsMetadata extends Meta
{
public static final IMetadataComponentSerializer serializer = new StatsMetadataSerializer();
- public final EstimatedHistogram estimatedRowSize;
+ public final EstimatedHistogram estimatedPartitionSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
+ public final int minLocalDeletionTime;
public final int maxLocalDeletionTime;
+ public final int minTTL;
+ public final int maxTTL;
public final double compressionRatio;
public final StreamingHistogram estimatedTombstoneDropTime;
public final int sstableLevel;
- public final List<ByteBuffer> maxColumnNames;
- public final List<ByteBuffer> minColumnNames;
+ public final List<ByteBuffer> minClusteringValues;
+ public final List<ByteBuffer> maxClusteringValues;
public final boolean hasLegacyCounterShards;
public final long repairedAt;
+ public final long totalColumnsSet;
+ public final long totalRows;
- public StatsMetadata(EstimatedHistogram estimatedRowSize,
+ public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
+ int minLocalDeletionTime,
int maxLocalDeletionTime,
+ int minTTL,
+ int maxTTL,
double compressionRatio,
StreamingHistogram estimatedTombstoneDropTime,
int sstableLevel,
- List<ByteBuffer> minColumnNames,
- List<ByteBuffer> maxColumnNames,
+ List<ByteBuffer> minClusteringValues,
+ List<ByteBuffer> maxClusteringValues,
boolean hasLegacyCounterShards,
- long repairedAt)
+ long repairedAt,
+ long totalColumnsSet,
+ long totalRows)
{
- this.estimatedRowSize = estimatedRowSize;
+ this.estimatedPartitionSize = estimatedPartitionSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
+ this.minLocalDeletionTime = minLocalDeletionTime;
this.maxLocalDeletionTime = maxLocalDeletionTime;
+ this.minTTL = minTTL;
+ this.maxTTL = maxTTL;
this.compressionRatio = compressionRatio;
this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
this.sstableLevel = sstableLevel;
@@@ -129,15 -118,13 +132,16 @@@
public StatsMetadata mutateLevel(int newLevel)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
newLevel,
@@@ -151,15 -136,13 +155,16 @@@
public StatsMetadata mutateRepairedAt(long newRepairedAt)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
@@@ -179,15 -160,13 +184,16 @@@
StatsMetadata that = (StatsMetadata) o;
return new EqualsBuilder()
- .append(estimatedRowSize, that.estimatedRowSize)
+ .append(estimatedPartitionSize, that.estimatedPartitionSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
+ .append(minLocalDeletionTime, that.minLocalDeletionTime)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
+ .append(minTTL, that.minTTL)
+ .append(maxTTL, that.maxTTL)
.append(compressionRatio, that.compressionRatio)
.append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime)
.append(sstableLevel, that.sstableLevel)
@@@ -204,15 -181,13 +210,16 @@@
public int hashCode()
{
return new HashCodeBuilder()
- .append(estimatedRowSize)
+ .append(estimatedPartitionSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
+ .append(minLocalDeletionTime)
.append(maxLocalDeletionTime)
+ .append(minTTL)
+ .append(maxTTL)
.append(compressionRatio)
.append(estimatedTombstoneDropTime)
.append(sstableLevel)
@@@ -227,78 -200,61 +234,84 @@@
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, StatsMetadata component) throws IOException
{
int size = 0;
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
- size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
- size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
- size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound);
+ if (version.storeRows())
+ size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+ else
+ size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
+ size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
+ size += TypeSizes.sizeof(component.sstableLevel);
// min column names
size += 4;
- for (ByteBuffer columnName : component.minColumnNames)
- size += 2 + columnName.remaining(); // with short length
+ for (ByteBuffer value : component.minClusteringValues)
+ size += 2 + value.remaining(); // with short length
// max column names
size += 4;
- for (ByteBuffer columnName : component.maxColumnNames)
- size += 2 + columnName.remaining(); // with short length
- size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ for (ByteBuffer value : component.maxClusteringValues)
+ size += 2 + value.remaining(); // with short length
+ size += TypeSizes.sizeof(component.hasLegacyCounterShards);
+ if (version.storeRows())
+ size += 8 + 8; // totalColumnsSet, totalRows
+ if (version.hasCommitLogLowerBound())
- size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound);
return size;
}
- public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
{
- EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(component.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
+ if (version.storeRows())
+ out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.maxLocalDeletionTime);
+ if (version.storeRows())
+ {
+ out.writeInt(component.minTTL);
+ out.writeInt(component.maxTTL);
+ }
out.writeDouble(component.compressionRatio);
StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
out.writeLong(component.repairedAt);
- out.writeInt(component.minColumnNames.size());
- for (ByteBuffer columnName : component.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(component.maxColumnNames.size());
- for (ByteBuffer columnName : component.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(component.minClusteringValues.size());
+ for (ByteBuffer value : component.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(component.maxClusteringValues.size());
+ for (ByteBuffer value : component.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
out.writeBoolean(component.hasLegacyCounterShards);
+
+ if (version.storeRows())
+ {
+ out.writeLong(component.totalColumnsSet);
+ out.writeLong(component.totalRows);
+ }
++
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
- public StatsMetadata deserialize(Version version, DataInput in) throws IOException
+ public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
+ // We use MAX_VALUE as that's the default value for "no deletion time"
+ int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
int maxLocalDeletionTime = in.readInt();
+ int minTTL = version.storeRows() ? in.readInt() : 0;
+ int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
double compressionRatio = in.readDouble();
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
@@@ -320,18 -276,15 +333,22 @@@
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ long totalColumnsSet = version.storeRows() ? in.readLong() : -1L;
+ long totalRows = version.storeRows() ? in.readLong() : -1L;
+
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
- return new StatsMetadata(rowSizes,
++
+ return new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
index 0000000,0000000..e32d12f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
index 0000000,0000000..790cbe1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
index 0000000,0000000..66d1d34
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++163939259
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
index 0000000,0000000..38d0537
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
index 0000000,0000000..a53b7c1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..e3d4377
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
index 0000000,0000000..c5aa7ce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
index 0000000,0000000..4e3bf89
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3387396134
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
index 0000000,0000000..54afd0c
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
index 0000000,0000000..9d9b085
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..5a34d6e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
index 0000000,0000000..55ad88d
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
index 0000000,0000000..86240b5
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++346435755
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
index 0000000,0000000..d1a401b
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
index 0000000,0000000..4f3f45a
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..32fa731
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
index 0000000,0000000..ca10874
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..98f5784
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3740016396
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
index 0000000,0000000..3661d77
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..bb1e3fb
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
[20/20] cassandra git commit: Merge branch 'cassandra-3.7' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-3.7' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bd39d73a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bd39d73a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bd39d73a
Branch: refs/heads/trunk
Commit: bd39d73a6bb1783786d6e7cc048a96b40c18442e
Parents: fbc07eb c9ac050
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu May 12 15:21:30 2016 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Thu May 12 15:21:30 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 283 ++++++++++++-------
.../org/apache/cassandra/db/Directories.java | 2 +-
src/java/org/apache/cassandra/db/Memtable.java | 70 +++--
.../db/commitlog/CommitLogReplayer.java | 59 ++--
.../cassandra/db/commitlog/ReplayPosition.java | 93 ++++--
.../compaction/AbstractCompactionStrategy.java | 3 -
.../compaction/CompactionStrategyManager.java | 3 -
.../apache/cassandra/db/lifecycle/Tracker.java | 53 +++-
.../org/apache/cassandra/db/lifecycle/View.java | 39 ++-
.../apache/cassandra/db/view/TableViews.java | 4 +-
.../io/sstable/format/SSTableReader.java | 5 -
.../cassandra/io/sstable/format/Version.java | 2 +
.../io/sstable/format/big/BigFormat.java | 12 +-
.../io/sstable/metadata/CompactionMetadata.java | 11 +-
.../metadata/LegacyMetadataSerializer.java | 13 +-
.../io/sstable/metadata/MetadataCollector.java | 36 ++-
.../io/sstable/metadata/StatsMetadata.java | 39 ++-
.../cassandra/tools/SSTableMetadataViewer.java | 3 +-
.../legacy_mb_clust/mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust/mb-1-big-Data.db | Bin 0 -> 5342 bytes
.../legacy_mb_clust/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../legacy_mb_clust/mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 83 bytes
.../legacy_mb_clust_compact/mb-1-big-Data.db | Bin 0 -> 5383 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_compact/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7058 bytes
.../legacy_mb_clust_compact/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../legacy_mb_clust_counter/mb-1-big-Data.db | Bin 0 -> 4625 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_clust_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_clust_counter/mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../legacy_mb_clust_counter/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_clust_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 75 bytes
.../mb-1-big-Data.db | Bin 0 -> 4639 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 157553 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 7067 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../legacy_mb_simple/mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../legacy_mb_simple/mb-1-big-Statistics.db | Bin 0 -> 4611 bytes
.../legacy_mb_simple/mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_compact/mb-1-big-Data.db | Bin 0 -> 91 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_compact/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_compact/mb-1-big-Index.db | Bin 0 -> 26 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4652 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_compact/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../legacy_mb_simple_counter/mb-1-big-Data.db | Bin 0 -> 115 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../legacy_mb_simple_counter/mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../legacy_mb_simple_counter/mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4620 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../legacy_mb_simple_counter/mb-1-big-TOC.txt | 8 +
.../mb-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../mb-1-big-Data.db | Bin 0 -> 114 bytes
.../mb-1-big-Digest.crc32 | 1 +
.../mb-1-big-Filter.db | Bin 0 -> 24 bytes
.../mb-1-big-Index.db | Bin 0 -> 27 bytes
.../mb-1-big-Statistics.db | Bin 0 -> 4661 bytes
.../mb-1-big-Summary.db | Bin 0 -> 47 bytes
.../mb-1-big-TOC.txt | 8 +
.../db/commitlog/CommitLogStressTest.java | 2 +-
.../cassandra/db/RecoveryManagerTest.java | 2 +-
.../cassandra/db/commitlog/CommitLogTest.java | 4 +-
.../db/commitlog/CommitLogTestReplayer.java | 2 +-
.../cassandra/db/lifecycle/TrackerTest.java | 30 +-
.../apache/cassandra/db/lifecycle/ViewTest.java | 2 +-
.../CompressedRandomAccessReaderTest.java | 4 +-
.../CompressedSequentialWriterTest.java | 2 +-
.../cassandra/io/sstable/LegacySSTableTest.java | 2 +-
.../metadata/MetadataSerializerTest.java | 83 +++++-
.../cassandra/io/util/MmappedRegionsTest.java | 3 +-
94 files changed, 644 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd39d73a/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/bd39d73a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
[16/20] cassandra git commit: Merge commit
'849a438690aa97a361227781108cc90355dcbcd9' into cassandra-3.0
Posted by sl...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/lifecycle/View.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/View.java
index b62c7e3,fba1627..17062b4
--- a/src/java/org/apache/cassandra/db/lifecycle/View.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/View.java
@@@ -33,16 -33,14 +33,16 @@@ import org.apache.cassandra.io.sstable.
import org.apache.cassandra.utils.Interval;
import static com.google.common.base.Predicates.equalTo;
++import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.not;
import static com.google.common.collect.ImmutableList.copyOf;
import static com.google.common.collect.ImmutableList.of;
import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterables.concat;
import static com.google.common.collect.Iterables.filter;
-import static java.util.Collections.singleton;
+import static com.google.common.collect.Iterables.transform;
- import static java.util.Collections.singleton;
import static org.apache.cassandra.db.lifecycle.Helpers.emptySet;
+import static org.apache.cassandra.db.lifecycle.Helpers.filterOut;
import static org.apache.cassandra.db.lifecycle.Helpers.replace;
/**
@@@ -68,18 -66,18 +68,19 @@@ public class Vie
* flushed. In chronologically ascending order.
*/
public final List<Memtable> flushingMemtables;
- public final Set<SSTableReader> compacting;
- public final Set<SSTableReader> sstables;
- public final Set<SSTableReader> premature;
+ final Set<SSTableReader> compacting;
+ final Set<SSTableReader> sstables;
++ final Set<SSTableReader> premature;
// we use a Map here so that we can easily perform identity checks as well as equality checks.
// When marking compacting, we now indicate if we expect the sstables to be present (by default we do),
// and we then check that not only are they all present in the live set, but that the exact instance present is
// the one we made our decision to compact against.
- public final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> sstablesMap;
+ final Map<SSTableReader, SSTableReader> compactingMap;
- public final SSTableIntervalTree intervalTree;
+ final SSTableIntervalTree intervalTree;
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, SSTableIntervalTree intervalTree)
- View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Set<SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
++ View(List<Memtable> liveMemtables, List<Memtable> flushingMemtables, Map<SSTableReader, SSTableReader> sstables, Map<SSTableReader, SSTableReader> compacting, Set<SSTableReader> premature, SSTableIntervalTree intervalTree)
{
assert liveMemtables != null;
assert flushingMemtables != null;
@@@ -92,8 -90,8 +93,9 @@@
this.sstablesMap = sstables;
this.sstables = sstablesMap.keySet();
- this.compacting = compacting;
+ this.compactingMap = compacting;
+ this.compacting = compactingMap.keySet();
+ this.premature = premature;
this.intervalTree = intervalTree;
}
@@@ -238,8 -158,8 +240,8 @@@
{
assert all(mark, Helpers.idIn(view.sstablesMap));
return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap,
- replace(view.compacting, unmark, mark),
+ replace(view.compactingMap, unmark, mark),
- view.intervalTree);
+ view.premature, view.intervalTree);
}
};
}
@@@ -270,7 -190,7 +272,7 @@@
public View apply(View view)
{
Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, remove, add);
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap,
- return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compacting, view.premature,
++ return new View(view.liveMemtables, view.flushingMemtables, sstableMap, view.compactingMap, view.premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
@@@ -285,7 -205,7 +287,7 @@@
{
List<Memtable> newLive = ImmutableList.<Memtable>builder().addAll(view.liveMemtables).add(newMemtable).build();
assert newLive.size() == view.liveMemtables.size() + 1;
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, view.flushingMemtables, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -304,7 -224,7 +306,7 @@@
filter(flushing, not(lessThan(toFlush)))));
assert newLive.size() == live.size() - 1;
assert newFlushing.size() == flushing.size() + 1;
- return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.intervalTree);
- return new View(newLive, newFlushing, view.sstablesMap, view.compacting, view.premature, view.intervalTree);
++ return new View(newLive, newFlushing, view.sstablesMap, view.compactingMap, view.premature, view.intervalTree);
}
};
}
@@@ -319,17 -239,35 +321,34 @@@
List<Memtable> flushingMemtables = copyOf(filter(view.flushingMemtables, not(equalTo(memtable))));
assert flushingMemtables.size() == view.flushingMemtables.size() - 1;
-- if (flushed == null)
++ if (flushed == null || flushed.isEmpty())
return new View(view.liveMemtables, flushingMemtables, view.sstablesMap,
- view.compactingMap, view.intervalTree);
- view.compacting, view.premature, view.intervalTree);
++ view.compactingMap, view.premature, view.intervalTree);
- Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), singleton(flushed));
- Set<SSTableReader> compacting = replace(view.compacting, emptySet(), singleton(flushed));
- Set<SSTableReader> premature = replace(view.premature, emptySet(), singleton(flushed));
- return new View(view.liveMemtables, flushingMemtables, sstableMap, compacting, premature,
+ Map<SSTableReader, SSTableReader> sstableMap = replace(view.sstablesMap, emptySet(), flushed);
- return new View(view.liveMemtables, flushingMemtables, sstableMap, view.compactingMap,
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, emptySet(), flushed);
++ Set<SSTableReader> premature = replace(view.premature, emptySet(), flushed);
++ return new View(view.liveMemtables, flushingMemtables, sstableMap, compactingMap, premature,
SSTableIntervalTree.build(sstableMap.keySet()));
}
};
}
- static Function<View, View> permitCompactionOfFlushed(final SSTableReader reader)
++ static Function<View, View> permitCompactionOfFlushed(final Collection<SSTableReader> readers)
+ {
++ Set<SSTableReader> expectAndRemove = ImmutableSet.copyOf(readers);
+ return new Function<View, View>()
+ {
-
- @Nullable
+ public View apply(View view)
+ {
- Set<SSTableReader> premature = ImmutableSet.copyOf(filter(view.premature, not(equalTo(reader))));
- Set<SSTableReader> compacting = ImmutableSet.copyOf(filter(view.compacting, not(equalTo(reader))));
- return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compacting, premature, view.intervalTree);
++ Set<SSTableReader> premature = replace(view.premature, expectAndRemove, emptySet());
++ Map<SSTableReader, SSTableReader> compactingMap = replace(view.compactingMap, expectAndRemove, emptySet());
++ return new View(view.liveMemtables, view.flushingMemtables, view.sstablesMap, compactingMap, premature, view.intervalTree);
+ }
+ };
+ }
+
+
private static <T extends Comparable<T>> Predicate<T> lessThan(final T lessThan)
{
return new Predicate<T>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/db/view/TableViews.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/view/TableViews.java
index 893bdd5,0000000..7feb67c
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@@ -1,481 -1,0 +1,481 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.view;
+
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.commitlog.ReplayPosition;
+import org.apache.cassandra.db.filter.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.partitions.*;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.service.StorageProxy;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+
+/**
+ * Groups all the views for a given table.
+ */
+public class TableViews extends AbstractCollection<View>
+{
+ private final CFMetaData baseTableMetadata;
+
+ // We need this to be thread-safe, but the number of times this is changed (when a view is created in the keyspace)
+ // massively exceeds the number of time it's read (for every mutation on the keyspace), so a copy-on-write list is the best option.
+ private final List<View> views = new CopyOnWriteArrayList();
+
+ public TableViews(CFMetaData baseTableMetadata)
+ {
+ this.baseTableMetadata = baseTableMetadata;
+ }
+
+ public int size()
+ {
+ return views.size();
+ }
+
+ public Iterator<View> iterator()
+ {
+ return views.iterator();
+ }
+
+ public boolean contains(String viewName)
+ {
+ return Iterables.any(views, view -> view.name.equals(viewName));
+ }
+
+ public boolean add(View view)
+ {
+ // We should have validated that there is no existing view with this name at this point
+ assert !contains(view.name);
+ return views.add(view);
+ }
+
+ public Iterable<ColumnFamilyStore> allViewsCfs()
+ {
+ Keyspace keyspace = Keyspace.open(baseTableMetadata.ksName);
+ return Iterables.transform(views, view -> keyspace.getColumnFamilyStore(view.getDefinition().viewName));
+ }
+
+ public void forceBlockingFlush()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.forceBlockingFlush();
+ }
+
+ public void dumpMemtables()
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ viewCfs.dumpMemtable();
+ }
+
- public void truncateBlocking(long truncatedAt)
++ public void truncateBlocking(ReplayPosition replayAfter, long truncatedAt)
+ {
+ for (ColumnFamilyStore viewCfs : allViewsCfs())
+ {
- ReplayPosition replayAfter = viewCfs.discardSSTables(truncatedAt);
++ viewCfs.discardSSTables(truncatedAt);
+ SystemKeyspace.saveTruncationRecord(viewCfs, truncatedAt, replayAfter);
+ }
+ }
+
+ public void removeByName(String viewName)
+ {
+ views.removeIf(v -> v.name.equals(viewName));
+ }
+
+ /**
+ * Calculates and pushes updates to the views replicas. The replicas are determined by
+ * {@link ViewUtils#getViewNaturalEndpoint(String, Token, Token)}.
+ *
+ * @param update an update on the base table represented by this object.
+ * @param writeCommitLog whether we should write the commit log for the view updates.
+ * @param baseComplete time from epoch in ms that the local base mutation was (or will be) completed
+ */
+ public void pushViewReplicaUpdates(PartitionUpdate update, boolean writeCommitLog, AtomicLong baseComplete)
+ {
+ assert update.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ Collection<View> views = updatedViews(update);
+ if (views.isEmpty())
+ return;
+
+ // Read modified rows
+ int nowInSec = FBUtilities.nowInSeconds();
+ SinglePartitionReadCommand command = readExistingRowsCommand(update, views, nowInSec);
+ if (command == null)
+ return;
+
+ ColumnFamilyStore cfs = Keyspace.openAndGetStore(update.metadata());
+ long start = System.nanoTime();
+ Collection<Mutation> mutations;
+ try (ReadOrderGroup orderGroup = command.startOrderGroup();
+ UnfilteredRowIterator existings = UnfilteredPartitionIterators.getOnlyElement(command.executeLocally(orderGroup), command);
+ UnfilteredRowIterator updates = update.unfilteredIterator())
+ {
+ mutations = generateViewUpdates(views, updates, existings, nowInSec);
+ }
+ Keyspace.openAndGetStore(update.metadata()).metric.viewReadTime.update(System.nanoTime() - start, TimeUnit.NANOSECONDS);
+
+ if (!mutations.isEmpty())
+ StorageProxy.mutateMV(update.partitionKey().getKey(), mutations, writeCommitLog, baseComplete);
+ }
+
+ /**
+ * Given some updates on the base table of this object and the existing values for the rows affected by that update, generates the
+ * mutation to be applied to the provided views.
+ *
+ * @param views the views potentially affected by {@code updates}.
+ * @param updates the base table updates being applied.
+ * @param existings the existing values for the rows affected by {@code updates}. This is used to decide if a view is
+ * obsoleted by the update and should be removed, gather the values for columns that may not be part of the update if
+ * a new view entry needs to be created, and compute the minimal updates to be applied if the view entry isn't changed
+ * but has simply some updated values. This will be empty for view building as we want to assume anything we'll pass
+ * to {@code updates} is new.
+ * @param nowInSec the current time in seconds.
+ * @return the mutations to apply to the {@code views}. This can be empty.
+ */
+ public Collection<Mutation> generateViewUpdates(Collection<View> views, UnfilteredRowIterator updates, UnfilteredRowIterator existings, int nowInSec)
+ {
+ assert updates.metadata().cfId.equals(baseTableMetadata.cfId);
+
+ List<ViewUpdateGenerator> generators = new ArrayList<>(views.size());
+ for (View view : views)
+ generators.add(new ViewUpdateGenerator(view, updates.partitionKey(), nowInSec));
+
+ DeletionTracker existingsDeletion = new DeletionTracker(existings.partitionLevelDeletion());
+ DeletionTracker updatesDeletion = new DeletionTracker(updates.partitionLevelDeletion());
+
+ /*
+ * We iterate through the updates and the existing rows in parallel. This allows us to know the consequence
+ * on the view of each update.
+ */
+ PeekingIterator<Unfiltered> existingsIter = Iterators.peekingIterator(existings);
+ PeekingIterator<Unfiltered> updatesIter = Iterators.peekingIterator(updates);
+
+ while (existingsIter.hasNext() && updatesIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.peek();
+ Unfiltered update = updatesIter.peek();
+
+ Row existingRow;
+ Row updateRow;
+ int cmp = baseTableMetadata.comparator.compare(update, existing);
+ if (cmp < 0)
+ {
+ // We have an update where there was nothing before
+ if (update.isRangeTombstoneMarker())
+ {
+ updatesDeletion.update(updatesIter.next());
+ continue;
+ }
+
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ existingRow = emptyRow(updateRow.clustering(), existingsDeletion.currentDeletion());
+ }
+ else if (cmp > 0)
+ {
+ // We have something existing but no update (which will happen either because it's a range tombstone marker in
+ // existing, or because we've fetched the existing row due to some partition/range deletion in the updates)
+ if (existing.isRangeTombstoneMarker())
+ {
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion());
+
+ // The way we build the read command used for existing rows, we should always have updatesDeletion.currentDeletion()
+ // that is not live, since we wouldn't have read the existing row otherwise. And we could assert that, but if we ever
+ // change the read method so that it can slightly over-read in some case, that would be an easily avoiding bug lurking,
+ // so we just handle the case.
+ if (updateRow == null)
+ continue;
+ }
+ else
+ {
+ // We're updating a row that had pre-existing data
+ if (update.isRangeTombstoneMarker())
+ {
+ assert existing.isRangeTombstoneMarker();
+ updatesDeletion.update(updatesIter.next());
+ existingsDeletion.update(existingsIter.next());
+ continue;
+ }
+
+ assert !existing.isRangeTombstoneMarker();
+ existingRow = ((Row)existingsIter.next()).withRowDeletion(existingsDeletion.currentDeletion());
+ updateRow = ((Row)updatesIter.next()).withRowDeletion(updatesDeletion.currentDeletion());
+ }
+
+ addToViewUpdateGenerators(existingRow, updateRow, generators, nowInSec);
+ }
+
+ // We only care about more existing rows if the update deletion isn't live, i.e. if we had a partition deletion
+ if (!updatesDeletion.currentDeletion().isLive())
+ {
+ while (existingsIter.hasNext())
+ {
+ Unfiltered existing = existingsIter.next();
+ // If it's a range tombstone, we don't care, we're only looking for existing entry that gets deleted by
+ // the new partition deletion
+ if (existing.isRangeTombstoneMarker())
+ continue;
+
+ Row existingRow = (Row)existing;
+ addToViewUpdateGenerators(existingRow, emptyRow(existingRow.clustering(), updatesDeletion.currentDeletion()), generators, nowInSec);
+ }
+ }
+ while (updatesIter.hasNext())
+ {
+ Unfiltered update = updatesIter.next();
+ // If it's a range tombstone, it removes nothing pre-exisiting, so we can ignore it for view updates
+ if (update.isRangeTombstoneMarker())
+ continue;
+
+ Row updateRow = (Row)update;
+ addToViewUpdateGenerators(emptyRow(updateRow.clustering(), DeletionTime.LIVE), updateRow, generators, nowInSec);
+ }
+
+ return buildMutations(baseTableMetadata, generators);
+ }
+
+ /**
+ * Return the views that are potentially updated by the provided updates.
+ *
+ * @param updates the updates applied to the base table.
+ * @return the views affected by {@code updates}.
+ */
+ public Collection<View> updatedViews(PartitionUpdate updates)
+ {
+ List<View> matchingViews = new ArrayList<>(views.size());
+
+ for (View view : views)
+ {
+ ReadQuery selectQuery = view.getReadQuery();
+ if (!selectQuery.selectsKey(updates.partitionKey()))
+ continue;
+
+ matchingViews.add(view);
+ }
+ return matchingViews;
+ }
+
+ /**
+ * Returns the command to use to read the existing rows required to generate view updates for the provided base
+ * base updates.
+ *
+ * @param updates the base table updates being applied.
+ * @param views the views potentially affected by {@code updates}.
+ * @param nowInSec the current time in seconds.
+ * @return the command to use to read the base table rows required to generate view updates for {@code updates}.
+ */
+ private SinglePartitionReadCommand readExistingRowsCommand(PartitionUpdate updates, Collection<View> views, int nowInSec)
+ {
+ Slices.Builder sliceBuilder = null;
+ DeletionInfo deletionInfo = updates.deletionInfo();
+ CFMetaData metadata = updates.metadata();
+ DecoratedKey key = updates.partitionKey();
+ // TODO: This is subtle: we need to gather all the slices that we have to fetch between partition del, range tombstones and rows.
+ if (!deletionInfo.isLive())
+ {
+ sliceBuilder = new Slices.Builder(metadata.comparator);
+ // Everything covered by a deletion might invalidate an existing view entry, which means we must read it to know. In practice
+ // though, the views involved might filter some base table clustering columns, in which case we can restrict what we read
+ // using those restrictions.
+ // If there is a partition deletion, then we can simply take each slices from each view select filter. They may overlap but
+ // the Slices.Builder handles that for us. Note that in many case this will just involve reading everything (as soon as any
+ // view involved has no clustering restrictions for instance).
+ // For range tombstone, we should theoretically take the difference between the range tombstoned and the slices selected
+ // by every views, but as we don't an easy way to compute that right now, we keep it simple and just use the tombstoned
+ // range.
+ // TODO: we should improve that latter part.
+ if (!deletionInfo.getPartitionDeletion().isLive())
+ {
+ for (View view : views)
+ sliceBuilder.addAll(view.getSelectStatement().clusteringIndexFilterAsSlices());
+ }
+ else
+ {
+ assert deletionInfo.hasRanges();
+ Iterator<RangeTombstone> iter = deletionInfo.rangeIterator(false);
+ while (iter.hasNext())
+ sliceBuilder.add(iter.next().deletedSlice());
+ }
+ }
+
+ // We need to read every row that is updated, unless we can prove that it has no impact on any view entries.
+
+ // If we had some slices from the deletions above, we'll continue using that. Otherwise, it's more efficient to build
+ // a names query.
+ BTreeSet.Builder<Clustering> namesBuilder = sliceBuilder == null ? BTreeSet.builder(metadata.comparator) : null;
+ for (Row row : updates)
+ {
+ // Don't read the existing state if we can prove the update won't affect any views
+ if (!affectsAnyViews(key, row, views))
+ continue;
+
+ if (namesBuilder == null)
+ sliceBuilder.add(Slice.make(row.clustering()));
+ else
+ namesBuilder.add(row.clustering());
+ }
+
+ NavigableSet<Clustering> names = namesBuilder == null ? null : namesBuilder.build();
+ // If we have a slice builder, it means we had some deletions and we have to read. But if we had
+ // only row updates, it's possible none of them affected the views, in which case we have nothing
+ // to do.
+ if (names != null && names.isEmpty())
+ return null;
+
+ ClusteringIndexFilter clusteringFilter = names == null
+ ? new ClusteringIndexSliceFilter(sliceBuilder.build(), false)
+ : new ClusteringIndexNamesFilter(names, false);
+ // If we have more than one view, we should merge the queried columns by each views but to keep it simple we just
+ // include everything. We could change that in the future.
+ ColumnFilter queriedColumns = views.size() == 1
+ ? Iterables.getOnlyElement(views).getSelectStatement().queriedColumns()
+ : ColumnFilter.all(metadata);
+ // Note that the views could have restrictions on regular columns, but even if that's the case we shouldn't apply those
+ // when we read, because even if an existing row doesn't match the view filter, the update can change that in which
+ // case we'll need to know the existing content. There is also no easy way to merge those RowFilter when we have multiple views.
+ // TODO: we could still make sense to special case for when there is a single view and a small number of updates (and
+ // no deletions). Indeed, in that case we could check whether any of the update modify any of the restricted regular
+ // column, and if that's not the case we could use view filter. We keep it simple for now though.
+ RowFilter rowFilter = RowFilter.NONE;
+ return SinglePartitionReadCommand.create(metadata, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, clusteringFilter);
+ }
+
+ private boolean affectsAnyViews(DecoratedKey partitionKey, Row update, Collection<View> views)
+ {
+ for (View view : views)
+ {
+ if (view.mayBeAffectedBy(partitionKey, update))
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Given an existing base row and the update that we're going to apply to this row, generate the modifications
+ * to apply to MVs using the provided {@code ViewUpdateGenerator}s.
+ *
+ * @param existingBaseRow the base table row as it is before an update.
+ * @param updateBaseRow the newly updates made to {@code existingBaseRow}.
+ * @param generators the view update generators to add the new changes to.
+ * @param nowInSec the current time in seconds. Used to decide if data is live or not.
+ */
+ private static void addToViewUpdateGenerators(Row existingBaseRow, Row updateBaseRow, Collection<ViewUpdateGenerator> generators, int nowInSec)
+ {
+ // Having existing empty is useful, it just means we'll insert a brand new entry for updateBaseRow,
+ // but if we have no update at all, we shouldn't get there.
+ assert !updateBaseRow.isEmpty();
+
+ // We allow existingBaseRow to be null, which we treat the same as being empty as an small optimization
+ // to avoid allocating empty row objects when we know there was nothing existing.
+ Row mergedBaseRow = existingBaseRow == null ? updateBaseRow : Rows.merge(existingBaseRow, updateBaseRow, nowInSec);
+ for (ViewUpdateGenerator generator : generators)
+ generator.addBaseTableUpdate(existingBaseRow, mergedBaseRow);
+ }
+
+ private static Row emptyRow(Clustering clustering, DeletionTime deletion)
+ {
+ // Returning null for an empty row is slightly ugly, but the case where there is no pre-existing row is fairly common
+ // (especially when building the view), so we want to avoid a dummy allocation of an empty row every time.
+ // And MultiViewUpdateBuilder knows how to deal with that.
+ return deletion.isLive() ? null : BTreeRow.emptyDeletedRow(clustering, Row.Deletion.regular(deletion));
+ }
+
+ /**
+ * Extracts (and potentially groups) the mutations generated by the provided view update generator.
+ * Returns the mutation that needs to be done to the views given the base table updates
+ * passed to {@link #addBaseTableUpdate}.
+ *
+ * @param baseTableMetadata the metadata for the base table being updated.
+ * @param generators the generators from which to extract the view mutations from.
+ * @return the mutations created by all the generators in {@code generators}.
+ */
+ private Collection<Mutation> buildMutations(CFMetaData baseTableMetadata, List<ViewUpdateGenerator> generators)
+ {
+ // One view is probably common enough and we can optimize a bit easily
+ if (generators.size() == 1)
+ {
+ Collection<PartitionUpdate> updates = generators.get(0).generateViewUpdates();
+ List<Mutation> mutations = new ArrayList<>(updates.size());
+ for (PartitionUpdate update : updates)
+ mutations.add(new Mutation(update));
+ return mutations;
+ }
+
+ Map<DecoratedKey, Mutation> mutations = new HashMap<>();
+ for (ViewUpdateGenerator generator : generators)
+ {
+ for (PartitionUpdate update : generator.generateViewUpdates())
+ {
+ DecoratedKey key = update.partitionKey();
+ Mutation mutation = mutations.get(key);
+ if (mutation == null)
+ {
+ mutation = new Mutation(baseTableMetadata.ksName, key);
+ mutations.put(key, mutation);
+ }
+ mutation.add(update);
+ }
+ }
+ return mutations.values();
+ }
+
+ /**
+ * A simple helper that tracks for a given {@code UnfilteredRowIterator} what is the current deletion at any time of the
+ * iteration. It will be the currently open range tombstone deletion if there is one and the partition deletion otherwise.
+ */
+ private static class DeletionTracker
+ {
+ private final DeletionTime partitionDeletion;
+ private DeletionTime deletion;
+
+ public DeletionTracker(DeletionTime partitionDeletion)
+ {
+ this.partitionDeletion = partitionDeletion;
+ }
+
+ public void update(Unfiltered marker)
+ {
+ assert marker instanceof RangeTombstoneMarker;
+ RangeTombstoneMarker rtm = (RangeTombstoneMarker)marker;
+ this.deletion = rtm.isOpen(false)
+ ? rtm.openDeletionTime(false)
+ : null;
+ }
+
+ public DeletionTime currentDeletion()
+ {
+ return deletion == null ? partitionDeletion : deletion;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/Version.java
index 16829ab,41a83e1..d9e289c
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@@ -60,16 -52,8 +60,18 @@@ public abstract class Versio
public abstract boolean hasNewFileName();
+ public abstract boolean storeRows();
+
+ public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far
+
+ public abstract boolean hasOldBfHashOrder();
+
+ public abstract boolean hasCompactionAncestors();
+
+ public abstract boolean hasBoundaries();
+
+ public abstract boolean hasCommitLogLowerBound();
+
public String getVersion()
{
return version;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 1f2a98f,9244bbb..e0fb3b1
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@@ -111,7 -125,7 +111,7 @@@ public class BigFormat implements SSTab
// we always incremented the major version.
static class BigVersion extends Version
{
- public static final String current_version = "ma";
- public static final String current_version = "lb";
++ public static final String current_version = "mb";
public static final String earliest_supported_version = "jb";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
@@@ -121,10 -135,7 +121,12 @@@
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
+ // lb (2.2.7): commit log lower bound included
+ // ma (3.0.0): swap bf hash order
+ // store rows natively
++ // mb (3.0.6): commit log lower bound included
+ //
+ // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
@@@ -134,53 -144,20 +136,56 @@@
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
+ public final boolean storeRows;
+ public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
+ public final boolean hasBoundaries;
+ /**
+ * CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
+ * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
+ */
+ private final boolean hasOldBfHashOrder;
+ private final boolean hasCommitLogLowerBound;
- public BigVersion(String version)
+ /**
+ * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+ */
+ private final boolean hasCompactionAncestors;
+
+ BigVersion(String version)
{
- super(instance,version);
+ super(instance, version);
isLatestVersion = version.compareTo(current_version) == 0;
hasSamplingLevel = version.compareTo("ka") >= 0;
newStatsFile = version.compareTo("ka") >= 0;
- hasAllAdlerChecksums = version.compareTo("ka") >= 0;
+
+ //For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell
+ //PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684
+ ChecksumType checksumType = ChecksumType.CRC32;
+ if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.uncompressedChecksumType = checksumType;
+
+ checksumType = ChecksumType.CRC32;
+ if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0)
+ checksumType = ChecksumType.Adler32;
+ this.compressedChecksumType = checksumType;
+
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
+
newFileName = version.compareTo("la") >= 0;
- hasCommitLogLowerBound = version.compareTo("lb") >= 0;
+
+ hasOldBfHashOrder = version.compareTo("ma") < 0;
+ hasCompactionAncestors = version.compareTo("ma") < 0;
+ storeRows = version.compareTo("ma") >= 0;
+ correspondingMessagingVersion = storeRows
+ ? MessagingService.VERSION_30
+ : MessagingService.VERSION_21;
+
+ hasBoundaries = version.compareTo("ma") < 0;
++ hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0)
++ || version.compareTo("mb") >= 0;
}
@Override
@@@ -243,25 -202,12 +248,30 @@@
return newFileName;
}
+ public boolean hasCommitLogLowerBound()
+ {
+ return hasCommitLogLowerBound;
+ }
+
@Override
+ public boolean storeRows()
+ {
+ return storeRows;
+ }
+
+ @Override
+ public int correspondingMessagingVersion()
+ {
+ return correspondingMessagingVersion;
+ }
+
+ @Override
+ public boolean hasBoundaries()
+ {
+ return hasBoundaries;
+ }
+
+ @Override
public boolean isCompatible()
{
return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index 4f4c04b,c8e6ee8..ef3453a
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@@ -72,14 -72,22 +72,23 @@@ public class CompactionMetadata extend
public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
{
- public int serializedSize(CompactionMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, CompactionMetadata component) throws IOException
{
- int size = 0;
- size += TypeSizes.NATIVE.sizeof(component.ancestors.size());
- for (int g : component.ancestors)
- size += TypeSizes.NATIVE.sizeof(g);
++ int sz = 0;
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ sz = 4;
++ }
byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
- return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
- size += TypeSizes.NATIVE.sizeof(serializedCardinality.length) + serializedCardinality.length;
- return size;
++ return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length + sz;
}
- public void serialize(CompactionMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
{
- out.writeInt(component.ancestors.size());
- for (int g : component.ancestors)
- out.writeInt(g);
++ if (version.hasCompactionAncestors())
++ { // write empty ancestor marker
++ out.writeInt(0);
++ }
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 2ed845c,bfeb930..4561520
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@@ -53,24 -51,28 +53,26 @@@ public class LegacyMetadataSerializer e
assert validation != null && stats != null && compaction != null && validation.partitioner != null;
- EstimatedHistogram.serializer.serialize(stats.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(stats.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(stats.replayPosition, out);
+ ReplayPosition.serializer.serialize(stats.commitLogUpperBound, out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
out.writeDouble(validation.bloomFilterFPChance);
out.writeDouble(stats.compressionRatio);
out.writeUTF(validation.partitioner);
- out.writeInt(compaction.ancestors.size());
- for (Integer g : compaction.ancestors)
- out.writeInt(g);
+ out.writeInt(0); // compaction ancestors
StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
out.writeInt(stats.sstableLevel);
- out.writeInt(stats.minColumnNames.size());
- for (ByteBuffer columnName : stats.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(stats.maxColumnNames.size());
- for (ByteBuffer columnName : stats.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(stats.minClusteringValues.size());
+ for (ByteBuffer value : stats.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(stats.maxClusteringValues.size());
+ for (ByteBuffer value : stats.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(stats.commitLogLowerBound, out);
}
/**
@@@ -88,11 -90,12 +90,12 @@@
}
else
{
- try (DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile))))
+ try (DataInputStreamPlus in = new DataInputStreamPlus(new BufferedInputStream(new FileInputStream(statsFile))))
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ ReplayPosition commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
@@@ -115,21 -120,21 +118,25 @@@
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
+
+ if (descriptor.version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
+
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
new ValidationMetadata(partitioner, bloomFilterFPChance));
if (types.contains(MetadataType.STATS))
components.put(MetadataType.STATS,
- new StatsMetadata(rowSizes,
+ new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ Integer.MAX_VALUE,
maxLocalDeletionTime,
+ 0,
+ Integer.MAX_VALUE,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index c2b0caf,579ff7a..53cf0b0
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -26,16 -26,17 +26,16 @@@ import java.util.List
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+ import com.google.common.collect.Ordering;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.io.sstable.ColumnNameHelper;
-import org.apache.cassandra.io.sstable.ColumnStats;
-import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
+import org.apache.cassandra.db.rows.Cell;
- import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
@@@ -67,9 -67,10 +67,10 @@@ public class MetadataCollector implemen
public static StatsMetadata defaultStatsMetadata()
{
- return new StatsMetadata(defaultRowSizeHistogram(),
- defaultColumnCountHistogram(),
+ return new StatsMetadata(defaultPartitionSizeHistogram(),
+ defaultCellPerPartitionCountHistogram(),
ReplayPosition.NONE,
+ ReplayPosition.NONE,
Long.MIN_VALUE,
Long.MAX_VALUE,
Integer.MAX_VALUE,
@@@ -82,26 -80,23 +83,27 @@@
Collections.<ByteBuffer>emptyList(),
Collections.<ByteBuffer>emptyList(),
true,
- ActiveRepairService.UNREPAIRED_SSTABLE);
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ -1,
+ -1);
}
- protected EstimatedHistogram estimatedRowSize = defaultRowSizeHistogram();
- protected EstimatedHistogram estimatedColumnCount = defaultColumnCountHistogram();
+ protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
+ // TODO: cound the number of row per partition (either with the number of cells, or instead)
+ protected EstimatedHistogram estimatedCellPerPartitionCount = defaultCellPerPartitionCountHistogram();
- protected ReplayPosition replayPosition = ReplayPosition.NONE;
+ protected ReplayPosition commitLogLowerBound = ReplayPosition.NONE;
+ protected ReplayPosition commitLogUpperBound = ReplayPosition.NONE;
- protected long minTimestamp = Long.MAX_VALUE;
- protected long maxTimestamp = Long.MIN_VALUE;
- protected int maxLocalDeletionTime = Integer.MIN_VALUE;
+ protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker();
+ protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
+ protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
protected double compressionRatio = NO_COMPRESSION_RATIO;
- protected Set<Integer> ancestors = new HashSet<>();
protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram();
protected int sstableLevel;
- protected List<ByteBuffer> minColumnNames = Collections.emptyList();
- protected List<ByteBuffer> maxColumnNames = Collections.emptyList();
+ protected ByteBuffer[] minClusteringValues;
+ protected ByteBuffer[] maxClusteringValues;
protected boolean hasLegacyCounterShards = false;
+ protected long totalColumnsSet;
+ protected long totalRows;
/**
* Default cardinality estimation method is to use HyperLogLog++.
@@@ -110,22 -105,52 +112,38 @@@
* See CASSANDRA-5906 for detail.
*/
protected ICardinality cardinality = new HyperLogLogPlus(13, 25);
- private final CellNameType columnNameComparator;
+ private final ClusteringComparator comparator;
- public MetadataCollector(CellNameType columnNameComparator)
+ public MetadataCollector(ClusteringComparator comparator)
{
- this.columnNameComparator = columnNameComparator;
+ this.comparator = comparator;
+
+ this.minClusteringValues = new ByteBuffer[comparator.size()];
+ this.maxClusteringValues = new ByteBuffer[comparator.size()];
}
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level, boolean skipAncestors)
+ public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
{
- this(columnNameComparator);
+ this(comparator);
- replayPosition(ReplayPosition.getReplayPosition(sstables));
+ ReplayPosition min = null, max = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (min == null)
+ {
+ min = sstable.getSSTableMetadata().commitLogLowerBound;
+ max = sstable.getSSTableMetadata().commitLogUpperBound;
+ }
+ else
+ {
+ min = Ordering.natural().min(min, sstable.getSSTableMetadata().commitLogLowerBound);
+ max = Ordering.natural().max(max, sstable.getSSTableMetadata().commitLogUpperBound);
+ }
+ }
+
+ commitLogLowerBound(min);
+ commitLogUpperBound(max);
sstableLevel(level);
- // Get the max timestamp of the precompacted sstables
- // and adds generation of live ancestors
- if (!skipAncestors)
- {
- for (SSTableReader sstable : sstables)
- {
- addAncestor(sstable.descriptor.generation);
- for (Integer i : sstable.getAncestors())
- if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
- addAncestor(i);
- }
- }
- }
-
- public MetadataCollector(Iterable<SSTableReader> sstables, CellNameType columnNameComparator, int level)
- {
- this(sstables, columnNameComparator, level, false);
}
public MetadataCollector addKey(ByteBuffer key)
@@@ -163,57 -188,45 +181,63 @@@
return this;
}
- public MetadataCollector updateMinTimestamp(long potentialMin)
+ public void update(LivenessInfo newInfo)
{
- minTimestamp = Math.min(minTimestamp, potentialMin);
- return this;
+ if (newInfo.isEmpty())
+ return;
+
+ updateTimestamp(newInfo.timestamp());
+ updateTTL(newInfo.ttl());
+ updateLocalDeletionTime(newInfo.localExpirationTime());
}
- public MetadataCollector updateMaxTimestamp(long potentialMax)
+ public void update(Cell cell)
{
- maxTimestamp = Math.max(maxTimestamp, potentialMax);
- return this;
+ updateTimestamp(cell.timestamp());
+ updateTTL(cell.ttl());
+ updateLocalDeletionTime(cell.localDeletionTime());
}
- public MetadataCollector updateMaxLocalDeletionTime(int maxLocalDeletionTime)
+ public void update(DeletionTime dt)
{
- this.maxLocalDeletionTime = Math.max(this.maxLocalDeletionTime, maxLocalDeletionTime);
- return this;
+ if (!dt.isLive())
+ {
+ updateTimestamp(dt.markedForDeleteAt());
+ updateLocalDeletionTime(dt.localDeletionTime());
+ }
}
- public MetadataCollector estimatedRowSize(EstimatedHistogram estimatedRowSize)
+ public void updateColumnSetPerRow(long columnSetInRow)
{
- this.estimatedRowSize = estimatedRowSize;
- return this;
+ totalColumnsSet += columnSetInRow;
+ ++totalRows;
}
- public MetadataCollector estimatedColumnCount(EstimatedHistogram estimatedColumnCount)
+ private void updateTimestamp(long newTimestamp)
{
- this.estimatedColumnCount = estimatedColumnCount;
- return this;
+ timestampTracker.update(newTimestamp);
+ }
+
+ private void updateLocalDeletionTime(int newLocalDeletionTime)
+ {
+ localDeletionTimeTracker.update(newLocalDeletionTime);
+ estimatedTombstoneDropTime.update(newLocalDeletionTime);
+ }
+
+ private void updateTTL(int newTTL)
+ {
+ ttlTracker.update(newTTL);
}
- public MetadataCollector replayPosition(ReplayPosition replayPosition)
+ public MetadataCollector commitLogLowerBound(ReplayPosition commitLogLowerBound)
+ {
+ this.commitLogLowerBound = commitLogLowerBound;
+ return this;
+ }
+
+ public MetadataCollector commitLogUpperBound(ReplayPosition commitLogUpperBound)
{
- this.replayPosition = replayPosition;
+ this.commitLogUpperBound = commitLogUpperBound;
return this;
}
@@@ -276,15 -280,13 +300,16 @@@
{
Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
- components.put(MetadataType.STATS, new StatsMetadata(estimatedRowSize,
- estimatedColumnCount,
+ components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
+ estimatedCellPerPartitionCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
- minTimestamp,
- maxTimestamp,
- maxLocalDeletionTime,
+ timestampTracker.min(),
+ timestampTracker.max(),
+ localDeletionTimeTracker.min(),
+ localDeletionTimeTracker.max(),
+ ttlTracker.min(),
+ ttlTracker.max(),
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index 824bf8b,3d48e34..07e35bb
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@@ -40,53 -41,43 +40,56 @@@ public class StatsMetadata extends Meta
{
public static final IMetadataComponentSerializer serializer = new StatsMetadataSerializer();
- public final EstimatedHistogram estimatedRowSize;
+ public final EstimatedHistogram estimatedPartitionSize;
public final EstimatedHistogram estimatedColumnCount;
- public final ReplayPosition replayPosition;
+ public final ReplayPosition commitLogLowerBound;
+ public final ReplayPosition commitLogUpperBound;
public final long minTimestamp;
public final long maxTimestamp;
+ public final int minLocalDeletionTime;
public final int maxLocalDeletionTime;
+ public final int minTTL;
+ public final int maxTTL;
public final double compressionRatio;
public final StreamingHistogram estimatedTombstoneDropTime;
public final int sstableLevel;
- public final List<ByteBuffer> maxColumnNames;
- public final List<ByteBuffer> minColumnNames;
+ public final List<ByteBuffer> minClusteringValues;
+ public final List<ByteBuffer> maxClusteringValues;
public final boolean hasLegacyCounterShards;
public final long repairedAt;
+ public final long totalColumnsSet;
+ public final long totalRows;
- public StatsMetadata(EstimatedHistogram estimatedRowSize,
+ public StatsMetadata(EstimatedHistogram estimatedPartitionSize,
EstimatedHistogram estimatedColumnCount,
- ReplayPosition replayPosition,
+ ReplayPosition commitLogLowerBound,
+ ReplayPosition commitLogUpperBound,
long minTimestamp,
long maxTimestamp,
+ int minLocalDeletionTime,
int maxLocalDeletionTime,
+ int minTTL,
+ int maxTTL,
double compressionRatio,
StreamingHistogram estimatedTombstoneDropTime,
int sstableLevel,
- List<ByteBuffer> minColumnNames,
- List<ByteBuffer> maxColumnNames,
+ List<ByteBuffer> minClusteringValues,
+ List<ByteBuffer> maxClusteringValues,
boolean hasLegacyCounterShards,
- long repairedAt)
+ long repairedAt,
+ long totalColumnsSet,
+ long totalRows)
{
- this.estimatedRowSize = estimatedRowSize;
+ this.estimatedPartitionSize = estimatedPartitionSize;
this.estimatedColumnCount = estimatedColumnCount;
- this.replayPosition = replayPosition;
+ this.commitLogLowerBound = commitLogLowerBound;
+ this.commitLogUpperBound = commitLogUpperBound;
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
+ this.minLocalDeletionTime = minLocalDeletionTime;
this.maxLocalDeletionTime = maxLocalDeletionTime;
+ this.minTTL = minTTL;
+ this.maxTTL = maxTTL;
this.compressionRatio = compressionRatio;
this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
this.sstableLevel = sstableLevel;
@@@ -129,15 -118,13 +132,16 @@@
public StatsMetadata mutateLevel(int newLevel)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
newLevel,
@@@ -151,15 -136,13 +155,16 @@@
public StatsMetadata mutateRepairedAt(long newRepairedAt)
{
- return new StatsMetadata(estimatedRowSize,
+ return new StatsMetadata(estimatedPartitionSize,
estimatedColumnCount,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
estimatedTombstoneDropTime,
sstableLevel,
@@@ -179,15 -160,13 +184,16 @@@
StatsMetadata that = (StatsMetadata) o;
return new EqualsBuilder()
- .append(estimatedRowSize, that.estimatedRowSize)
+ .append(estimatedPartitionSize, that.estimatedPartitionSize)
.append(estimatedColumnCount, that.estimatedColumnCount)
- .append(replayPosition, that.replayPosition)
+ .append(commitLogLowerBound, that.commitLogLowerBound)
+ .append(commitLogUpperBound, that.commitLogUpperBound)
.append(minTimestamp, that.minTimestamp)
.append(maxTimestamp, that.maxTimestamp)
+ .append(minLocalDeletionTime, that.minLocalDeletionTime)
.append(maxLocalDeletionTime, that.maxLocalDeletionTime)
+ .append(minTTL, that.minTTL)
+ .append(maxTTL, that.maxTTL)
.append(compressionRatio, that.compressionRatio)
.append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime)
.append(sstableLevel, that.sstableLevel)
@@@ -204,15 -181,13 +210,16 @@@
public int hashCode()
{
return new HashCodeBuilder()
- .append(estimatedRowSize)
+ .append(estimatedPartitionSize)
.append(estimatedColumnCount)
- .append(replayPosition)
+ .append(commitLogLowerBound)
+ .append(commitLogUpperBound)
.append(minTimestamp)
.append(maxTimestamp)
+ .append(minLocalDeletionTime)
.append(maxLocalDeletionTime)
+ .append(minTTL)
+ .append(maxTTL)
.append(compressionRatio)
.append(estimatedTombstoneDropTime)
.append(sstableLevel)
@@@ -227,78 -200,61 +234,84 @@@
public static class StatsMetadataSerializer implements IMetadataComponentSerializer<StatsMetadata>
{
- public int serializedSize(StatsMetadata component, Version version) throws IOException
+ public int serializedSize(Version version, StatsMetadata component) throws IOException
{
int size = 0;
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedRowSize, TypeSizes.NATIVE);
- size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount, TypeSizes.NATIVE);
- size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound, TypeSizes.NATIVE);
- size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
- size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime, TypeSizes.NATIVE);
- size += TypeSizes.NATIVE.sizeof(component.sstableLevel);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
+ size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
- size += ReplayPosition.serializer.serializedSize(component.replayPosition);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogUpperBound);
+ if (version.storeRows())
+ size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
+ else
+ size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
+ size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
+ size += TypeSizes.sizeof(component.sstableLevel);
// min column names
size += 4;
- for (ByteBuffer columnName : component.minColumnNames)
- size += 2 + columnName.remaining(); // with short length
+ for (ByteBuffer value : component.minClusteringValues)
+ size += 2 + value.remaining(); // with short length
// max column names
size += 4;
- for (ByteBuffer columnName : component.maxColumnNames)
- size += 2 + columnName.remaining(); // with short length
- size += TypeSizes.NATIVE.sizeof(component.hasLegacyCounterShards);
+ for (ByteBuffer value : component.maxClusteringValues)
+ size += 2 + value.remaining(); // with short length
+ size += TypeSizes.sizeof(component.hasLegacyCounterShards);
+ if (version.storeRows())
+ size += 8 + 8; // totalColumnsSet, totalRows
+ if (version.hasCommitLogLowerBound())
- size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound, TypeSizes.NATIVE);
++ size += ReplayPosition.serializer.serializedSize(component.commitLogLowerBound);
return size;
}
- public void serialize(StatsMetadata component, Version version, DataOutputPlus out) throws IOException
+ public void serialize(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
{
- EstimatedHistogram.serializer.serialize(component.estimatedRowSize, out);
+ EstimatedHistogram.serializer.serialize(component.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(component.estimatedColumnCount, out);
- ReplayPosition.serializer.serialize(component.replayPosition, out);
+ ReplayPosition.serializer.serialize(component.commitLogUpperBound, out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
+ if (version.storeRows())
+ out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.maxLocalDeletionTime);
+ if (version.storeRows())
+ {
+ out.writeInt(component.minTTL);
+ out.writeInt(component.maxTTL);
+ }
out.writeDouble(component.compressionRatio);
StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
out.writeLong(component.repairedAt);
- out.writeInt(component.minColumnNames.size());
- for (ByteBuffer columnName : component.minColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
- out.writeInt(component.maxColumnNames.size());
- for (ByteBuffer columnName : component.maxColumnNames)
- ByteBufferUtil.writeWithShortLength(columnName, out);
+ out.writeInt(component.minClusteringValues.size());
+ for (ByteBuffer value : component.minClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
+ out.writeInt(component.maxClusteringValues.size());
+ for (ByteBuffer value : component.maxClusteringValues)
+ ByteBufferUtil.writeWithShortLength(value, out);
out.writeBoolean(component.hasLegacyCounterShards);
+
+ if (version.storeRows())
+ {
+ out.writeLong(component.totalColumnsSet);
+ out.writeLong(component.totalRows);
+ }
++
+ if (version.hasCommitLogLowerBound())
+ ReplayPosition.serializer.serialize(component.commitLogLowerBound, out);
}
- public StatsMetadata deserialize(Version version, DataInput in) throws IOException
+ public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
- EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
+ EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
- ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
+ ReplayPosition commitLogLowerBound = ReplayPosition.NONE, commitLogUpperBound;
+ commitLogUpperBound = ReplayPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
+ // We use MAX_VALUE as that's the default value for "no deletion time"
+ int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
int maxLocalDeletionTime = in.readInt();
+ int minTTL = version.storeRows() ? in.readInt() : 0;
+ int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
double compressionRatio = in.readDouble();
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
@@@ -320,18 -276,15 +333,22 @@@
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
+ long totalColumnsSet = version.storeRows() ? in.readLong() : -1L;
+ long totalRows = version.storeRows() ? in.readLong() : -1L;
+
+ if (version.hasCommitLogLowerBound())
+ commitLogLowerBound = ReplayPosition.serializer.deserialize(in);
- return new StatsMetadata(rowSizes,
++
+ return new StatsMetadata(partitionSizes,
columnCounts,
- replayPosition,
+ commitLogLowerBound,
+ commitLogUpperBound,
minTimestamp,
maxTimestamp,
+ minLocalDeletionTime,
maxLocalDeletionTime,
+ minTTL,
+ maxTTL,
compressionRatio,
tombstoneHistogram,
sstableLevel,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-CompressionInfo.db
index 0000000,0000000..e32d12f
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Data.db
index 0000000,0000000..790cbe1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
index 0000000,0000000..66d1d34
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++163939259
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Index.db
index 0000000,0000000..38d0537
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Statistics.db
index 0000000,0000000..a53b7c1
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..e3d4377
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Data.db
index 0000000,0000000..c5aa7ce
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
index 0000000,0000000..4e3bf89
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3387396134
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Index.db
index 0000000,0000000..54afd0c
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Statistics.db
index 0000000,0000000..9d9b085
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-CompressionInfo.db
index 0000000,0000000..5a34d6e
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Data.db
index 0000000,0000000..55ad88d
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
index 0000000,0000000..86240b5
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++346435755
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Index.db
index 0000000,0000000..d1a401b
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Statistics.db
index 0000000,0000000..4f3f45a
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-CompressionInfo.db
index 0000000,0000000..32fa731
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Data.db
index 0000000,0000000..ca10874
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
index 0000000,0000000..98f5784
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Digest.crc32
@@@ -1,0 -1,0 +1,1 @@@
++3740016396
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Filter.db
index 0000000,0000000..2e1d5d2
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Index.db
index 0000000,0000000..3661d77
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Statistics.db
index 0000000,0000000..bb1e3fb
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-Summary.db
index 0000000,0000000..9b24e04
new file mode 100644
Binary files differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/78a3d2bb/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
----------------------------------------------------------------------
diff --cc test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
index 0000000,0000000..fe9581f
new file mode 100644
--- /dev/null
+++ b/test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_counter_compact/mb-1-big-TOC.txt
@@@ -1,0 -1,0 +1,8 @@@
++CompressionInfo.db
++Summary.db
++TOC.txt
++Digest.crc32
++Filter.db
++Statistics.db
++Data.db
++Index.db