You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2015/05/15 18:24:47 UTC
[1/3] cassandra git commit: Fix canonical view returning early opened
SSTables
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 01087489e -> 7d6a60ddd
refs/heads/trunk 6c489afed -> 266c5043c
Fix canonical view returning early opened SSTables
patch by benedict; reviewed by yukim for CASSANDRA-9396
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d6a60dd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d6a60dd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d6a60dd
Branch: refs/heads/cassandra-2.1
Commit: 7d6a60ddd0e156f6028c05d97fa9179f86fe02ee
Parents: 0108748
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Fri May 15 11:19:23 2015 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Fri May 15 11:21:25 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 8 ++--
.../io/sstable/SSTableRewriterTest.java | 47 +++++++++++++++++++-
3 files changed, 52 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 80619d0..129f6a1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,7 @@
* Fix anticompaction blocking ANTI_ENTROPY stage (CASSANDRA-9151)
* Repair waits for anticompaction to finish (CASSANDRA-9097)
* Fix streaming not holding ref when stream error (CASSANDRA-9295)
+ * Fix canonical view returning early opened SSTables (CASSANDRA-9396)
Merged from 2.0:
* Clone SliceQueryFilter in AbstractReadCommand implementations (CASSANDRA-8940)
* Push correct protocol notification for DROP INDEX (CASSANDRA-9310)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 978037e..bdc2d8b 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2892,10 +2892,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> apply(DataTracker.View view)
{
List<SSTableReader> sstables = new ArrayList<>();
- sstables.addAll(view.compacting);
+ for (SSTableReader sstable : view.compacting)
+ if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+ sstables.add(sstable);
for (SSTableReader sstable : view.sstables)
- if (!view.compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY)
- sstables.add(sstable);
+ if (!view.compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY)
+ sstables.add(sstable);
return sstables;
}
};
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 09937bc..b940b7b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
@@ -40,10 +41,15 @@ import org.apache.cassandra.db.compaction.SSTableSplitter;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.Refs;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -81,7 +87,7 @@ public class SSTableRewriterTest extends SchemaLoader
}
}
Collection<SSTableReader> newsstables = writer.finish();
- cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables , OperationType.COMPACTION);
+ cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables, OperationType.COMPACTION);
Thread.sleep(100);
validateCFS(cfs);
int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list(), 0, 0);
@@ -733,6 +739,45 @@ public class SSTableRewriterTest extends SchemaLoader
validateCFS(cfs);
}
+ @Test
+ public void testCanonicalView() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ cfs.truncateBlocking();
+
+ SSTableReader s = writeFile(cfs, 1000);
+ cfs.addSSTable(s);
+ Set<SSTableReader> sstables = Sets.newHashSet(cfs.markAllCompacting());
+ assertEquals(1, sstables.size());
+ SSTableRewriter.overrideOpenInterval(10000000);
+ SSTableRewriter writer = new SSTableRewriter(cfs, sstables, 1000, false);
+ boolean checked = false;
+ try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables))
+ {
+ ISSTableScanner scanner = scanners.scanners.get(0);
+ CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis()));
+ writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory));
+ while (scanner.hasNext())
+ {
+ AbstractCompactedRow row = new LazilyCompactedRow(controller, Collections.singletonList(scanner.next()));
+ writer.append(row);
+ if (!checked && writer.currentWriter().getFilePointer() > 15000000)
+ {
+ checked = true;
+ ColumnFamilyStore.ViewFragment viewFragment = cfs.select(ColumnFamilyStore.CANONICAL_SSTABLES);
+ // canonical view should have only one SSTable which is not opened early.
+ assertEquals(1, viewFragment.sstables.size());
+ SSTableReader sstable = viewFragment.sstables.get(0);
+ assertEquals(s.descriptor, sstable.descriptor);
+ assertTrue("Found early opened SSTable in canonical view: " + sstable.getFilename(), sstable.openReason != SSTableReader.OpenReason.EARLY);
+ }
+ }
+ }
+ writer.finish();
+ cfs.getDataTracker().unmarkCompacting(sstables);
+ }
+
private void validateKeys(Keyspace ks)
{
for (int i = 0; i < 100; i++)
[2/3] cassandra git commit: Fix canonical view returning early opened
SSTables
Posted by yu...@apache.org.
Fix canonical view returning early opened SSTables
patch by benedict; reviewed by yukim for CASSANDRA-9396
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d6a60dd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d6a60dd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d6a60dd
Branch: refs/heads/trunk
Commit: 7d6a60ddd0e156f6028c05d97fa9179f86fe02ee
Parents: 0108748
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Fri May 15 11:19:23 2015 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Fri May 15 11:21:25 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 8 ++--
.../io/sstable/SSTableRewriterTest.java | 47 +++++++++++++++++++-
3 files changed, 52 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 80619d0..129f6a1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,7 @@
* Fix anticompaction blocking ANTI_ENTROPY stage (CASSANDRA-9151)
* Repair waits for anticompaction to finish (CASSANDRA-9097)
* Fix streaming not holding ref when stream error (CASSANDRA-9295)
+ * Fix canonical view returning early opened SSTables (CASSANDRA-9396)
Merged from 2.0:
* Clone SliceQueryFilter in AbstractReadCommand implementations (CASSANDRA-8940)
* Push correct protocol notification for DROP INDEX (CASSANDRA-9310)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 978037e..bdc2d8b 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2892,10 +2892,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> apply(DataTracker.View view)
{
List<SSTableReader> sstables = new ArrayList<>();
- sstables.addAll(view.compacting);
+ for (SSTableReader sstable : view.compacting)
+ if (sstable.openReason != SSTableReader.OpenReason.EARLY)
+ sstables.add(sstable);
for (SSTableReader sstable : view.sstables)
- if (!view.compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY)
- sstables.add(sstable);
+ if (!view.compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY)
+ sstables.add(sstable);
return sstables;
}
};
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d6a60dd/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 09937bc..b940b7b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
@@ -40,10 +41,15 @@ import org.apache.cassandra.db.compaction.SSTableSplitter;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataIntegrityMetadata;
+import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.Refs;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -81,7 +87,7 @@ public class SSTableRewriterTest extends SchemaLoader
}
}
Collection<SSTableReader> newsstables = writer.finish();
- cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables , OperationType.COMPACTION);
+ cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables, OperationType.COMPACTION);
Thread.sleep(100);
validateCFS(cfs);
int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list(), 0, 0);
@@ -733,6 +739,45 @@ public class SSTableRewriterTest extends SchemaLoader
validateCFS(cfs);
}
+ @Test
+ public void testCanonicalView() throws IOException
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+ cfs.truncateBlocking();
+
+ SSTableReader s = writeFile(cfs, 1000);
+ cfs.addSSTable(s);
+ Set<SSTableReader> sstables = Sets.newHashSet(cfs.markAllCompacting());
+ assertEquals(1, sstables.size());
+ SSTableRewriter.overrideOpenInterval(10000000);
+ SSTableRewriter writer = new SSTableRewriter(cfs, sstables, 1000, false);
+ boolean checked = false;
+ try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables))
+ {
+ ISSTableScanner scanner = scanners.scanners.get(0);
+ CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis()));
+ writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory));
+ while (scanner.hasNext())
+ {
+ AbstractCompactedRow row = new LazilyCompactedRow(controller, Collections.singletonList(scanner.next()));
+ writer.append(row);
+ if (!checked && writer.currentWriter().getFilePointer() > 15000000)
+ {
+ checked = true;
+ ColumnFamilyStore.ViewFragment viewFragment = cfs.select(ColumnFamilyStore.CANONICAL_SSTABLES);
+ // canonical view should have only one SSTable which is not opened early.
+ assertEquals(1, viewFragment.sstables.size());
+ SSTableReader sstable = viewFragment.sstables.get(0);
+ assertEquals(s.descriptor, sstable.descriptor);
+ assertTrue("Found early opened SSTable in canonical view: " + sstable.getFilename(), sstable.openReason != SSTableReader.OpenReason.EARLY);
+ }
+ }
+ }
+ writer.finish();
+ cfs.getDataTracker().unmarkCompacting(sstables);
+ }
+
private void validateKeys(Keyspace ks)
{
for (int i = 0; i < 100; i++)
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by yu...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/266c5043
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/266c5043
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/266c5043
Branch: refs/heads/trunk
Commit: 266c5043c24bef02dd1a0c32a433fe7733677fd4
Parents: 6c489af 7d6a60d
Author: Yuki Morishita <yu...@apache.org>
Authored: Fri May 15 11:24:30 2015 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Fri May 15 11:24:30 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 4 +-
.../io/sstable/SSTableRewriterTest.java | 40 ++++++++++++++++++++
3 files changed, 44 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/266c5043/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/266c5043/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/266c5043/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------