You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2016/03/29 11:23:16 UTC
[01/10] cassandra git commit: Notify that sstables were changed after
compaction was cancelled
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 897ffe87e -> aefea62a0
refs/heads/cassandra-3.0 0c91977da -> 01048346e
refs/heads/cassandra-3.5 0a2508544 -> df18c9b6c
refs/heads/trunk f04224707 -> b5b2ffa8d
Notify that sstables were changed after compaction was cancelled
Patch by marcuse; reviewed by benedict for CASSANDRA-11373
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/aefea62a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/aefea62a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/aefea62a
Branch: refs/heads/cassandra-2.2
Commit: aefea62a065bbb400242c5adf412da8e3e21a147
Parents: 897ffe8
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 22 09:24:35 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:16:32 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableRewriterTest.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 098d062..5e8828b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 9b52269..59cee50 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -181,7 +181,10 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
accumulate = markObsolete(null, obsolete, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 07820fd..c2cc6e3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -929,6 +930,8 @@ public class SSTableRewriterTest extends SchemaLoader
assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ if (cfs.getSSTables().size() > 0)
+ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore(System.currentTimeMillis()/1000), false).isEmpty());
}
public static int assertFileCounts(String [] files, int expectedtmplinkCount, int expectedtmpCount)
[04/10] cassandra git commit: Notify that sstables were changed after
compaction was cancelled
Posted by ma...@apache.org.
Notify that sstables were changed after compaction was cancelled
Patch by marcuse; reviewed by benedict for CASSANDRA-11373
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/aefea62a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/aefea62a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/aefea62a
Branch: refs/heads/trunk
Commit: aefea62a065bbb400242c5adf412da8e3e21a147
Parents: 897ffe8
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 22 09:24:35 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:16:32 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableRewriterTest.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 098d062..5e8828b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 9b52269..59cee50 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -181,7 +181,10 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
accumulate = markObsolete(null, obsolete, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 07820fd..c2cc6e3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -929,6 +930,8 @@ public class SSTableRewriterTest extends SchemaLoader
assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ if (cfs.getSSTables().size() > 0)
+ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore(System.currentTimeMillis()/1000), false).isEmpty());
}
public static int assertFileCounts(String [] files, int expectedtmplinkCount, int expectedtmpCount)
[03/10] cassandra git commit: Notify that sstables were changed after
compaction was cancelled
Posted by ma...@apache.org.
Notify that sstables were changed after compaction was cancelled
Patch by marcuse; reviewed by benedict for CASSANDRA-11373
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/aefea62a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/aefea62a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/aefea62a
Branch: refs/heads/cassandra-3.5
Commit: aefea62a065bbb400242c5adf412da8e3e21a147
Parents: 897ffe8
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 22 09:24:35 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:16:32 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableRewriterTest.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 098d062..5e8828b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 9b52269..59cee50 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -181,7 +181,10 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
accumulate = markObsolete(null, obsolete, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 07820fd..c2cc6e3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -929,6 +930,8 @@ public class SSTableRewriterTest extends SchemaLoader
assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ if (cfs.getSSTables().size() > 0)
+ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore(System.currentTimeMillis()/1000), false).isEmpty());
}
public static int assertFileCounts(String [] files, int expectedtmplinkCount, int expectedtmpCount)
[05/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/01048346
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/01048346
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/01048346
Branch: refs/heads/cassandra-3.5
Commit: 01048346e436cacfd28a38447469bfd35e689883
Parents: 0c91977 aefea62
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:20:55 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:20:55 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fdc873e,5e8828b..a9ccf98
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,16 -1,5 +1,17 @@@
-2.2.6
+3.0.5
+ * Enable SO_REUSEADDR for JMX RMI server sockets (CASSANDRA-11093)
+ * Allocate merkletrees with the correct size (CASSANDRA-11390)
+ * Support streaming pre-3.0 sstables (CASSANDRA-10990)
+ * Add backpressure to compressed commit log (CASSANDRA-10971)
+ * SSTableExport supports secondary index tables (CASSANDRA-11330)
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index a5eb01f,59cee50..91515aa
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@@ -236,15 -176,15 +236,18 @@@ public class LifecycleTransaction exten
// mark obsolete all readers that are not versions of those present in the original set
Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
logger.trace("Obsoleting {}", obsolete);
- // we don't pass the tracker in for the obsoletion, since these readers have never been notified externally
- // nor had their size accounting affected
- accumulate = markObsolete(null, obsolete, accumulate);
+
+ accumulate = prepareForObsoletion(obsolete, log, obsoletions = new ArrayList<>(), accumulate);
+ // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
+ // a failure to abort, which is useful information to have for debug
+ accumulate = log.abort(accumulate);
+ accumulate = markObsolete(obsoletions, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 0af743d,0000000..2db92f7
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@@ -1,166 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
++import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SSTableWriterTestBase extends SchemaLoader
+{
+
+ protected static final String KEYSPACE = "SSTableRewriterTest";
+ protected static final String CF = "Standard1";
+
+ private static Config.DiskAccessMode standardMode;
+ private static Config.DiskAccessMode indexMode;
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ if (FBUtilities.isWindows())
+ {
+ standardMode = DatabaseDescriptor.getDiskAccessMode();
+ indexMode = DatabaseDescriptor.getIndexAccessMode();
+
+ DatabaseDescriptor.setDiskAccessMode(Config.DiskAccessMode.standard);
+ DatabaseDescriptor.setIndexAccessMode(Config.DiskAccessMode.standard);
+ }
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF));
+ }
+
+ @AfterClass
+ public static void revertDiskAccess()
+ {
+ DatabaseDescriptor.setDiskAccessMode(standardMode);
+ DatabaseDescriptor.setIndexAccessMode(indexMode);
+ }
+
+ @After
+ public void truncateCF()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ }
+
+ public static void truncate(ColumnFamilyStore cfs)
+ {
+ cfs.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
+ assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount());
+ validateCFS(cfs);
+ }
+
+ public static void validateCFS(ColumnFamilyStore cfs)
+ {
+ Set<Integer> liveDescriptors = new HashSet<>();
+ long spaceUsed = 0;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ assertFalse(sstable.isMarkedCompacted());
+ assertEquals(1, sstable.selfRef().globalCount());
+ liveDescriptors.add(sstable.descriptor.generation);
+ spaceUsed += sstable.bytesOnDisk();
+ }
+ for (File dir : cfs.getDirectories().getCFDirectories())
+ {
+ for (File f : dir.listFiles())
+ {
+ if (f.getName().contains("Data"))
+ {
+ Descriptor d = Descriptor.fromFilename(f.getAbsolutePath());
+ assertTrue(d.toString(), liveDescriptors.contains(d.generation));
+ }
+ }
+ }
+ assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
++ if(cfs.getLiveSSTables().size() > 0)
++ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore((int) (System.currentTimeMillis() / 1000)), false).isEmpty());
+ }
+
+ public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
+ {
+ String filename = cfs.getSSTablePath(directory);
+ return SSTableWriter.create(filename, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS), txn);
+ }
+
+ public static ByteBuffer random(int i, int size)
+ {
+ byte[] bytes = new byte[size + 4];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ ByteBuffer r = ByteBuffer.wrap(bytes);
+ r.putInt(0, i);
+ return r;
+ }
+
+ public static int assertFileCounts(String [] files)
+ {
+ int tmplinkcount = 0;
+ int tmpcount = 0;
+ int datacount = 0;
+ for (String f : files)
+ {
+ if (f.endsWith("-CRC.db"))
+ continue;
+ if (f.contains("tmplink-"))
+ tmplinkcount++;
+ else if (f.contains("tmp-"))
+ tmpcount++;
+ else if (f.contains("Data"))
+ datacount++;
+ }
+ assertEquals(0, tmplinkcount);
+ assertEquals(0, tmpcount);
+ return datacount;
+ }
+}
[07/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/01048346
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/01048346
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/01048346
Branch: refs/heads/trunk
Commit: 01048346e436cacfd28a38447469bfd35e689883
Parents: 0c91977 aefea62
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:20:55 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:20:55 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fdc873e,5e8828b..a9ccf98
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,16 -1,5 +1,17 @@@
-2.2.6
+3.0.5
+ * Enable SO_REUSEADDR for JMX RMI server sockets (CASSANDRA-11093)
+ * Allocate merkletrees with the correct size (CASSANDRA-11390)
+ * Support streaming pre-3.0 sstables (CASSANDRA-10990)
+ * Add backpressure to compressed commit log (CASSANDRA-10971)
+ * SSTableExport supports secondary index tables (CASSANDRA-11330)
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index a5eb01f,59cee50..91515aa
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@@ -236,15 -176,15 +236,18 @@@ public class LifecycleTransaction exten
// mark obsolete all readers that are not versions of those present in the original set
Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
logger.trace("Obsoleting {}", obsolete);
- // we don't pass the tracker in for the obsoletion, since these readers have never been notified externally
- // nor had their size accounting affected
- accumulate = markObsolete(null, obsolete, accumulate);
+
+ accumulate = prepareForObsoletion(obsolete, log, obsoletions = new ArrayList<>(), accumulate);
+ // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
+ // a failure to abort, which is useful information to have for debug
+ accumulate = log.abort(accumulate);
+ accumulate = markObsolete(obsoletions, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 0af743d,0000000..2db92f7
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@@ -1,166 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
++import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SSTableWriterTestBase extends SchemaLoader
+{
+
+ protected static final String KEYSPACE = "SSTableRewriterTest";
+ protected static final String CF = "Standard1";
+
+ private static Config.DiskAccessMode standardMode;
+ private static Config.DiskAccessMode indexMode;
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ if (FBUtilities.isWindows())
+ {
+ standardMode = DatabaseDescriptor.getDiskAccessMode();
+ indexMode = DatabaseDescriptor.getIndexAccessMode();
+
+ DatabaseDescriptor.setDiskAccessMode(Config.DiskAccessMode.standard);
+ DatabaseDescriptor.setIndexAccessMode(Config.DiskAccessMode.standard);
+ }
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF));
+ }
+
+ @AfterClass
+ public static void revertDiskAccess()
+ {
+ DatabaseDescriptor.setDiskAccessMode(standardMode);
+ DatabaseDescriptor.setIndexAccessMode(indexMode);
+ }
+
+ @After
+ public void truncateCF()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ }
+
+ public static void truncate(ColumnFamilyStore cfs)
+ {
+ cfs.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
+ assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount());
+ validateCFS(cfs);
+ }
+
+ public static void validateCFS(ColumnFamilyStore cfs)
+ {
+ Set<Integer> liveDescriptors = new HashSet<>();
+ long spaceUsed = 0;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ assertFalse(sstable.isMarkedCompacted());
+ assertEquals(1, sstable.selfRef().globalCount());
+ liveDescriptors.add(sstable.descriptor.generation);
+ spaceUsed += sstable.bytesOnDisk();
+ }
+ for (File dir : cfs.getDirectories().getCFDirectories())
+ {
+ for (File f : dir.listFiles())
+ {
+ if (f.getName().contains("Data"))
+ {
+ Descriptor d = Descriptor.fromFilename(f.getAbsolutePath());
+ assertTrue(d.toString(), liveDescriptors.contains(d.generation));
+ }
+ }
+ }
+ assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
++ if(cfs.getLiveSSTables().size() > 0)
++ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore((int) (System.currentTimeMillis() / 1000)), false).isEmpty());
+ }
+
+ public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
+ {
+ String filename = cfs.getSSTablePath(directory);
+ return SSTableWriter.create(filename, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS), txn);
+ }
+
+ public static ByteBuffer random(int i, int size)
+ {
+ byte[] bytes = new byte[size + 4];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ ByteBuffer r = ByteBuffer.wrap(bytes);
+ r.putInt(0, i);
+ return r;
+ }
+
+ public static int assertFileCounts(String [] files)
+ {
+ int tmplinkcount = 0;
+ int tmpcount = 0;
+ int datacount = 0;
+ for (String f : files)
+ {
+ if (f.endsWith("-CRC.db"))
+ continue;
+ if (f.contains("tmplink-"))
+ tmplinkcount++;
+ else if (f.contains("tmp-"))
+ tmpcount++;
+ else if (f.contains("Data"))
+ datacount++;
+ }
+ assertEquals(0, tmplinkcount);
+ assertEquals(0, tmpcount);
+ return datacount;
+ }
+}
[02/10] cassandra git commit: Notify that sstables were changed after
compaction was cancelled
Posted by ma...@apache.org.
Notify that sstables were changed after compaction was cancelled
Patch by marcuse; reviewed by benedict for CASSANDRA-11373
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/aefea62a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/aefea62a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/aefea62a
Branch: refs/heads/cassandra-3.0
Commit: aefea62a065bbb400242c5adf412da8e3e21a147
Parents: 897ffe8
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 22 09:24:35 2016 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:16:32 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableRewriterTest.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 098d062..5e8828b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.6
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 9b52269..59cee50 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -181,7 +181,10 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
accumulate = markObsolete(null, obsolete, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/aefea62a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index 07820fd..c2cc6e3 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LazilyCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -929,6 +930,8 @@ public class SSTableRewriterTest extends SchemaLoader
assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
assertTrue(cfs.getTracker().getCompacting().isEmpty());
+ if (cfs.getSSTables().size() > 0)
+ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore(System.currentTimeMillis()/1000), false).isEmpty());
}
public static int assertFileCounts(String [] files, int expectedtmplinkCount, int expectedtmpCount)
[09/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.5
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/df18c9b6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/df18c9b6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/df18c9b6
Branch: refs/heads/trunk
Commit: df18c9b6c13225c9f62ce6ace544060f3264c8f0
Parents: 0a25085 0104834
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:21:11 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:21:11 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df18c9b6/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df18c9b6/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
----------------------------------------------------------------------
[08/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.5
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.5
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/df18c9b6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/df18c9b6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/df18c9b6
Branch: refs/heads/cassandra-3.5
Commit: df18c9b6c13225c9f62ce6ace544060f3264c8f0
Parents: 0a25085 0104834
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:21:11 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:21:11 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df18c9b6/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/df18c9b6/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
----------------------------------------------------------------------
[10/10] cassandra git commit: Merge branch 'cassandra-3.5' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.5' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b5b2ffa8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b5b2ffa8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b5b2ffa8
Branch: refs/heads/trunk
Commit: b5b2ffa8d75c462762bea29fb62421371e1ffcb7
Parents: f042247 df18c9b
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:21:50 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:21:50 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5b2ffa8/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b5b2ffa8/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
[06/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by ma...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/01048346
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/01048346
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/01048346
Branch: refs/heads/cassandra-3.0
Commit: 01048346e436cacfd28a38447469bfd35e689883
Parents: 0c91977 aefea62
Author: Marcus Eriksson <ma...@apache.org>
Authored: Tue Mar 29 11:20:55 2016 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Tue Mar 29 11:20:55 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/lifecycle/LifecycleTransaction.java | 5 ++++-
.../org/apache/cassandra/io/sstable/SSTableWriterTestBase.java | 3 +++
3 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fdc873e,5e8828b..a9ccf98
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,16 -1,5 +1,17 @@@
-2.2.6
+3.0.5
+ * Enable SO_REUSEADDR for JMX RMI server sockets (CASSANDRA-11093)
+ * Allocate merkletrees with the correct size (CASSANDRA-11390)
+ * Support streaming pre-3.0 sstables (CASSANDRA-10990)
+ * Add backpressure to compressed commit log (CASSANDRA-10971)
+ * SSTableExport supports secondary index tables (CASSANDRA-11330)
+ * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
+ * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
+ * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
+ * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
+ * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
+ * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
+Merged from 2.2:
+ * Notify when sstables change after cancelling compaction (CASSANDRA-11373)
* cqlsh: COPY FROM should check that explicit column names are valid (CASSANDRA-11333)
* Add -Dcassandra.start_gossip startup option (CASSANDRA-10809)
* Fix UTF8Validator.validate() for modified UTF-8 (CASSANDRA-10748)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index a5eb01f,59cee50..91515aa
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@@ -236,15 -176,15 +236,18 @@@ public class LifecycleTransaction exten
// mark obsolete all readers that are not versions of those present in the original set
Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
logger.trace("Obsoleting {}", obsolete);
- // we don't pass the tracker in for the obsoletion, since these readers have never been notified externally
- // nor had their size accounting affected
- accumulate = markObsolete(null, obsolete, accumulate);
+
+ accumulate = prepareForObsoletion(obsolete, log, obsoletions = new ArrayList<>(), accumulate);
+ // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
+ // a failure to abort, which is useful information to have for debug
+ accumulate = log.abort(accumulate);
+ accumulate = markObsolete(obsoletions, accumulate);
// replace all updated readers with a version restored to its original state
- accumulate = tracker.apply(updateLiveSet(logged.update, restoreUpdatedOriginals()), accumulate);
+ List<SSTableReader> restored = restoreUpdatedOriginals();
+ List<SSTableReader> invalid = Lists.newArrayList(Iterables.concat(logged.update, logged.obsolete));
+ accumulate = tracker.apply(updateLiveSet(logged.update, restored), accumulate);
+ accumulate = tracker.notifySSTablesChanged(invalid, restored, OperationType.COMPACTION, accumulate);
// setReplaced immediately preceding versions that have not been obsoleted
accumulate = setReplaced(logged.update, accumulate);
// we have replaced all of logged.update and never made visible staged.update,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/01048346/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 0af743d,0000000..2db92f7
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@@ -1,166 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SerializationHeader;
++import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.EncodingStats;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class SSTableWriterTestBase extends SchemaLoader
+{
+
+ protected static final String KEYSPACE = "SSTableRewriterTest";
+ protected static final String CF = "Standard1";
+
+ private static Config.DiskAccessMode standardMode;
+ private static Config.DiskAccessMode indexMode;
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException
+ {
+ if (FBUtilities.isWindows())
+ {
+ standardMode = DatabaseDescriptor.getDiskAccessMode();
+ indexMode = DatabaseDescriptor.getIndexAccessMode();
+
+ DatabaseDescriptor.setDiskAccessMode(Config.DiskAccessMode.standard);
+ DatabaseDescriptor.setIndexAccessMode(Config.DiskAccessMode.standard);
+ }
+
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE,
+ KeyspaceParams.simple(1),
+ SchemaLoader.standardCFMD(KEYSPACE, CF));
+ }
+
+ @AfterClass
+ public static void revertDiskAccess()
+ {
+ DatabaseDescriptor.setDiskAccessMode(standardMode);
+ DatabaseDescriptor.setIndexAccessMode(indexMode);
+ }
+
+ @After
+ public void truncateCF()
+ {
+ Keyspace keyspace = Keyspace.open(KEYSPACE);
+ ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
+ store.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ }
+
+ public static void truncate(ColumnFamilyStore cfs)
+ {
+ cfs.truncateBlocking();
+ LifecycleTransaction.waitForDeletions();
+ Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
+ assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount());
+ validateCFS(cfs);
+ }
+
+ public static void validateCFS(ColumnFamilyStore cfs)
+ {
+ Set<Integer> liveDescriptors = new HashSet<>();
+ long spaceUsed = 0;
+ for (SSTableReader sstable : cfs.getLiveSSTables())
+ {
+ assertFalse(sstable.isMarkedCompacted());
+ assertEquals(1, sstable.selfRef().globalCount());
+ liveDescriptors.add(sstable.descriptor.generation);
+ spaceUsed += sstable.bytesOnDisk();
+ }
+ for (File dir : cfs.getDirectories().getCFDirectories())
+ {
+ for (File f : dir.listFiles())
+ {
+ if (f.getName().contains("Data"))
+ {
+ Descriptor d = Descriptor.fromFilename(f.getAbsolutePath());
+ assertTrue(d.toString(), liveDescriptors.contains(d.generation));
+ }
+ }
+ }
+ assertEquals(spaceUsed, cfs.metric.liveDiskSpaceUsed.getCount());
+ assertEquals(spaceUsed, cfs.metric.totalDiskSpaceUsed.getCount());
+ assertTrue(cfs.getTracker().getCompacting().isEmpty());
++ if(cfs.getLiveSSTables().size() > 0)
++ assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore((int) (System.currentTimeMillis() / 1000)), false).isEmpty());
+ }
+
+ public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn)
+ {
+ String filename = cfs.getSSTablePath(directory);
+ return SSTableWriter.create(filename, 0, 0, new SerializationHeader(true, cfs.metadata, cfs.metadata.partitionColumns(), EncodingStats.NO_STATS), txn);
+ }
+
+ public static ByteBuffer random(int i, int size)
+ {
+ byte[] bytes = new byte[size + 4];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ ByteBuffer r = ByteBuffer.wrap(bytes);
+ r.putInt(0, i);
+ return r;
+ }
+
+ public static int assertFileCounts(String [] files)
+ {
+ int tmplinkcount = 0;
+ int tmpcount = 0;
+ int datacount = 0;
+ for (String f : files)
+ {
+ if (f.endsWith("-CRC.db"))
+ continue;
+ if (f.contains("tmplink-"))
+ tmplinkcount++;
+ else if (f.contains("tmp-"))
+ tmpcount++;
+ else if (f.contains("Data"))
+ datacount++;
+ }
+ assertEquals(0, tmplinkcount);
+ assertEquals(0, tmpcount);
+ return datacount;
+ }
+}