You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2014/11/24 14:30:34 UTC
[1/3] cassandra git commit: Ignore Paxos commits for truncated tables
Repository: cassandra
Updated Branches:
refs/heads/trunk 41435ef6c -> 584113103
Ignore Paxos commits for truncated tables
patch by Sam Tunnicliffe; reviewed by Aleksey Yeschenko for
CASSANDRA-7538
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17de36f2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17de36f2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17de36f2
Branch: refs/heads/trunk
Commit: 17de36f246c912287b85eb7015583a35f5040919
Parents: 0e3d9fc
Author: Sam Tunnicliffe <sa...@beobal.com>
Authored: Mon Nov 24 16:07:17 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Nov 24 16:07:17 2014 +0300
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/paxos/PaxosState.java | 17 ++-
.../cassandra/service/PaxosStateTest.java | 108 +++++++++++++++++++
3 files changed, 122 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17de36f2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 412eb59..fe23248 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.12:
+ * Ignore Paxos commits for truncated tables (CASSANDRA-7538)
* Validate size of indexed column values (CASSANDRA-8280)
* Make LCS split compaction results over all data directories (CASSANDRA-8329)
* Fix some failing queries that use multi-column relations
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17de36f2/src/java/org/apache/cassandra/service/paxos/PaxosState.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosState.java b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
index 0196122..2adecec 100644
--- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.db.RowMutation;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.UUIDGen;
public class PaxosState
{
@@ -132,10 +133,18 @@ public class PaxosState
// Committing it is however always safe due to column timestamps, so always do it. However,
// if our current in-progress ballot is strictly greater than the proposal one, we shouldn't
// erase the in-progress update.
- Tracing.trace("Committing proposal {}", proposal);
- RowMutation rm = proposal.makeMutation();
- Keyspace.open(rm.getKeyspaceName()).apply(rm, true);
-
+ // The table may have been truncated since the proposal was initiated. In that case, we
+ // don't want to perform the mutation and potentially resurrect truncated data
+ if (UUIDGen.unixTimestamp(proposal.ballot) >= SystemKeyspace.getTruncatedAt(proposal.update.metadata().cfId))
+ {
+ Tracing.trace("Committing proposal {}", proposal);
+ RowMutation rm = proposal.makeMutation();
+ Keyspace.open(rm.getKeyspaceName()).apply(rm, true);
+ }
+ else
+ {
+ Tracing.trace("Not committing proposal {} as ballot timestamp predates last truncation time", proposal);
+ }
// We don't need to lock, we're just blindly updating
SystemKeyspace.savePaxosCommit(proposal);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/17de36f2/test/unit/org/apache/cassandra/service/PaxosStateTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
new file mode 100644
index 0000000..306c424
--- /dev/null
+++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.service;
+
+import java.nio.ByteBuffer;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ArrayBackedSortedColumns;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.service.paxos.Commit;
+import org.apache.cassandra.service.paxos.PaxosState;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+
+public class PaxosStateTest
+{
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ }
+
+ @AfterClass
+ public static void stopGossiper()
+ {
+ Gossiper.instance.stop();
+ }
+
+ @Test
+ public void testCommittingAfterTruncation() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ DecoratedKey key = Util.dk("key" + System.nanoTime());
+ ByteBuffer name = ByteBufferUtil.bytes("col");
+ ByteBuffer value = ByteBufferUtil.bytes(0);
+ ColumnFamily update = ArrayBackedSortedColumns.factory.create(cfs.metadata);
+ update.addColumn(name, value, FBUtilities.timestampMicros());
+
+ // CFS should be empty initially
+ assertNoDataPresent(cfs, key);
+
+ // Commit the proposal & verify the data is present
+ Commit beforeTruncate = newProposal(0, key.key, update);
+ PaxosState.commit(beforeTruncate);
+ assertDataPresent(cfs, key, name, value);
+
+ // Truncate then attempt to commit again, mutation should
+ // be ignored as the proposal predates the truncation
+ cfs.truncateBlocking();
+ PaxosState.commit(beforeTruncate);
+ assertNoDataPresent(cfs, key);
+
+ // Now try again with a ballot created after the truncation
+ long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().cfId) + 1;
+ Commit afterTruncate = newProposal(timestamp, key.key, update);
+ PaxosState.commit(afterTruncate);
+ assertDataPresent(cfs, key, name, value);
+ }
+
+ private Commit newProposal(long ballotMillis, ByteBuffer key, ColumnFamily update)
+ {
+ return Commit.newProposal(key, UUIDGen.getTimeUUID(ballotMillis), update);
+ }
+
+ private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, ByteBuffer name, ByteBuffer value)
+ {
+ ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis()));
+ assertFalse(cf.isEmpty());
+ assertEquals(0, ByteBufferUtil.compareUnsigned(value, cf.getColumn(name).value()));
+ }
+
+ private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key)
+ {
+ ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis()));
+ assertNull(cf);
+ }
+}
[2/3] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by al...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/service/paxos/PaxosState.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eac7781e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eac7781e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eac7781e
Branch: refs/heads/trunk
Commit: eac7781e7c429ac25b56ddc2ec20bc34f3244df6
Parents: cab2b25 17de36f
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Nov 24 16:27:56 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Nov 24 16:27:56 2014 +0300
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/paxos/PaxosState.java | 17 ++-
.../cassandra/service/PaxosStateTest.java | 104 +++++++++++++++++++
3 files changed, 118 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eac7781e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9db65e9,fe23248..c9e35d5
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,18 -1,5 +1,19 @@@
-2.0.12:
+2.1.3
+ * Fix high size calculations for prepared statements (CASSANDRA-8231)
+ * Centralize shared executors (CASSANDRA-8055)
+ * Fix filtering for CONTAINS (KEY) relations on frozen collection
+ clustering columns when the query is restricted to a single
+ partition (CASSANDRA-8203)
+ * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
+ * Add more log info if readMeter is null (CASSANDRA-8238)
+ * add check of the system wall clock time at startup (CASSANDRA-8305)
+ * Support for frozen collections (CASSANDRA-7859)
+ * Fix overflow on histogram computation (CASSANDRA-8028)
+ * Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801)
+ * Fix incremental repair not remove parent session on remote (CASSANDRA-8291)
+ * Improve JBOD disk utilization (CASSANDRA-7386)
+Merged from 2.0:
+ * Ignore Paxos commits for truncated tables (CASSANDRA-7538)
* Validate size of indexed column values (CASSANDRA-8280)
* Make LCS split compaction results over all data directories (CASSANDRA-8329)
* Fix some failing queries that use multi-column relations
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eac7781e/src/java/org/apache/cassandra/service/paxos/PaxosState.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/service/paxos/PaxosState.java
index abd173c,2adecec..01e03f4
--- a/src/java/org/apache/cassandra/service/paxos/PaxosState.java
+++ b/src/java/org/apache/cassandra/service/paxos/PaxosState.java
@@@ -18,17 -19,19 +18,18 @@@
* under the License.
*
*/
-
+package org.apache.cassandra.service.paxos;
import java.nio.ByteBuffer;
+import java.util.concurrent.locks.Lock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import com.google.common.util.concurrent.Striped;
import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.RowMutation;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.*;
import org.apache.cassandra.tracing.Tracing;
+ import org.apache.cassandra.utils.UUIDGen;
public class PaxosState
{
@@@ -131,10 -133,18 +132,18 @@@
// Committing it is however always safe due to column timestamps, so always do it. However,
// if our current in-progress ballot is strictly greater than the proposal one, we shouldn't
// erase the in-progress update.
- Tracing.trace("Committing proposal {}", proposal);
- Mutation mutation = proposal.makeMutation();
- Keyspace.open(mutation.getKeyspaceName()).apply(mutation, true);
-
+ // The table may have been truncated since the proposal was initiated. In that case, we
+ // don't want to perform the mutation and potentially resurrect truncated data
+ if (UUIDGen.unixTimestamp(proposal.ballot) >= SystemKeyspace.getTruncatedAt(proposal.update.metadata().cfId))
+ {
+ Tracing.trace("Committing proposal {}", proposal);
- RowMutation rm = proposal.makeMutation();
- Keyspace.open(rm.getKeyspaceName()).apply(rm, true);
++ Mutation mutation = proposal.makeMutation();
++ Keyspace.open(mutation.getKeyspaceName()).apply(mutation, true);
+ }
+ else
+ {
+ Tracing.trace("Not committing proposal {} as ballot timestamp predates last truncation time", proposal);
+ }
// We don't need to lock, we're just blindly updating
SystemKeyspace.savePaxosCommit(proposal);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eac7781e/test/unit/org/apache/cassandra/service/PaxosStateTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/service/PaxosStateTest.java
index 0000000,306c424..d41d89f
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/service/PaxosStateTest.java
+++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java
@@@ -1,0 -1,108 +1,104 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.cassandra.service;
+
+ import java.nio.ByteBuffer;
+
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+
+ import org.apache.cassandra.SchemaLoader;
+ import org.apache.cassandra.Util;
-import org.apache.cassandra.db.ArrayBackedSortedColumns;
-import org.apache.cassandra.db.ColumnFamily;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.SystemKeyspace;
++import org.apache.cassandra.db.*;
++import org.apache.cassandra.db.composites.CellName;
+ import org.apache.cassandra.db.filter.QueryFilter;
+ import org.apache.cassandra.gms.Gossiper;
+ import org.apache.cassandra.service.paxos.Commit;
+ import org.apache.cassandra.service.paxos.PaxosState;
+ import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FBUtilities;
+ import org.apache.cassandra.utils.UUIDGen;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertNull;
+
+ public class PaxosStateTest
+ {
+ @BeforeClass
+ public static void setUpClass() throws Throwable
+ {
+ SchemaLoader.loadSchema();
+ }
+
+ @AfterClass
+ public static void stopGossiper()
+ {
+ Gossiper.instance.stop();
+ }
+
+ @Test
+ public void testCommittingAfterTruncation() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ DecoratedKey key = Util.dk("key" + System.nanoTime());
- ByteBuffer name = ByteBufferUtil.bytes("col");
++ CellName name = Util.cellname("col");
+ ByteBuffer value = ByteBufferUtil.bytes(0);
+ ColumnFamily update = ArrayBackedSortedColumns.factory.create(cfs.metadata);
+ update.addColumn(name, value, FBUtilities.timestampMicros());
+
+ // CFS should be empty initially
+ assertNoDataPresent(cfs, key);
+
+ // Commit the proposal & verify the data is present
- Commit beforeTruncate = newProposal(0, key.key, update);
++ Commit beforeTruncate = newProposal(0, key.getKey(), update);
+ PaxosState.commit(beforeTruncate);
+ assertDataPresent(cfs, key, name, value);
+
+ // Truncate then attempt to commit again, mutation should
+ // be ignored as the proposal predates the truncation
+ cfs.truncateBlocking();
+ PaxosState.commit(beforeTruncate);
+ assertNoDataPresent(cfs, key);
+
+ // Now try again with a ballot created after the truncation
+ long timestamp = SystemKeyspace.getTruncatedAt(update.metadata().cfId) + 1;
- Commit afterTruncate = newProposal(timestamp, key.key, update);
++ Commit afterTruncate = newProposal(timestamp, key.getKey(), update);
+ PaxosState.commit(afterTruncate);
+ assertDataPresent(cfs, key, name, value);
+ }
+
+ private Commit newProposal(long ballotMillis, ByteBuffer key, ColumnFamily update)
+ {
+ return Commit.newProposal(key, UUIDGen.getTimeUUID(ballotMillis), update);
+ }
+
- private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, ByteBuffer name, ByteBuffer value)
++ private void assertDataPresent(ColumnFamilyStore cfs, DecoratedKey key, CellName name, ByteBuffer value)
+ {
+ ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis()));
+ assertFalse(cf.isEmpty());
+ assertEquals(0, ByteBufferUtil.compareUnsigned(value, cf.getColumn(name).value()));
+ }
+
+ private void assertNoDataPresent(ColumnFamilyStore cfs, DecoratedKey key)
+ {
+ ColumnFamily cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(key, cfs.name, System.currentTimeMillis()));
+ assertNull(cf);
+ }
+ }
[3/3] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by al...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/58411310
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/58411310
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/58411310
Branch: refs/heads/trunk
Commit: 5841131037155ab76fcfb0e84cacb3b00400830b
Parents: 41435ef eac7781
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Nov 24 16:28:23 2014 +0300
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Nov 24 16:28:51 2014 +0300
----------------------------------------------------------------------
CHANGES.txt | 4 +-
.../cassandra/service/paxos/PaxosState.java | 17 ++-
.../cassandra/service/PaxosStateTest.java | 104 +++++++++++++++++++
3 files changed, 120 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/58411310/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 1beb2e2,c9e35d5..af73426
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,38 -1,5 +1,39 @@@
+3.0
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer apis (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support pure user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063, 7813)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
- * improve concurrency of repair (CASSANDRA-6455, 8208)
++ * Improve concurrency of repair (CASSANDRA-6455, 8208)
++
+
2.1.3
- * Fix high size calculations for prepared statements (CASSANDRA-8231)
* Centralize shared executors (CASSANDRA-8055)
* Fix filtering for CONTAINS (KEY) relations on frozen collection
clustering columns when the query is restricted to a single