You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2014/08/01 17:19:38 UTC
[1/2] git commit: Fix truncate to always call flush on table
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1.0 1921b9859 -> 18ce8c72a
Fix truncate to always call flush on table
Patch by Jeremiah Jordan; reviewed by tjake for (CASSANDRA-7511)
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/60eab4e4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/60eab4e4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/60eab4e4
Branch: refs/heads/cassandra-2.1.0
Commit: 60eab4e45e18d6b08350187acf56deed9654fda7
Parents: 1879d99
Author: Jake Luciani <ja...@apache.org>
Authored: Fri Aug 1 10:30:48 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Fri Aug 1 10:30:48 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 5 +++
.../apache/cassandra/db/ColumnFamilyStore.java | 29 +++---------------
.../org/apache/cassandra/db/DataTracker.java | 18 -----------
test/conf/cassandra.yaml | 1 +
.../org/apache/cassandra/db/CommitLogTest.java | 32 ++++++++++++++++++++
6 files changed, 44 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1fcb556..33bab82 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.10
+ * Fix truncate to always flush (CASSANDRA-7511)
* Remove shuffle and taketoken (CASSANDRA-7601)
* Switch liveRatio-related log messages to DEBUG (CASSANDRA-7467)
* (cqlsh) Add tab-completion for CREATE/DROP USER IF [NOT] EXISTS (CASSANDRA-7611)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index bf0307b..d4c1f26 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1131,6 +1131,11 @@ public class DatabaseDescriptor
return conf.auto_snapshot;
}
+ @VisibleForTesting
+ public static void setAutoSnapshot(boolean autoSnapshot) {
+ conf.auto_snapshot = autoSnapshot;
+ }
+
public static boolean isAutoBootstrap()
{
return conf.auto_bootstrap;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 2824924..a3c080a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2002,31 +2002,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// position in the System keyspace.
logger.debug("truncating {}", name);
- if (DatabaseDescriptor.isAutoSnapshot())
- {
- // flush the CF being truncated before forcing the new segment
- forceBlockingFlush();
-
- // sleep a little to make sure that our truncatedAt comes after any sstable
- // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
- Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
- }
+ // flush the CF being truncated before forcing the new segment
+ forceBlockingFlush();
- // nuke the memtable data w/o writing to disk first
- Keyspace.switchLock.writeLock().lock();
- try
- {
- for (ColumnFamilyStore cfs : concatWithIndexes())
- {
- Memtable mt = cfs.getMemtableThreadSafe();
- if (!mt.isClean())
- mt.cfs.data.renewMemtable();
- }
- }
- finally
- {
- Keyspace.switchLock.writeLock().unlock();
- }
+ // sleep a little to make sure that our truncatedAt comes after any sstable
+ // that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
Runnable truncateRunnable = new Runnable()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index a9eef98..a0f880a 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -123,24 +123,6 @@ public class DataTracker
return toFlushMemtable;
}
- /**
- * Renew the current memtable without putting the old one for a flush.
- * Used when we flush but a memtable is clean (in which case we must
- * change it because it was frozen).
- */
- public void renewMemtable()
- {
- Memtable newMemtable = new Memtable(cfstore, view.get().memtable);
- View currentView, newView;
- do
- {
- currentView = view.get();
- newView = currentView.renewMemtable(newMemtable);
- }
- while (!view.compareAndSet(currentView, newView));
- notifyRenewed(currentView.memtable);
- }
-
public void replaceFlushed(Memtable memtable, SSTableReader sstable)
{
// sstable may be null if we flushed batchlog and nothing needed to be retained
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/test/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml
index d92eba6..3bb29bb 100644
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@ -6,6 +6,7 @@ cluster_name: Test Cluster
in_memory_compaction_limit_in_mb: 1
commitlog_sync: batch
commitlog_sync_batch_window_in_ms: 1.0
+commitlog_segment_size_in_mb: 1
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
listen_address: 127.0.0.1
storage_port: 7010
http://git-wip-us.apache.org/repos/asf/cassandra/blob/60eab4e4/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CommitLogTest.java b/test/unit/org/apache/cassandra/db/CommitLogTest.java
index 036ce15..a7df871 100644
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@ -22,6 +22,7 @@ package org.apache.cassandra.db;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.UUID;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
@@ -35,8 +36,10 @@ import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@ -257,4 +260,33 @@ public class CommitLogTest extends SchemaLoader
}
}
+ @Test
+ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException
+ {
+ CommitLog.instance.resetUnsafe();
+ boolean prev = DatabaseDescriptor.isAutoSnapshot();
+ DatabaseDescriptor.setAutoSnapshot(false);
+ ColumnFamilyStore cfs1 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs2 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard2");
+
+ final RowMutation rm1 = new RowMutation("Keyspace1", bytes("k"));
+ rm1.add("Standard1", bytes("c1"), ByteBuffer.allocate(100), 0);
+ rm1.apply();
+ cfs1.truncateBlocking();
+ DatabaseDescriptor.setAutoSnapshot(prev);
+ final RowMutation rm2 = new RowMutation("Keyspace1", bytes("k"));
+ rm2.add("Standard2", bytes("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0);
+
+ for (int i = 0 ; i < 5 ; i++)
+ CommitLog.instance.add(rm2);
+
+ Assert.assertEquals(2, CommitLog.instance.activeSegments());
+ ReplayPosition position = CommitLog.instance.getContext().get();
+ for (Keyspace ks : Keyspace.system())
+ for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
+ CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
+ CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
+ Assert.assertEquals(1, CommitLog.instance.activeSegments());
+ }
+
}
[2/2] git commit: Merge branch 'cassandra-2.0' into cassandra-2.1.0
Posted by ja...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1.0
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
src/java/org/apache/cassandra/db/DataTracker.java
test/unit/org/apache/cassandra/db/CommitLogTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/18ce8c72
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/18ce8c72
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/18ce8c72
Branch: refs/heads/cassandra-2.1.0
Commit: 18ce8c72a355949ffd8cdc8c083f3edf85c449d1
Parents: 1921b98 60eab4e
Author: Jake Luciani <ja...@apache.org>
Authored: Fri Aug 1 11:16:57 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Fri Aug 1 11:16:57 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/DatabaseDescriptor.java | 5 +++
.../apache/cassandra/db/ColumnFamilyStore.java | 23 +++-----------
test/conf/cassandra.yaml | 1 +
.../org/apache/cassandra/db/CommitLogTest.java | 32 ++++++++++++++++++++
5 files changed, 44 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18ce8c72/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index fcccccd,33bab82..7823f87
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,48 -1,11 +1,49 @@@
-2.0.10
- * Fix truncate to always flush (CASSANDRA-7511)
+2.1.0-final
+ * Tolerate min/max cell names of different lengths (CASSANDRA-7651)
+ * Filter cached results correctly (CASSANDRA-7636)
+ * Fix tracing on the new SEPExecutor (CASSANDRA-7644)
* Remove shuffle and taketoken (CASSANDRA-7601)
- * Switch liveRatio-related log messages to DEBUG (CASSANDRA-7467)
- * (cqlsh) Add tab-completion for CREATE/DROP USER IF [NOT] EXISTS (CASSANDRA-7611)
- * Always merge ranges owned by a single node (CASSANDRA-6930)
- * Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
+ * Clean up Windows batch scripts (CASSANDRA-7619)
+ * Fix native protocol drop user type notification (CASSANDRA-7571)
+ * Give read access to system.schema_usertypes to all authenticated users
+ (CASSANDRA-7578)
+ * (cqlsh) Fix cqlsh display when zero rows are returned (CASSANDRA-7580)
+ * Get java version correctly when JAVA_TOOL_OPTIONS is set (CASSANDRA-7572)
+ * Fix NPE when dropping index from non-existent keyspace, AssertionError when
+ dropping non-existent index with IF EXISTS (CASSANDRA-7590)
+ * Fix sstablelevelresetter hang (CASSANDRA-7614)
+ * (cqlsh) Fix deserialization of blobs (CASSANDRA-7603)
+ * Use "keyspace updated" schema change message for UDT changes in v1 and
+ v2 protocols (CASSANDRA-7617)
+ * Fix tracing of range slices and secondary index lookups that are local
+ to the coordinator (CASSANDRA-7599)
+ * Set -Dcassandra.storagedir for all tool shell scripts (CASSANDRA-7587)
+ * Don't swap max/min col names when mutating sstable metadata (CASSANDRA-7596)
+ * (cqlsh) Correctly handle paged result sets (CASSANDRA-7625)
+ * (cqlsh) Improve waiting for a trace to complete (CASSANDRA-7626)
+ * Fix tracing of concurrent range slices and 2ary index queries (CASSANDRA-7626)
+Merged from 2.0:
++ * Always flush on truncate (CASSANDRA-7511)
* Fix ReversedType(DateType) mapping to native protocol (CASSANDRA-7576)
+ * Always merge ranges owned by a single node (CASSANDRA-6930)
+ * Track max/min timestamps for range tombstones (CASSANDRA-7647)
+ * Fix NPE when listing saved caches dir (CASSANDRA-7632)
+
+
+2.1.0-rc4
+ * Fix word count hadoop example (CASSANDRA-7200)
+ * Updated memtable_cleanup_threshold and memtable_flush_writers defaults
+ (CASSANDRA-7551)
+ * (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
+ * Anti-compaction proceeds if any part of the repair failed (CASANDRA-7521)
+ * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539)
+ * Bump CQL version to 3.2.0 and update CQL documentation (CASSANDRA-7527)
+ * Fix configuration error message when running nodetool ring (CASSANDRA-7508)
+ * Support conditional updates, tuple type, and the v3 protocol in cqlsh (CASSANDRA-7509)
+ * Handle queries on multiple secondary index types (CASSANDRA-7525)
+ * Fix cqlsh authentication with v3 native protocol (CASSANDRA-7564)
+ * Fix NPE when unknown prepared statement ID is used (CASSANDRA-7454)
+Merged from 2.0:
* (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
* Fix range merging when DES scores are zero (CASSANDRA-7535)
* Warn when SSL certificates have expired (CASSANDRA-7528)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18ce8c72/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18ce8c72/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18ce8c72/test/conf/cassandra.yaml
----------------------------------------------------------------------
diff --cc test/conf/cassandra.yaml
index 43995e6,3bb29bb..ec988e2
--- a/test/conf/cassandra.yaml
+++ b/test/conf/cassandra.yaml
@@@ -3,9 -3,10 +3,10 @@@
# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file.
#
cluster_name: Test Cluster
-in_memory_compaction_limit_in_mb: 1
+memtable_allocation_type: offheap_objects
commitlog_sync: batch
commitlog_sync_batch_window_in_ms: 1.0
-commitlog_segment_size_in_mb: 1
++commitlog_segment_size_in_mb: 5
partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
listen_address: 127.0.0.1
storage_port: 7010
http://git-wip-us.apache.org/repos/asf/cassandra/blob/18ce8c72/test/unit/org/apache/cassandra/db/CommitLogTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/CommitLogTest.java
index 7046536,a7df871..a58549a
--- a/test/unit/org/apache/cassandra/db/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/CommitLogTest.java
@@@ -36,10 -36,10 +37,12 @@@ import org.apache.cassandra.config.Conf
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogDescriptor;
+ import org.apache.cassandra.db.commitlog.ReplayPosition;
+import org.apache.cassandra.db.commitlog.CommitLogSegment;
+import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@@@ -295,4 -260,33 +298,33 @@@ public class CommitLogTest extends Sche
}
}
+ @Test
+ public void testTruncateWithoutSnapshot() throws ExecutionException, InterruptedException
+ {
+ CommitLog.instance.resetUnsafe();
+ boolean prev = DatabaseDescriptor.isAutoSnapshot();
+ DatabaseDescriptor.setAutoSnapshot(false);
+ ColumnFamilyStore cfs1 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ ColumnFamilyStore cfs2 = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard2");
+
- final RowMutation rm1 = new RowMutation("Keyspace1", bytes("k"));
- rm1.add("Standard1", bytes("c1"), ByteBuffer.allocate(100), 0);
++ final Mutation rm1 = new Mutation("Keyspace1", bytes("k"));
++ rm1.add("Standard1", Util.cellname("c1"), ByteBuffer.allocate(100), 0);
+ rm1.apply();
+ cfs1.truncateBlocking();
+ DatabaseDescriptor.setAutoSnapshot(prev);
- final RowMutation rm2 = new RowMutation("Keyspace1", bytes("k"));
- rm2.add("Standard2", bytes("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0);
++ final Mutation rm2 = new Mutation("Keyspace1", bytes("k"));
++ rm2.add("Standard2", Util.cellname("c1"), ByteBuffer.allocate(DatabaseDescriptor.getCommitLogSegmentSize() / 4), 0);
+
+ for (int i = 0 ; i < 5 ; i++)
+ CommitLog.instance.add(rm2);
+
+ Assert.assertEquals(2, CommitLog.instance.activeSegments());
- ReplayPosition position = CommitLog.instance.getContext().get();
++ ReplayPosition position = CommitLog.instance.getContext();
+ for (Keyspace ks : Keyspace.system())
+ for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
+ CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
+ CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
+ Assert.assertEquals(1, CommitLog.instance.activeSegments());
+ }
+
}