You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2012/03/07 18:56:12 UTC
[2/6] git commit: Merge branch 'cassandra-1.0' into cassandra-1.1.0
Merge branch 'cassandra-1.0' into cassandra-1.1.0
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
src/java/org/apache/cassandra/db/DataTracker.java
src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d78db44c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d78db44c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d78db44c
Branch: refs/heads/cassandra-1.1
Commit: d78db44cb6b77acaee3fe05711ad6e8a14bb4558
Parents: 5f31757 f239907
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Wed Mar 7 18:53:07 2012 +0100
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Wed Mar 7 18:53:07 2012 +0100
----------------------------------------------------------------------
CHANGES.txt | 7 +++
src/java/org/apache/cassandra/db/ColumnFamily.java | 20 +++++++
.../org/apache/cassandra/db/ColumnFamilyStore.java | 16 ++++--
src/java/org/apache/cassandra/db/DataTracker.java | 15 +++---
.../cassandra/db/compaction/CompactionManager.java | 8 ++-
.../cassandra/db/compaction/CompactionTask.java | 16 ++++--
.../db/compaction/LeveledCompactionStrategy.java | 13 ++++-
.../cassandra/db/compaction/LeveledManifest.java | 41 ++++++++++-----
.../cassandra/db/compaction/OperationType.java | 1 +
.../org/apache/cassandra/io/util/FileUtils.java | 17 ------
.../SSTableListChangedNotification.java | 11 +++-
.../cassandra/db/compaction/CompactionsTest.java | 3 +-
12 files changed, 112 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a9e1c10,925a4a9..d7a595b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,31 -1,4 +1,35 @@@
++<<<<<<< HEAD
+1.1-dev
+ * start hint replay as soon as FD notifies that the target is back up
+ (CASSANDRA-3958)
+ * avoid unproductive deserializing of cached rows during compaction
+ (CASSANDRA-3921)
+ * fix concurrency issues with CQL keyspace creation (CASSANDRA-3903)
+ * Show Effective Owership via Nodetool ring <keyspace> (CASSANDRA-3412)
+ * Update ORDER BY syntax for CQL3 (CASSANDRA-3925)
+ * Fix BulkRecordWriter to not throw NPE if reducer gets no map data from Hadoop (CASSANDRA-3944)
+ * Fix bug with counters in super columns (CASSANDRA-3821)
+ * Remove deprecated merge_shard_chance (CASSANDRA-3940)
+ * add a convenient way to reset a node's schema (CASSANDRA-2963)
+ * fix for intermittent SchemaDisagreementException (CASSANDRA-3884)
+ * ignore deprecated KsDef/CfDef/ColumnDef fields in native schema (CASSANDRA-3963)
+ * CLI to report when unsupported column_metadata pair was given (CASSANDRA-3959)
+ * reincarnate removed and deprecated KsDef/CfDef attributes (CASSANDRA-3953)
+ * Fix race between writes and read for cache (CASSANDRA-3862)
+ * perform static initialization of StorageProxy on start-up (CASSANDRA-3797)
+ * support trickling fsync() on writes (CASSANDRA-3950)
+ * expose counters for unavailable/timeout exceptions given to thrift clients (CASSANDRA-3671)
+ * avoid quadratic startup time in LeveledManifest (CASSANDRA-3952)
+Merged from 1.0:
+ * remove the wait on hint future during write (CASSANDRA-3870)
+ * (cqlsh) ignore missing CfDef opts (CASSANDRA-3933)
+ * (cqlsh) look for cqlshlib relative to realpath (CASSANDRA-3767)
+ * Fix short read protection (CASSANDRA-3934)
+ * don't change manifest level for cleanup, scrub, and upgradesstables
+ operations under LeveledCompactionStrategy (CASSANDRA-3989)
++=======
+ 1.0.9
++>>>>>>> cassandra-1.0
* always compact away deleted hints immediately after handoff (CASSANDRA-3955)
* delete hints from dropped ColumnFamilies on handoff instead of
erroring out (CASSANDRA-3975)
@@@ -34,89 -7,11 +38,92 @@@
* Pig: Composite column support (CASSANDRA-384)
* Avoid NPE during repair when a keyspace has no CFs (CASSANDRA-3988)
* Fix division-by-zero error on get_slice (CASSANDRA-4000)
+ * don't change manifest level for cleanup, scrub, and upgradesstables
+ operations under LeveledCompactionStrategy (CASSANDRA-3989)
+ * fix race leading to super columns assertion failure (CASSANDRA-3957)
+1.1-beta1
+ * add nodetool rebuild_index (CASSANDRA-3583)
+ * add nodetool rangekeysample (CASSANDRA-2917)
+ * Fix streaming too much data during move operations (CASSANDRA-3639)
+ * Nodetool and CLI connect to localhost by default (CASSANDRA-3568)
+ * Reduce memory used by primary index sample (CASSANDRA-3743)
+ * (Hadoop) separate input/output configurations (CASSANDRA-3197, 3765)
+ * avoid returning internal Cassandra classes over JMX (CASSANDRA-2805)
+ * add row-level isolation via SnapTree (CASSANDRA-2893)
+ * Optimize key count estimation when opening sstable on startup
+ (CASSANDRA-2988)
+ * multi-dc replication optimization supporting CL > ONE (CASSANDRA-3577)
+ * add command to stop compactions (CASSANDRA-1740, 3566, 3582)
+ * multithreaded streaming (CASSANDRA-3494)
+ * removed in-tree redhat spec (CASSANDRA-3567)
+ * "defragment" rows for name-based queries under STCS, again (CASSANDRA-2503)
+ * Recycle commitlog segments for improved performance
+ (CASSANDRA-3411, 3543, 3557, 3615)
+ * update size-tiered compaction to prioritize small tiers (CASSANDRA-2407)
+ * add message expiration logic to OutboundTcpConnection (CASSANDRA-3005)
+ * off-heap cache to use sun.misc.Unsafe instead of JNA (CASSANDRA-3271)
+ * EACH_QUORUM is only supported for writes (CASSANDRA-3272)
+ * replace compactionlock use in schema migration by checking CFS.isValid
+ (CASSANDRA-3116)
+ * recognize that "SELECT first ... *" isn't really "SELECT *" (CASSANDRA-3445)
+ * Use faster bytes comparison (CASSANDRA-3434)
+ * Bulk loader is no longer a fat client, (HADOOP) bulk load output format
+ (CASSANDRA-3045)
+ * (Hadoop) add support for KeyRange.filter
+ * remove assumption that keys and token are in bijection
+ (CASSANDRA-1034, 3574, 3604)
+ * always remove endpoints from delevery queue in HH (CASSANDRA-3546)
+ * fix race between cf flush and its 2ndary indexes flush (CASSANDRA-3547)
+ * fix potential race in AES when a repair fails (CASSANDRA-3548)
+ * Remove columns shadowed by a deleted container even when we cannot purge
+ (CASSANDRA-3538)
+ * Improve memtable slice iteration performance (CASSANDRA-3545)
+ * more efficient allocation of small bloom filters (CASSANDRA-3618)
+ * Use separate writer thread in SSTableSimpleUnsortedWriter (CASSANDRA-3619)
+ * fsync the directory after new sstable or commitlog segment are created (CASSANDRA-3250)
+ * fix minor issues reported by FindBugs (CASSANDRA-3658)
+ * global key/row caches (CASSANDRA-3143, 3849)
+ * optimize memtable iteration during range scan (CASSANDRA-3638)
+ * introduce 'crc_check_chance' in CompressionParameters to support
+ a checksum percentage checking chance similarly to read-repair (CASSANDRA-3611)
+ * a way to deactivate global key/row cache on per-CF basis (CASSANDRA-3667)
+ * fix LeveledCompactionStrategy broken because of generation pre-allocation
+ in LeveledManifest (CASSANDRA-3691)
+ * finer-grained control over data directories (CASSANDRA-2749)
+ * Fix ClassCastException during hinted handoff (CASSANDRA-3694)
+ * Upgrade Thrift to 0.7 (CASSANDRA-3213)
+ * Make stress.java insert operation to use microseconds (CASSANDRA-3725)
+ * Allows (internally) doing a range query with a limit of columns instead of
+ rows (CASSANDRA-3742)
+ * Allow rangeSlice queries to be start/end inclusive/exclusive (CASSANDRA-3749)
+ * Fix BulkLoader to support new SSTable layout and add stream
+ throttling to prevent an NPE when there is no yaml config (CASSANDRA-3752)
+ * Allow concurrent schema migrations (CASSANDRA-1391, 3832)
+ * Add SnapshotCommand to trigger snapshot on remote node (CASSANDRA-3721)
+ * Make CFMetaData conversions to/from thrift/native schema inverses
+ (CASSANDRA_3559)
+ * Add initial code for CQL 3.0-beta (CASSANDRA-3781, 3753)
+ * Add wide row support for ColumnFamilyInputFormat (CASSANDRA-3264)
+ * Allow extending CompositeType comparator (CASSANDRA-3657)
+ * Avoids over-paging during get_count (CASSANDRA-3798)
+ * Add new command to rebuild a node without (repair) merkle tree calculations
+ (CASSANDRA-3483, 3922)
+ * respect not only row cache capacity but caching mode when
+ trying to read data (CASSANDRA-3812)
+ * fix system tests (CASSANDRA-3827)
+ * CQL support for altering key_validation_class in ALTER TABLE (CASSANDRA-3781)
+ * turn compression on by default (CASSANDRA-3871)
+ * make hexToBytes refuse invalid input (CASSANDRA-2851)
+ * Make secondary indexes CF inherit compression and compaction from their
+ parent CF (CASSANDRA-3877)
+ * Finish cleanup up tombstone purge code (CASSANDRA-3872)
+ * Avoid NPE on aboarted stream-out sessions (CASSANDRA-3904)
+ * BulkRecordWriter throws NPE for counter columns (CASSANDRA-3906)
+ * Support compression using BulkWriter (CASSANDRA-3907)
+
+
1.0.8
* fix race between cleanup and flush on secondary index CFSes (CASSANDRA-3712)
* avoid including non-queried nodes in rangeslice read repair
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/ColumnFamily.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index a8d1fc8,60a3487..a69989e
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -51,7 -46,7 +51,8 @@@ import org.apache.cassandra.db.commitlo
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
+import org.apache.cassandra.db.filter.ExtendedFilter;
+ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.filter.IFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
@@@ -713,26 -779,18 +714,29 @@@ public class ColumnFamilyStore implemen
public void updateRowCache(DecoratedKey key, ColumnFamily columnFamily)
{
- if (rowCache.isPutCopying())
+ if (metadata.cfId == null)
+ return; // secondary index
+
+ RowCacheKey cacheKey = new RowCacheKey(metadata.cfId, key);
+
+ // always invalidate a copying cache value
+ if (CacheService.instance.rowCache.isPutCopying())
{
- invalidateCachedRow(key);
+ invalidateCachedRow(cacheKey);
+ return;
}
- else
+
+ // invalidate a normal cache value if it's a sentinel, so the read will retry (and include the new update)
+ IRowCacheEntry cachedRow = getCachedRowInternal(cacheKey);
+ if (cachedRow != null)
{
- ColumnFamily cachedRow = getRawCachedRow(key);
- if (cachedRow != null)
+ if (cachedRow instanceof RowCacheSentinel)
+ invalidateCachedRow(cacheKey);
+ else
- ((ColumnFamily) cachedRow).addAll(columnFamily, HeapAllocator.instance);
+ // columnFamily is what is written in the commit log. Because of the PeriodicCommitLog, this can be done in concurrency
+ // with this. So columnFamily shouldn't be modified and if it contains super columns, neither should they. So for super
+ // columns, we must make sure to clone them when adding to the cache. That's what addAllWithSCCopy does (see #3957)
- cachedRow.addAllWithSCCopy(columnFamily, HeapAllocator.instance);
++ ((ColumnFamily) cachedRow).addAllWithSCCopy(columnFamily, HeapAllocator.instance);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/DataTracker.java
index 3b1db56,c4b8a7b..d6b5fe5
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@@ -30,7 -30,11 +30,8 @@@ import com.google.common.collect.*
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.cache.AutoSavingCache;
import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
@@@ -290,8 -259,9 +291,8 @@@ public class DataTracke
// notifySSTablesChanged -> LeveledManifest.promote doesn't like a no-op "promotion"
return;
}
- notifySSTablesChanged(notCompacting, Collections.<SSTableReader>emptySet());
-
- replace(sstables, Collections.<SSTableReader>emptyList());
- notifySSTablesChanged(sstables, Collections.<SSTableReader>emptyList(), OperationType.UNKNOWN);
++ notifySSTablesChanged(notCompacting, Collections.<SSTableReader>emptySet(), OperationType.UNKNOWN);
+ postReplace(notCompacting, Collections.<SSTableReader>emptySet());
}
/** (Re)initializes the tracker, purging all references. */
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index 94bc788,55fab3c..ff50db5
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -775,10 -796,10 +777,10 @@@ public class CompactionManager implemen
logger.info(String.format(format, writer.getFilename(), startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
}
- // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
+ // flush to ensure we don't lose the tombstones on a restart, since they are not commitlog'd
cfs.indexManager.flushIndexesBlocking();
- cfs.replaceCompactedSSTables(Arrays.asList(sstable), results);
+ cfs.replaceCompactedSSTables(Arrays.asList(sstable), results, OperationType.CLEANUP);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 683241f,5847bf7..9c19070
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -42,15 -41,19 +42,17 @@@ import org.apache.cassandra.utils.FBUti
public class CompactionTask extends AbstractCompactionTask
{
protected static final Logger logger = LoggerFactory.getLogger(CompactionTask.class);
- protected String compactionFileLocation;
protected final int gcBefore;
protected boolean isUserDefined;
+ protected OperationType compactionType;
protected static long totalBytesCompacted = 0;
public CompactionTask(ColumnFamilyStore cfs, Collection<SSTableReader> sstables, final int gcBefore)
{
super(cfs, sstables);
- compactionFileLocation = null;
this.gcBefore = gcBefore;
this.isUserDefined = false;
+ this.compactionType = OperationType.COMPACTION;
}
public static synchronized long addToTotalBytesCompacted(long bytesCompacted)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index d2edebe,6dc6ce9..32c0a9b
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@@ -166,11 -164,9 +166,10 @@@ public class LeveledManifes
int maximumLevel = 0;
for (SSTableReader sstable : removed)
{
- int thisLevel = levelOf(sstable);
+ int thisLevel = remove(sstable);
+ assert thisLevel >= 0;
maximumLevel = Math.max(maximumLevel, thisLevel);
minimumLevel = Math.min(minimumLevel, thisLevel);
- remove(sstable);
}
// it's valid to do a remove w/o an add (e.g. on truncate)
@@@ -286,19 -295,20 +298,20 @@@
private int levelOf(SSTableReader sstable)
{
- for (int level = 0; level < generations.length; level++)
- {
- if (generations[level].contains(sstable))
- return level;
- }
- return -1;
+ Integer level = sstableGenerations.get(sstable);
+ if (level == null)
+ return -1;
+
+ return level.intValue();
}
- private void remove(SSTableReader reader)
+ private int remove(SSTableReader reader)
{
int level = levelOf(reader);
assert level >= 0 : reader + " not present in manifest";
generations[level].remove(reader);
+ sstableGenerations.remove(reader);
+ return level;
}
private void add(SSTableReader sstable, int level)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/db/compaction/OperationType.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
index 26a52b5,9f31b4e..92456ca
--- a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
@@@ -23,11 -23,17 +23,15 @@@ package org.apache.cassandra.notificati
import org.apache.cassandra.io.sstable.SSTableReader;
-import java.util.List;
-
+ import org.apache.cassandra.db.compaction.OperationType;
+
public class SSTableListChangedNotification implements INotification
{
- public Iterable<SSTableReader> removed;
- public Iterable<SSTableReader> added;
- public SSTableListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed)
+ public final Iterable<SSTableReader> removed;
+ public final Iterable<SSTableReader> added;
+ public final OperationType compactionType;
+
+ public SSTableListChangedNotification(Iterable<SSTableReader> added, Iterable<SSTableReader> removed, OperationType compactionType)
{
this.removed = removed;
this.added = added;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d78db44c/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------