You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2012/07/27 00:17:34 UTC
[2/13] git commit: merge from 1.0
merge from 1.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/070c1d8d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/070c1d8d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/070c1d8d
Branch: refs/heads/trunk
Commit: 070c1d8dd3c7b674034acfbc421555d658aabb60
Parents: 10372c2 1ad710b
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Jul 26 17:16:39 2012 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jul 26 17:16:39 2012 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../db/index/PerColumnSecondaryIndex.java | 8 +
.../cassandra/db/index/PerRowSecondaryIndex.java | 7 +
.../apache/cassandra/db/index/SecondaryIndex.java | 3 +
.../cassandra/db/index/SecondaryIndexManager.java | 19 +-
.../apache/cassandra/thrift/ThriftValidation.java | 14 +-
.../cassandra/db/SecondaryIndexColumnSizeTest.java | 215 +++++++++++++++
.../cassandra/thrift/ThriftValidationTest.java | 52 ----
8 files changed, 258 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 84db73d,79278aa..bcf56b7
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,154 -1,6 +1,155 @@@
-1.0.11
+1.1.3
+ * (JMX) rename getRangeKeySample to sampleKeyRange to avoid returning
+ multi-MB results as an attribute (CASSANDRA-4452)
+ * flush based on data size, not throughput; overwritten columns no
+ longer artificially inflate liveRatio (CASSANDRA-4399)
+ * update default commitlog segment size to 32MB and total commitlog
+ size to 32/1024 MB for 32/64 bit JVMs, respectively (CASSANDRA-4422)
+ * avoid using global partitioner to estimate ranges in index sstables
+ (CASSANDRA-4403)
+ * restore pre-CASSANDRA-3862 approach to removing expired tombstones
+ from row cache during compaction (CASSANDRA-4364)
+ * (stress) support for CQL prepared statements (CASSANDRA-3633)
+ * Correctly catch exception when Snappy cannot be loaded (CASSANDRA-4400)
+ * (cql3) Support ORDER BY when IN condition is given in WHERE clause (CASSANDRA-4327)
+ * (cql3) delete "component_index" column on DROP TABLE call (CASSANDRA-4420)
+ * change nanoTime() to currentTimeInMillis() in schema related code (CASSANDRA-4432)
+ * add a token generation tool (CASSANDRA-3709)
+ * Fix LCS bug with sstable containing only 1 row (CASSANDRA-4411)
+ * fix "Can't Modify Index Name" problem on CF update (CASSANDRA-4439)
+ * Fix assertion error in getOverlappingSSTables during repair (CASSANDRA-4456)
+ * fix nodetool's setcompactionthreshold command (CASSANDRA-4455)
+ * Ensure compacted files are never used, to avoid counter overcount (CASSANDRA-4436)
+Merged from 1.0:
++ * Push the validation of secondary index values to the SecondaryIndexManager (CASSANDRA-4240)
* allow dropping columns shadowed by not-yet-expired supercolumn or row
tombstones in PrecompactedRow (CASSANDRA-4396)
+ * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195)
+ * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419)
+ * Bootstraps that fail are detected upon restart and will retry safely without
+ needing to delete existing data first (CASSANDRA-4427)
+ * (cqlsh) add a COPY TO command to copy a CF to a CSV file (CASSANDRA-4434)
+
+
+1.1.2
+ * Fix cleanup not deleting index entries (CASSANDRA-4379)
+ * Use correct partitioner when saving + loading caches (CASSANDRA-4331)
+ * Check schema before trying to export sstable (CASSANDRA-2760)
+ * Raise a meaningful exception instead of NPE when PFS encounters
+ an unconfigured node + no default (CASSANDRA-4349)
+ * fix bug in sstable blacklisting with LCS (CASSANDRA-4343)
+ * LCS no longer promotes tiny sstables out of L0 (CASSANDRA-4341)
+ * skip tombstones during hint replay (CASSANDRA-4320)
+ * fix NPE in compactionstats (CASSANDRA-4318)
+ * enforce 1m min keycache for auto (CASSANDRA-4306)
+ * Have DeletedColumn.isMFD always return true (CASSANDRA-4307)
+ * (cql3) exeption message for ORDER BY constraints said primary filter can be
+ an IN clause, which is misleading (CASSANDRA-4319)
+ * (cql3) Reject (not yet supported) creation of 2ndardy indexes on tables with
+ composite primary keys (CASSANDRA-4328)
+ * Set JVM stack size to 160k for java 7 (CASSANDRA-4275)
+ * cqlsh: add COPY command to load data from CSV flat files (CASSANDRA-4012)
+ * CFMetaData.fromThrift to throw ConfigurationException upon error (CASSANDRA-4353)
+ * Use CF comparator to sort indexed columns in SecondaryIndexManager
+ (CASSANDRA-4365)
+ * add strategy_options to the KSMetaData.toString() output (CASSANDRA-4248)
+ * (cql3) fix range queries containing unqueried results (CASSANDRA-4372)
+ * (cql3) allow updating column_alias types (CASSANDRA-4041)
+ * (cql3) Fix deletion bug (CASSANDRA-4193)
+ * Fix computation of overlapping sstable for leveled compaction (CASSANDRA-4321)
+ * Improve scrub and allow to run it offline (CASSANDRA-4321)
+ * Fix assertionError in StorageService.bulkLoad (CASSANDRA-4368)
+ * (cqlsh) add option to authenticate to a keyspace at startup (CASSANDRA-4108)
+ * (cqlsh) fix ASSUME functionality (CASSANDRA-4352)
+ * Fix ColumnFamilyRecordReader to not return progress > 100% (CASSANDRA-3942)
+Merged from 1.0:
+ * Set gc_grace on index CF to 0 (CASSANDRA-4314)
+
+
+1.1.1
+ * allow larger cache capacities than 2GB (CASSANDRA-4150)
+ * add getsstables command to nodetool (CASSANDRA-4199)
+ * apply parent CF compaction settings to secondary index CFs (CASSANDRA-4280)
+ * preserve commitlog size cap when recycling segments at startup
+ (CASSANDRA-4201)
+ * (Hadoop) fix split generation regression (CASSANDRA-4259)
+ * ignore min/max compactions settings in LCS, while preserving
+ behavior that min=max=0 disables autocompaction (CASSANDRA-4233)
+ * log number of rows read from saved cache (CASSANDRA-4249)
+ * calculate exact size required for cleanup operations (CASSANDRA-1404)
+ * avoid blocking additional writes during flush when the commitlog
+ gets behind temporarily (CASSANDRA-1991)
+ * enable caching on index CFs based on data CF cache setting (CASSANDRA-4197)
+ * warn on invalid replication strategy creation options (CASSANDRA-4046)
+ * remove [Freeable]Memory finalizers (CASSANDRA-4222)
+ * include tombstone size in ColumnFamily.size, which can prevent OOM
+ during sudden mass delete operations by yielding a nonzero liveRatio
+ (CASSANDRA-3741)
+ * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142)
+ * Optimize reads when row deletion timestamps allow us to restrict
+ the set of sstables we check (CASSANDRA-4116)
+ * add support for commitlog archiving and point-in-time recovery
+ (CASSANDRA-3690)
+ * avoid generating redundant compaction tasks during streaming
+ (CASSANDRA-4174)
+ * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to
+ StorageService mbean (CASSANDRA-556)
+ * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079)
+ * optimize truncate when autosnapshot is disabled (CASSANDRA-4153)
+ * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966)
+ * add column limit to cli (CASSANDRA-3012, 4098)
+ * clean up and optimize DataOutputBuffer, used by CQL compression and
+ CompositeType (CASSANDRA-4072)
+ * optimize commitlog checksumming (CASSANDRA-3610)
+ * identify and blacklist corrupted SSTables from future compactions
+ (CASSANDRA-2261)
+ * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
+ * Expose API to repair a user provided range (CASSANDRA-3912)
+ * Add way to force the cassandra-cli to refresh its schema (CASSANDRA-4052)
+ * Avoid having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889)
+ * (cql3) Backwards compatibility for composite comparators in non-cql3-aware
+ clients (CASSANDRA-4093)
+ * (cql3) Fix order by for reversed queries (CASSANDRA-4160)
+ * (cql3) Add ReversedType support (CASSANDRA-4004)
+ * (cql3) Add timeuuid type (CASSANDRA-4194)
+ * (cql3) Minor fixes (CASSANDRA-4185)
+ * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202)
+ * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186)
+ * (cql3) Move max/min compaction thresholds to compaction strategy options
+ (CASSANDRA-4187)
+ * Fix exception during move when localhost is the only source (CASSANDRA-4200)
+ * (cql3) Allow paging through non-ordered partitioner results (CASSANDRA-3771)
+ * (cql3) Fix drop index (CASSANDRA-4192)
+ * (cql3) Don't return range ghosts anymore (CASSANDRA-3982)
+ * fix re-creating Keyspaces/ColumnFamilies with the same name as dropped
+ ones (CASSANDRA-4219)
+ * fix SecondaryIndex LeveledManifest save upon snapshot (CASSANDRA-4230)
+ * fix missing arrayOffset in FBUtilities.hash (CASSANDRA-4250)
+ * (cql3) Add name of parameters in CqlResultSet (CASSANDRA-4242)
+ * (cql3) Correctly validate order by queries (CASSANDRA-4246)
+ * rename stress to cassandra-stress for saner packaging (CASSANDRA-4256)
+ * Fix exception on colum metadata with non-string comparator (CASSANDRA-4269)
+ * Check for unknown/invalid compression options (CASSANDRA-4266)
+ * (cql3) Adds simple access to column timestamp and ttl (CASSANDRA-4217)
+ * (cql3) Fix range queries with secondary indexes (CASSANDRA-4257)
+ * Better error messages from improper input in cli (CASSANDRA-3865)
+ * Try to stop all compaction upon Keyspace or ColumnFamily drop (CASSANDRA-4221)
+ * (cql3) Allow keyspace properties to contain hyphens (CASSANDRA-4278)
+ * (cql3) Correctly validate keyspace access in create table (CASSANDRA-4296)
+ * Avoid deadlock in migration stage (CASSANDRA-3882)
+ * Take supercolumn names and deletion info into account in memtable throughput
+ (CASSANDRA-4264)
+ * Add back backward compatibility for old style replication factor (CASSANDRA-4294)
+ * Preserve compatibility with pre-1.1 index queries (CASSANDRA-4262)
+Merged from 1.0:
+ * Fix super columns bug where cache is not updated (CASSANDRA-4190)
+ * fix maxTimestamp to include row tombstones (CASSANDRA-4116)
+ * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129)
+ * Avoids possible deadlock during bootstrap (CASSANDRA-4159)
+ * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128)
+ * stress tool to return appropriate exit code on failure (CASSANDRA-4188)
+ * fix compaction NPE when out of disk space and assertions disabled
+ (CASSANDRA-3985)
* synchronize LCS getEstimatedTasks to avoid CME (CASSANDRA-4255)
* ensure unique streaming session id's (CASSANDRA-4223)
* kick off background compaction when min/max thresholds change
@@@ -156,44 -8,15 +157,46 @@@
* improve ability of STCS.getBuckets to deal with 100s of 1000s of
sstables, such as when convertinb back from LCS (CASSANDRA-4287)
* Oversize integer in CQL throws NumberFormatException (CASSANDRA-4291)
- * Set gc_grace on index CF to 0 (CASSANDRA-4314)
+ * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195)
+ * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419)
- * Push the validation of secondary index values to the SecondaryIndexManager (CASSANDRA-4240)
-1.0.10
- * fix maxTimestamp to include row tombstones (CASSANDRA-4116)
+
+1.1.0-final
+ * average a reduced liveRatio estimate with the previous one (CASSANDRA-4065)
+ * Allow KS and CF names up to 48 characters (CASSANDRA-4157)
+ * fix stress build (CASSANDRA-4140)
+ * add time remaining estimate to nodetool compactionstats (CASSANDRA-4167)
+ * (cql) fix NPE in cql3 ALTER TABLE (CASSANDRA-4163)
+ * (cql) Add support for CL.TWO and CL.THREE in CQL (CASSANDRA-4156)
+ * (cql) Fix type in CQL3 ALTER TABLE preventing update (CASSANDRA-4170)
+ * (cql) Throw invalid exception from CQL3 on obsolete options (CASSANDRA-4171)
+ * (cqlsh) fix recognizing uppercase SELECT keyword (CASSANDRA-4161)
+ * Pig: wide row support (CASSANDRA-3909)
+Merged from 1.0:
* avoid streaming empty files with bulk loader if sstablewriter errors out
(CASSANDRA-3946)
+
+
+1.1-rc1
+ * Include stress tool in binary builds (CASSANDRA-4103)
+ * (Hadoop) fix wide row iteration when last row read was deleted
+ (CASSANDRA-4154)
+ * fix read_repair_chance to really default to 0.1 in the cli (CASSANDRA-4114)
+ * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042)
+ * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087)
+ * fix KEYS index from skipping results (CASSANDRA-3996)
+ * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076)
+ * make loadNewSStable preserve sstable version (CASSANDRA-4077)
+ * Respect 1.0 cache settings as much as possible when upgrading
+ (CASSANDRA-4088)
+ * relax path length requirement for sstable files when upgrading on
+ non-Windows platforms (CASSANDRA-4110)
+ * fix terminination of the stress.java when errors were encountered
+ (CASSANDRA-4128)
+ * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
+ * Fix get_paged_slice (CASSANDRA-4136)
+ * CQL3: Support slice with exclusive start and stop (CASSANDRA-3785)
+Merged from 1.0:
* support PropertyFileSnitch in bulk loader (CASSANDRA-4145)
* add auto_snapshot option allowing disabling snapshot before drop/truncate
(CASSANDRA-3710)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
index ba7360d,a7bd3fb..05fa70b
--- a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java
@@@ -55,9 -57,15 +57,15 @@@ public abstract class PerColumnSecondar
* @param col all the column info
*/
public abstract void updateColumn(DecoratedKey<?> valueKey, ByteBuffer rowKey, IColumn col) throws IOException;
-
+
public String getNameForSystemTable(ByteBuffer column)
{
- return getIndexName();
+ return getIndexName();
}
+
+ @Override
+ public boolean validate(Column column)
+ {
+ return column.value.remaining() < FBUtilities.MAX_UNSIGNED_SHORT;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndex.java
index 64a696d,3e38f28..3732086
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
@@@ -327,7 -319,9 +328,9 @@@ public abstract class SecondaryInde
index.addColumnDef(cdef);
index.validateOptions();
index.setBaseCfs(baseCfs);
-
+
return index;
}
+
+ public abstract boolean validate(Column column);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
index ba066e2,ecdf8b1..e923575
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
@@@ -20,7 -20,7 +20,14 @@@ package org.apache.cassandra.db.index
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
--import java.util.concurrent.*;
++import java.util.concurrent.ConcurrentNavigableMap;
++import java.util.concurrent.ConcurrentSkipListMap;
++import java.util.concurrent.ExecutionException;
++import java.util.concurrent.Future;
++
++import org.apache.commons.lang.StringUtils;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.ConfigurationException;
@@@ -31,11 -31,13 +38,9 @@@ import org.apache.cassandra.dht.Abstrac
import org.apache.cassandra.dht.LocalToken;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
import org.apache.cassandra.io.sstable.SSTableReader;
+ import org.apache.cassandra.thrift.Column;
-import org.apache.cassandra.thrift.IndexClause;
import org.apache.cassandra.thrift.IndexExpression;
import org.apache.cassandra.utils.ByteBufferUtil;
--import org.apache.commons.lang.StringUtils;
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
/**
* Manages all the indexes associated with a given CFS
@@@ -590,20 -556,14 +595,26 @@@ public class SecondaryIndexManage
//We currently don't support searching across multiple index types
if (indexSearchers.size() > 1)
throw new RuntimeException("Unable to search across multiple secondary index types");
-
-
- return indexSearchers.get(0).search(clause, range, dataFilter);
+
+
+ return indexSearchers.get(0).search(clause, range, maxResults, dataFilter, maxIsColumns);
+ }
+
+ public void setIndexBuilt(Collection<ByteBuffer> indexes)
+ {
+ for (ByteBuffer colName : indexes)
+ indexesByColumn.get(colName).setIndexBuilt(colName);
+ }
+
+ public void setIndexRemoved(Collection<ByteBuffer> indexes)
+ {
+ for (ByteBuffer colName : indexes)
+ indexesByColumn.get(colName).setIndexRemoved(colName);
}
+
+ public boolean validate(Column column)
+ {
+ SecondaryIndex index = getIndexForColumn(column.name);
+ return index != null ? index.validate(column) : true;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/thrift/ThriftValidation.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
index 9c131f9,c025037..af0e668
--- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
+++ b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java
@@@ -21,23 -21,18 +21,20 @@@ package org.apache.cassandra.thrift
*/
- import java.nio.ByteBuffer;
-
import org.junit.Test;
-import org.apache.cassandra.CleanupHelper;
+import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
- import org.apache.cassandra.utils.FBUtilities;
-public class ThriftValidationTest extends CleanupHelper
+public class ThriftValidationTest extends SchemaLoader
{
@Test(expected=InvalidRequestException.class)
public void testValidateCommutativeWithStandard() throws InvalidRequestException