You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/06/10 00:15:26 UTC
svn commit: r1134109 - in /cassandra/branches/cassandra-0.8: ./ contrib/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/service/ test/unit/org/apache/cassandra/
test/unit/org/apache/cassandra/db/ test/unit/org/apac...
Author: jbellis
Date: Thu Jun 9 22:15:25 2011
New Revision: 1134109
URL: http://svn.apache.org/viewvc?rev=1134109&view=rev
Log:
merge from 0.7
Modified:
cassandra/branches/cassandra-0.8/ (props changed)
cassandra/branches/cassandra-0.8/CHANGES.txt
cassandra/branches/cassandra-0.8/contrib/ (props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java
cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java
Propchange: cassandra/branches/cassandra-0.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7:1026516-1133391
+/cassandra/branches/cassandra-0.7:1026516-1133874
/cassandra/branches/cassandra-0.7.0:1053690-1055654
/cassandra/branches/cassandra-0.8:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0:1125021-1130369
Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1134109&r1=1134108&r2=1134109&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Thu Jun 9 22:15:25 2011
@@ -42,6 +42,8 @@
by nio sockets (CASSANDRA-2654)
* restrict repair streaming to specific columnfamilies (CASSANDRA-2280)
* fix nodetool ring use with Ec2Snitch (CASSANDRA-2733)
+ * fix removing columns and subcolumns that are supressed by a row or
+ supercolumn tombstone during replica resolution (CASSANDRA-2590)
0.8.0-final
Propchange: cassandra/branches/cassandra-0.8/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1133391
+/cassandra/branches/cassandra-0.7/contrib:1026516-1133874
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133391
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1133874
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133391
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1133874
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133391
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1133874
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133391
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1133874
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
Propchange: cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 22:15:25 2011
@@ -1,5 +1,5 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133391
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1133874
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125041
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java?rev=1134109&r1=1134108&r2=1134109&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java (original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/RowRepairResolver.java Thu Jun 9 22:15:25 2011
@@ -26,7 +26,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.commons.collections.iterators.CollatingIterator;
+
import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
@@ -122,19 +127,30 @@ public class RowRepairResolver extends A
ColumnFamily resolved = null;
for (ColumnFamily cf : versions)
{
- if (cf != null)
- {
- resolved = cf.cloneMe();
- break;
- }
+ if (cf == null)
+ continue;
+
+ if (resolved == null)
+ resolved = cf.cloneMeShallow();
+ else
+ resolved.delete(cf);
}
if (resolved == null)
return null;
- for (ColumnFamily cf : versions)
- resolved.resolve(cf);
-
- return resolved;
+ // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes.
+ // this will handle removing columns and subcolumns that are supressed by a row or
+ // supercolumn tombstone.
+ QueryFilter filter = new QueryFilter(null, new QueryPath(resolved.metadata().cfName), new IdentityQueryFilter());
+ CollatingIterator iter = new CollatingIterator(resolved.metadata().comparator.columnComparator);
+ for (ColumnFamily version : versions)
+ {
+ if (version == null)
+ continue;
+ iter.addIterator(version.getColumnsMap().values().iterator());
+ }
+ filter.collectCollatedColumns(resolved, iter, Integer.MIN_VALUE);
+ return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE);
}
public Row getData() throws IOException
Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java?rev=1134109&r1=1134108&r2=1134109&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java (original)
+++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/Util.java Thu Jun 9 22:15:25 2011
@@ -58,6 +58,14 @@ public class Util
return new Column(ByteBufferUtil.bytes(name), ByteBufferUtil.bytes(value), timestamp);
}
+ public static SuperColumn superColumn(ColumnFamily cf, String name, Column... columns)
+ {
+ SuperColumn sc = new SuperColumn(ByteBufferUtil.bytes(name), cf.metadata().comparator);
+ for (Column c : columns)
+ sc.addColumn(c);
+ return sc;
+ }
+
public static Token token(String key)
{
return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key));
Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java?rev=1134109&r1=1134108&r2=1134109&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java (original)
+++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/TableTest.java Thu Jun 9 22:15:25 2011
@@ -511,7 +511,18 @@ public class TableTest extends CleanupHe
public static void assertColumns(ColumnFamily cf, String... columnNames)
{
- Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns();
+ assertColumns((IColumnContainer)cf, columnNames);
+ }
+
+ public static void assertSubColumns(ColumnFamily cf, String scName, String... columnNames)
+ {
+ IColumnContainer sc = cf == null ? null : ((IColumnContainer)cf.getColumn(ByteBufferUtil.bytes(scName)));
+ assertColumns(sc, columnNames);
+ }
+
+ public static void assertColumns(IColumnContainer container, String... columnNames)
+ {
+ Collection<IColumn> columns = container == null ? new TreeSet<IColumn>() : container.getSortedColumns();
List<String> L = new ArrayList<String>();
for (IColumn column : columns)
{
@@ -540,9 +551,28 @@ public class TableTest extends CleanupHe
assert Arrays.equals(la, columnNames1)
: String.format("Columns [%s(as string: %s)])] is not expected [%s]",
- ((cf == null) ? "" : cf.getComparator().getColumnsString(columns)),
+ ((container == null) ? "" : container.getComparator().getColumnsString(columns)),
lasb.toString(),
StringUtils.join(columnNames1, ","));
}
+ public static void assertColumn(ColumnFamily cf, String name, String value, long timestamp)
+ {
+ assertColumn(cf.getColumn(ByteBufferUtil.bytes(name)), value, timestamp);
+ }
+
+ public static void assertSubColumn(ColumnFamily cf, String scName, String name, String value, long timestamp)
+ {
+ SuperColumn sc = (SuperColumn)cf.getColumn(ByteBufferUtil.bytes(scName));
+ assertColumn(sc.getSubColumn(ByteBufferUtil.bytes(name)), value, timestamp);
+ }
+
+ public static void assertColumn(IColumn column, String value, long timestamp)
+ {
+ assertNotNull(column);
+ assertEquals(0, ByteBufferUtil.compareUnsigned(column.value(), ByteBufferUtil.bytes(value)));
+ assertEquals(timestamp, column.timestamp());
+ }
+
+
}
Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java?rev=1134109&r1=1134108&r2=1134109&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java (original)
+++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/service/RowResolverTest.java Thu Jun 9 22:15:25 2011
@@ -23,14 +23,16 @@ package org.apache.cassandra.service;
import java.util.Arrays;
-import org.apache.cassandra.SchemaLoader;
import org.junit.Test;
+import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.SuperColumn;
-import static org.apache.cassandra.db.TableTest.assertColumns;
+import static junit.framework.Assert.*;
import static org.apache.cassandra.Util.column;
-import static junit.framework.Assert.assertNull;
+import static org.apache.cassandra.Util.superColumn;
+import static org.apache.cassandra.db.TableTest.*;
public class RowResolverTest extends SchemaLoader
{
@@ -93,4 +95,110 @@ public class RowResolverTest extends Sch
{
assertNull(RowRepairResolver.resolveSuperset(Arrays.<ColumnFamily>asList(null, null)));
}
+
+ @Test
+ public void testResolveDeleted()
+ {
+ // one CF with columns timestamped before a delete in another cf
+ ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf1.addColumn(column("one", "A", 0));
+
+ ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf2.delete((int) (System.currentTimeMillis() / 1000), 1);
+
+ ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2));
+ // no columns in the cf
+ assertColumns(resolved);
+ assertTrue(resolved.isMarkedForDelete());
+ assertEquals(1, resolved.getMarkedForDeleteAt());
+
+ ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+ scf1.addColumn(superColumn(scf1, "super-foo", column("one", "A", 0)));
+
+ ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+ scf2.delete((int) (System.currentTimeMillis() / 1000), 1);
+
+ ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2));
+ // no columns in the cf
+ assertColumns(superResolved);
+ assertTrue(superResolved.isMarkedForDelete());
+ assertEquals(1, superResolved.getMarkedForDeleteAt());
+ }
+
+ @Test
+ public void testResolveDeletedSuper()
+ {
+ // subcolumn is newer than a tombstone on its parent, but not newer than the row deletion
+ ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+ SuperColumn sc = superColumn(scf1, "super-foo", column("one", "A", 1));
+ sc.markForDeleteAt((int) (System.currentTimeMillis() / 1000), 0);
+ scf1.addColumn(sc);
+
+ ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+ scf2.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+ ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2));
+ // no columns in the cf
+ assertColumns(superResolved);
+ assertTrue(superResolved.isMarkedForDelete());
+ assertEquals(2, superResolved.getMarkedForDeleteAt());
+ }
+
+ @Test
+ public void testResolveMultipleDeleted()
+ {
+ // deletes and columns with interleaved timestamp, with out of order return sequence
+
+ ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf1.delete((int) (System.currentTimeMillis() / 1000), 0);
+
+ // these columns created after the previous deletion
+ ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf2.addColumn(column("one", "A", 1));
+ cf2.addColumn(column("two", "A", 1));
+
+ //this column created after the next delete
+ ColumnFamily cf3 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf3.addColumn(column("two", "B", 3));
+
+ ColumnFamily cf4 = ColumnFamily.create("Keyspace1", "Standard1");
+ cf4.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+ ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2, cf3, cf4));
+ // will have deleted marker and one column
+ assertColumns(resolved, "two");
+ assertColumn(resolved, "two", "B", 3);
+ assertTrue(resolved.isMarkedForDelete());
+ assertEquals(2, resolved.getMarkedForDeleteAt());
+
+
+ ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+ scf1.delete((int) (System.currentTimeMillis() / 1000), 0);
+
+ // these columns created after the previous deletion
+ ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+ scf2.addColumn(superColumn(scf2, "super1", column("one", "A", 1), column("two", "A", 1)));
+
+ //these columns created after the next delete
+ ColumnFamily scf3 = ColumnFamily.create("Keyspace1", "Super1");
+ scf3.addColumn(superColumn(scf3, "super1", column("two", "B", 3)));
+ scf3.addColumn(superColumn(scf3, "super2", column("three", "A", 3), column("four", "A", 3)));
+
+ ColumnFamily scf4 = ColumnFamily.create("Keyspace1", "Super1");
+ scf4.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+ ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1, scf2, scf3, scf4));
+ // will have deleted marker and two super cols
+ assertColumns(superResolved, "super1", "super2");
+
+ assertSubColumns(superResolved, "super1", "two");
+ assertSubColumn(superResolved, "super1", "two", "B", 3);
+
+ assertSubColumns(superResolved, "super2", "four", "three");
+ assertSubColumn(superResolved, "super2", "three", "A", 3);
+ assertSubColumn(superResolved, "super2", "four", "A", 3);
+
+ assertTrue(superResolved.isMarkedForDelete());
+ assertEquals(2, superResolved.getMarkedForDeleteAt());
+ }
}