You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2011/11/23 09:04:46 UTC
svn commit: r1205319 - in /cassandra/trunk: ./ contrib/
interface/thrift/gen-java/org/apache/cassandra/thrift/
src/java/org/apache/cassandra/db/context/
test/unit/org/apache/cassandra/db/context/
Author: slebresne
Date: Wed Nov 23 08:04:45 2011
New Revision: 1205319
URL: http://svn.apache.org/viewvc?rev=1205319&view=rev
Log:
merge from 1.0
Modified:
cassandra/trunk/ (props changed)
cassandra/trunk/CHANGES.txt
cassandra/trunk/contrib/ (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed)
cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed)
cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java
cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7:1026516-1183000
/cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0:1125021-1130369
/cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Nov 23 08:04:45 2011
@@ -21,6 +21,7 @@
* fix incorrect query results due to invalid max timestamp (CASSANDRA-3510)
Merged from 0.8:
* fix concurrence issue in the FailureDetector (CASSANDRA-3519)
+ * fix array out of bounds error in counter shard removal (CASSANDRA-3514)
1.0.3
* revert name-based query defragmentation aka CASSANDRA-2503 (CASSANDRA-3491)
Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
/cassandra/branches/cassandra-0.7/contrib:1026516-1183000
/cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
/cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/contrib:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1183000
/cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
/cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
/cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1204646,1204869,1204893,1205317
/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
/cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
/cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java Wed Nov 23 08:04:45 2011
@@ -657,6 +657,7 @@ public class CounterContext implements I
int hlength = headerLength(context);
ContextState state = new ContextState(context, hlength);
int removedShards = 0;
+ int removedDelta = 0;
while (state.hasRemaining())
{
long clock = state.getClock();
@@ -684,7 +685,11 @@ public class CounterContext implements I
}
if (-((int)(clock / 1000)) < gcBefore)
+ {
removedShards++;
+ if (state.isDelta())
+ removedDelta++;
+ }
}
state.moveToNext();
}
@@ -692,9 +697,9 @@ public class CounterContext implements I
if (removedShards == 0)
return context;
-
- int removedHeaderSize = removedShards * HEADER_ELT_LENGTH;
- int newSize = context.remaining() - removedHeaderSize - (removedShards * STEP_LENGTH);
+ int removedHeaderSize = removedDelta * HEADER_ELT_LENGTH;
+ int removedBodySize = removedShards * STEP_LENGTH;
+ int newSize = context.remaining() - removedHeaderSize - removedBodySize;
int newHlength = hlength - removedHeaderSize;
ByteBuffer cleanedContext = HeapAllocator.instance.allocate(newSize);
cleanedContext.putShort(cleanedContext.position(), (short) ((newHlength - HEADER_SIZE_LENGTH) / HEADER_ELT_LENGTH));
@@ -704,10 +709,11 @@ public class CounterContext implements I
while (state.hasRemaining())
{
long clock = state.getClock();
- if (!(clock < 0 && state.getCount() == 0))
+ if (clock >= 0 || state.getCount() != 0 || -((int)(clock / 1000)) >= gcBefore)
{
state.copyTo(cleaned);
}
+
state.moveToNext();
}
return cleanedContext;
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java Wed Nov 23 08:04:45 2011
@@ -423,4 +423,81 @@ public class CounterContextTest
assert cc.total(ctx.context) == cc.total(cleaned);
assert cleaned.remaining() == ctx.context.remaining() - stepLength - 2;
}
+
+ @Test
+ public void testRemoveOldShardsNotAllExpiring()
+ {
+ runRemoveOldShardsNotAllExpiring(HeapAllocator.instance);
+ runRemoveOldShardsNotAllExpiring(bumpedSlab());
+ }
+
+ private void runRemoveOldShardsNotAllExpiring(Allocator allocator)
+ {
+ NodeId id1 = NodeId.fromInt(1);
+ NodeId id3 = NodeId.fromInt(3);
+ NodeId id6 = NodeId.fromInt(6);
+ List<NodeId.NodeIdRecord> records = new ArrayList<NodeId.NodeIdRecord>();
+ records.add(new NodeId.NodeIdRecord(id1, 2L));
+ records.add(new NodeId.NodeIdRecord(id3, 4L));
+ records.add(new NodeId.NodeIdRecord(id6, 10L));
+
+ ContextState ctx = ContextState.allocate(6, 3, allocator);
+ ctx.writeElement(id1, 0L, 1L, true);
+ ctx.writeElement(NodeId.fromInt(2), 0L, 2L);
+ ctx.writeElement(id3, 0L, 3L, true);
+ ctx.writeElement(NodeId.fromInt(4), 0L, 3L);
+ ctx.writeElement(NodeId.fromInt(5), 0L, 3L, true);
+ ctx.writeElement(id6, 0L, 6L);
+
+ int timeFirstMerge = (int)(System.currentTimeMillis() / 1000);
+
+ // First, only merge the first id
+ ByteBuffer merger = cc.computeOldShardMerger(ctx.context, records, 3L);
+ ByteBuffer merged = cc.merge(ctx.context, merger, allocator);
+ assert cc.total(ctx.context) == cc.total(merged);
+
+ try
+ {
+ Thread.sleep(2000);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError();
+ }
+
+ // merge the second one
+ ByteBuffer merger2 = cc.computeOldShardMerger(merged, records, 7L);
+ ByteBuffer merged2 = cc.merge(merged, merger2, allocator);
+ assert cc.total(ctx.context) == cc.total(merged2);
+
+ ByteBuffer cleaned = cc.removeOldShards(merged2, timeFirstMerge + 1);
+ assert cc.total(ctx.context) == cc.total(cleaned);
+ assert cleaned.remaining() == ctx.context.remaining();
+
+ // We should have cleaned id1 but not id3
+ ContextState m = new ContextState(cleaned);
+ m.moveToNext();
+ assert m.getNodeId().equals(id3);
+
+ }
+
+ @Test
+ public void testRemoveNotDeltaOldShards()
+ {
+ runRemoveNotDeltaOldShards(HeapAllocator.instance);
+ runRemoveNotDeltaOldShards(bumpedSlab());
+ }
+
+ private void runRemoveNotDeltaOldShards(Allocator allocator)
+ {
+ ContextState ctx = ContextState.allocate(4, 1, allocator);
+ ctx.writeElement(NodeId.fromInt(1), 1L, 1L, true);
+ ctx.writeElement(NodeId.fromInt(2), -System.currentTimeMillis(), 0L);
+ ctx.writeElement(NodeId.fromInt(3), -System.currentTimeMillis(), 0L);
+ ctx.writeElement(NodeId.fromInt(4), -System.currentTimeMillis(), 0L);
+
+ ByteBuffer cleaned = cc.removeOldShards(ctx.context, (int)(System.currentTimeMillis() / 1000) + 1);
+ assert cc.total(ctx.context) == cc.total(cleaned);
+ assert cleaned.remaining() == ctx.context.remaining() - 3 * stepLength;
+ }
}