You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2017/10/06 14:19:05 UTC
[01/12] cassandra git commit: Fix handling of cells for removed
column when reading legacy sstables
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 c07441251 -> 8a424cef3
refs/heads/cassandra-3.11 710657d82 -> 3d09901b4
refs/heads/trunk cfee3e93b -> 9663c0093
Fix handling of cells for removed column when reading legacy sstables
Patch by Sylvain Lebresne; reviewed by Robert Stupp for CASSANDRA-13939
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5378ba27
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5378ba27
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5378ba27
Branch: refs/heads/cassandra-3.0
Commit: 5378ba27ff66b4f39e34d613f99b0f2eddf7990d
Parents: c074412
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Oct 5 10:30:56 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:12:56 2017 +0200
----------------------------------------------------------------------
.../org/apache/cassandra/db/LegacyLayout.java | 70 +++++++++++++-------
.../cassandra/db/UnfilteredDeserializer.java | 29 +++++---
2 files changed, 67 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java
index 40b9fd3..3ba96a6 100644
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ b/src/java/org/apache/cassandra/db/LegacyLayout.java
@@ -649,7 +649,7 @@ public abstract class LegacyLayout
boolean foundOne = false;
LegacyAtom atom;
- while ((atom = readLegacyAtom(metadata, in, false)) != null)
+ while ((atom = readLegacyAtomSkippingUnknownColumn(metadata,in)) != null)
{
if (atom.isCell())
{
@@ -672,6 +672,23 @@ public abstract class LegacyLayout
return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW;
}
+ private static LegacyAtom readLegacyAtomSkippingUnknownColumn(CFMetaData metadata, DataInputPlus in)
+ throws IOException
+ {
+ while (true)
+ {
+ try
+ {
+ return readLegacyAtom(metadata, in, false);
+ }
+ catch (UnknownColumnException e)
+ {
+ // Simply skip, as the method name implies.
+ }
+ }
+
+ }
+
private static Row getNextRow(CellGrouper grouper, PeekingIterator<? extends LegacyAtom> cells)
{
if (!cells.hasNext())
@@ -1020,29 +1037,36 @@ public abstract class LegacyLayout
};
}
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
+ public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
+ throws IOException, UnknownColumnException
{
- while (true)
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
-
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and ff that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- assert metadata.ksName.equals(SystemKeyspace.NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
- }
+ ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
+ if (!cellname.hasRemaining())
+ return null; // END_OF_ROW
+
+ try
+ {
+ int b = in.readUnsignedByte();
+ return (b & RANGE_TOMBSTONE_MASK) != 0
+ ? readLegacyRangeTombstoneBody(metadata, in, cellname)
+ : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
+ }
+ catch (UnknownColumnException e)
+ {
+ // We legitimately can get here in 2 cases:
+ // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
+ // 2) for dropped columns.
+ // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
+ // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
+ // but we don't do this here and re-throw the exception because the calling code sometimes has to know
+ // about this happening. This does mean code calling this method should handle this case properly.
+ if (!metadata.ksName.equals(SystemKeyspace.NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
+ throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
+ "This suggest a problem with the schema which doesn't list " +
+ "this column. Even if that column was dropped, it should have " +
+ "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
+
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index ea65633..0aa5741 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -274,16 +274,26 @@ public abstract class UnfilteredDeserializer
private LegacyLayout.LegacyAtom readAtom()
{
- try
- {
- long pos = currentPosition();
- LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
- bytesReadForNextAtom = currentPosition() - pos;
- return atom;
- }
- catch (IOException e)
+ while (true)
{
- throw new IOError(e);
+ try
+ {
+ long pos = currentPosition();
+ LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
+ bytesReadForNextAtom = currentPosition() - pos;
+ return atom;
+ }
+ catch (UnknownColumnException e)
+ {
+ // This is ok, see LegacyLayout.readLegacyAtom() for why this only happens in case were we're ok
+ // skipping the cell. We do want to catch this at this level however because when that happen,
+ // we should *not* count the byte of that discarded cell as part of the bytes for the atom
+ // we will eventually return, as doing so could throw the logic bytesReadForNextAtom participates in.
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
}
}
@@ -407,6 +417,7 @@ public abstract class UnfilteredDeserializer
saved = null;
iterator.clearState();
lastConsumedPosition = currentPosition();
+ bytesReadForNextAtom = 0;
}
// Groups atoms from the input into proper Unfiltered.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[06/12] cassandra git commit: Merge commit
'1527684462f903be6481324e401dc018da7e0e39' into trunk
Posted by sl...@apache.org.
Merge commit '1527684462f903be6481324e401dc018da7e0e39' into trunk
* commit '1527684462f903be6481324e401dc018da7e0e39':
Fix handling of cells for removed column when reading legacy sstables
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9815c466
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9815c466
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9815c466
Branch: refs/heads/trunk
Commit: 9815c46679ad14b038f0314a9b9611e099750724
Parents: cfee3e9 1527684
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:16:53 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:16:53 2017 +0200
----------------------------------------------------------------------
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[09/12] cassandra git commit: Ninja-fix changelog
Posted by sl...@apache.org.
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8a424cef
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8a424cef
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8a424cef
Branch: refs/heads/cassandra-3.11
Commit: 8a424cef32270e75b3a4565793b2d436e6df181f
Parents: 5378ba2
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:07 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:07 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8a424cef/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index df05f7f..6977be7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
3.0.16
- * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
-
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
3.0.15
* Improve TRUNCATE performance (CASSANDRA-13909)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[02/12] cassandra git commit: Fix handling of cells for removed
column when reading legacy sstables
Posted by sl...@apache.org.
Fix handling of cells for removed column when reading legacy sstables
Patch by Sylvain Lebresne; reviewed by Robert Stupp for CASSANDRA-13939
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5378ba27
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5378ba27
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5378ba27
Branch: refs/heads/cassandra-3.11
Commit: 5378ba27ff66b4f39e34d613f99b0f2eddf7990d
Parents: c074412
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Oct 5 10:30:56 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:12:56 2017 +0200
----------------------------------------------------------------------
.../org/apache/cassandra/db/LegacyLayout.java | 70 +++++++++++++-------
.../cassandra/db/UnfilteredDeserializer.java | 29 +++++---
2 files changed, 67 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java
index 40b9fd3..3ba96a6 100644
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ b/src/java/org/apache/cassandra/db/LegacyLayout.java
@@ -649,7 +649,7 @@ public abstract class LegacyLayout
boolean foundOne = false;
LegacyAtom atom;
- while ((atom = readLegacyAtom(metadata, in, false)) != null)
+ while ((atom = readLegacyAtomSkippingUnknownColumn(metadata,in)) != null)
{
if (atom.isCell())
{
@@ -672,6 +672,23 @@ public abstract class LegacyLayout
return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW;
}
+ private static LegacyAtom readLegacyAtomSkippingUnknownColumn(CFMetaData metadata, DataInputPlus in)
+ throws IOException
+ {
+ while (true)
+ {
+ try
+ {
+ return readLegacyAtom(metadata, in, false);
+ }
+ catch (UnknownColumnException e)
+ {
+ // Simply skip, as the method name implies.
+ }
+ }
+
+ }
+
private static Row getNextRow(CellGrouper grouper, PeekingIterator<? extends LegacyAtom> cells)
{
if (!cells.hasNext())
@@ -1020,29 +1037,36 @@ public abstract class LegacyLayout
};
}
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
+ public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
+ throws IOException, UnknownColumnException
{
- while (true)
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
-
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and ff that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- assert metadata.ksName.equals(SystemKeyspace.NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
- }
+ ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
+ if (!cellname.hasRemaining())
+ return null; // END_OF_ROW
+
+ try
+ {
+ int b = in.readUnsignedByte();
+ return (b & RANGE_TOMBSTONE_MASK) != 0
+ ? readLegacyRangeTombstoneBody(metadata, in, cellname)
+ : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
+ }
+ catch (UnknownColumnException e)
+ {
+ // We legitimately can get here in 2 cases:
+ // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
+ // 2) for dropped columns.
+ // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
+ // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
+ // but we don't do this here and re-throw the exception because the calling code sometimes has to know
+ // about this happening. This does mean code calling this method should handle this case properly.
+ if (!metadata.ksName.equals(SystemKeyspace.NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
+ throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
+ "This suggest a problem with the schema which doesn't list " +
+ "this column. Even if that column was dropped, it should have " +
+ "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
+
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index ea65633..0aa5741 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -274,16 +274,26 @@ public abstract class UnfilteredDeserializer
private LegacyLayout.LegacyAtom readAtom()
{
- try
- {
- long pos = currentPosition();
- LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
- bytesReadForNextAtom = currentPosition() - pos;
- return atom;
- }
- catch (IOException e)
+ while (true)
{
- throw new IOError(e);
+ try
+ {
+ long pos = currentPosition();
+ LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
+ bytesReadForNextAtom = currentPosition() - pos;
+ return atom;
+ }
+ catch (UnknownColumnException e)
+ {
+ // This is ok, see LegacyLayout.readLegacyAtom() for why this only happens in case were we're ok
+ // skipping the cell. We do want to catch this at this level however because when that happen,
+ // we should *not* count the byte of that discarded cell as part of the bytes for the atom
+ // we will eventually return, as doing so could throw the logic bytesReadForNextAtom participates in.
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
}
}
@@ -407,6 +417,7 @@ public abstract class UnfilteredDeserializer
saved = null;
iterator.clearState();
lastConsumedPosition = currentPosition();
+ bytesReadForNextAtom = 0;
}
// Groups atoms from the input into proper Unfiltered.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[11/12] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by sl...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
* cassandra-3.0:
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d09901b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d09901b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d09901b
Branch: refs/heads/trunk
Commit: 3d09901b4a0e67ce162283f85c34b1fe3c55dce3
Parents: 1527684 8a424ce
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:36 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:36 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d09901b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index e231bf2,6977be7..879397b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,8 +1,23 @@@
-3.0.16
+3.11.2
+Merged from 3.0:
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
* Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
-3.0.15
+
+3.11.1
+ * Fix the computation of cdc_total_space_in_mb for exabyte filesystems (CASSANDRA-13808)
+ * AbstractTokenTreeBuilder#serializedSize returns wrong value when there is a single leaf and overflow collisions (CASSANDRA-13869)
+ * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
+ * BTree.Builder memory leak (CASSANDRA-13754)
+ * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
+ * Add a skip read validation flag to cassandra-stress (CASSANDRA-13772)
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
* Improve TRUNCATE performance (CASSANDRA-13909)
* Implement short read protection on partition boundaries (CASSANDRA-13595)
* Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[05/12] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by sl...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
* cassandra-3.0:
Fix handling of cells for removed column when reading legacy sstables
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15276844
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15276844
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15276844
Branch: refs/heads/trunk
Commit: 1527684462f903be6481324e401dc018da7e0e39
Parents: 710657d 5378ba2
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:16:25 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:16:25 2017 +0200
----------------------------------------------------------------------
.../org/apache/cassandra/db/LegacyLayout.java | 70 +++++++++++++-------
.../cassandra/db/UnfilteredDeserializer.java | 29 +++++---
2 files changed, 67 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15276844/src/java/org/apache/cassandra/db/LegacyLayout.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/LegacyLayout.java
index 77aa59a,3ba96a6..9679a32
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ b/src/java/org/apache/cassandra/db/LegacyLayout.java
@@@ -1021,29 -1037,36 +1038,36 @@@ public abstract class LegacyLayou
};
}
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
+ public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
+ throws IOException, UnknownColumnException
{
- while (true)
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
+ ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
+ if (!cellname.hasRemaining())
+ return null; // END_OF_ROW
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and ff that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- assert metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
- }
+ try
+ {
+ int b = in.readUnsignedByte();
+ return (b & RANGE_TOMBSTONE_MASK) != 0
+ ? readLegacyRangeTombstoneBody(metadata, in, cellname)
+ : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
+ }
+ catch (UnknownColumnException e)
+ {
+ // We legitimately can get here in 2 cases:
+ // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
+ // 2) for dropped columns.
+ // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
+ // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
+ // but we don't do this here and re-throw the exception because the calling code sometimes has to know
+ // about this happening. This does mean code calling this method should handle this case properly.
- if (!metadata.ksName.equals(SystemKeyspace.NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
++ if (!metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
+ throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
+ "This suggest a problem with the schema which doesn't list " +
+ "this column. Even if that column was dropped, it should have " +
+ "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
+
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15276844/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[08/12] cassandra git commit: Ninja-fix changelog
Posted by sl...@apache.org.
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8a424cef
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8a424cef
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8a424cef
Branch: refs/heads/trunk
Commit: 8a424cef32270e75b3a4565793b2d436e6df181f
Parents: 5378ba2
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:07 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:07 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8a424cef/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index df05f7f..6977be7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
3.0.16
- * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
-
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
3.0.15
* Improve TRUNCATE performance (CASSANDRA-13909)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[04/12] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by sl...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
* cassandra-3.0:
Fix handling of cells for removed column when reading legacy sstables
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/15276844
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/15276844
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/15276844
Branch: refs/heads/cassandra-3.11
Commit: 1527684462f903be6481324e401dc018da7e0e39
Parents: 710657d 5378ba2
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:16:25 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:16:25 2017 +0200
----------------------------------------------------------------------
.../org/apache/cassandra/db/LegacyLayout.java | 70 +++++++++++++-------
.../cassandra/db/UnfilteredDeserializer.java | 29 +++++---
2 files changed, 67 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15276844/src/java/org/apache/cassandra/db/LegacyLayout.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/LegacyLayout.java
index 77aa59a,3ba96a6..9679a32
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ b/src/java/org/apache/cassandra/db/LegacyLayout.java
@@@ -1021,29 -1037,36 +1038,36 @@@ public abstract class LegacyLayou
};
}
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
+ public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
+ throws IOException, UnknownColumnException
{
- while (true)
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
+ ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
+ if (!cellname.hasRemaining())
+ return null; // END_OF_ROW
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and ff that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- assert metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
- }
+ try
+ {
+ int b = in.readUnsignedByte();
+ return (b & RANGE_TOMBSTONE_MASK) != 0
+ ? readLegacyRangeTombstoneBody(metadata, in, cellname)
+ : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
+ }
+ catch (UnknownColumnException e)
+ {
+ // We legitimately can get here in 2 cases:
+ // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
+ // 2) for dropped columns.
+ // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
+ // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
+ // but we don't do this here and re-throw the exception because the calling code sometimes has to know
+ // about this happening. This does mean code calling this method should handle this case properly.
- if (!metadata.ksName.equals(SystemKeyspace.NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
++ if (!metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
+ throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
+ "This suggest a problem with the schema which doesn't list " +
+ "this column. Even if that column was dropped, it should have " +
+ "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
+
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/15276844/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[10/12] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by sl...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
* cassandra-3.0:
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d09901b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d09901b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d09901b
Branch: refs/heads/cassandra-3.11
Commit: 3d09901b4a0e67ce162283f85c34b1fe3c55dce3
Parents: 1527684 8a424ce
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:36 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:36 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d09901b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index e231bf2,6977be7..879397b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,22 -1,8 +1,23 @@@
-3.0.16
+3.11.2
+Merged from 3.0:
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
* Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
-3.0.15
+
+3.11.1
+ * Fix the computation of cdc_total_space_in_mb for exabyte filesystems (CASSANDRA-13808)
+ * AbstractTokenTreeBuilder#serializedSize returns wrong value when there is a single leaf and overflow collisions (CASSANDRA-13869)
+ * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
+ * BTree.Builder memory leak (CASSANDRA-13754)
+ * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
+ * Add a skip read validation flag to cassandra-stress (CASSANDRA-13772)
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
* Improve TRUNCATE performance (CASSANDRA-13909)
* Implement short read protection on partition boundaries (CASSANDRA-13595)
* Fix ISE thrown by UPI.Serializer.hasNext() for some SELECT queries (CASSANDRA-13911)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[12/12] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by sl...@apache.org.
Merge branch 'cassandra-3.11' into trunk
* cassandra-3.11:
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9663c009
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9663c009
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9663c009
Branch: refs/heads/trunk
Commit: 9663c0093840a352faa08e3ffda6160745df75b8
Parents: 9815c46 3d09901
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:46 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:46 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9663c009/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 90c6b9c,879397b..eeed794
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,155 -1,6 +1,156 @@@
+4.0
+ * Checksum sstable metadata (CASSANDRA-13321)
+ * Expose recent histograms in JmxHistograms (CASSANDRA-13642)
+ * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899)
+ * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906)
+ * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925)
+ * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961)
+ * Correctly close netty channels when a stream session ends (CASSANDRA-13905)
+ * Update lz4 to 1.4.0 (CASSANDRA-13741)
+ * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862)
+ * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299)
+ * Use compaction threshold for STCS in L0 (CASSANDRA-13861)
+ * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703)
+ * Add extra information to SASI timeout exception (CASSANDRA-13677)
+ * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818)
+ * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786)
+ * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846)
+ * Add keyspace and table name in schema validation exception (CASSANDRA-13845)
+ * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771)
+ * Make netty EventLoopGroups daemon threads (CASSANDRA-13837)
+ * Race condition when closing stream sessions (CASSANDRA-13852)
+ * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831)
+ * Allow changing log levels via nodetool for related classes (CASSANDRA-12696)
+ * Add stress profile yaml with LWT (CASSANDRA-7960)
+ * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789)
+ * Simplify mx4j configuration (Cassandra-13578)
+ * Fix trigger example on 4.0 (CASSANDRA-13796)
+ * Force minumum timeout value (CASSANDRA-9375)
+ * Use netty for streaming (CASSANDRA-12229)
+ * Use netty for internode messaging (CASSANDRA-8457)
+ * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774)
+ * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758)
+ * Fix pending repair manager index out of bounds check (CASSANDRA-13769)
+ * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576)
+ * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664)
+ * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594)
+ * Fix race / ref leak in anticompaction (CASSANDRA-13688)
+ * Expose tasks queue length via JMX (CASSANDRA-12758)
+ * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751)
+ * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615)
+ * Improve sstablemetadata output (CASSANDRA-11483)
+ * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371)
+ * Introduce error metrics for repair (CASSANDRA-13387)
+ * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732)
+ * Update metrics to 3.1.5 (CASSANDRA-13648)
+ * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699)
+ * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725)
+ * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727)
+ * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996)
+ * Default for start_native_transport now true if not set in config (CASSANDRA-13656)
+ * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583)
+ * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148)
+ * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271)
+ * Use common nowInSec for validation compactions (CASSANDRA-13671)
+ * Improve handling of IR prepare failures (CASSANDRA-13672)
+ * Send IR coordinator messages synchronously (CASSANDRA-13673)
+ * Flush system.repair table before IR finalize promise (CASSANDRA-13660)
+ * Fix column filter creation for wildcard queries (CASSANDRA-13650)
+ * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614)
+ * fix race condition in PendingRepairManager (CASSANDRA-13659)
+ * Allow noop incremental repair state transitions (CASSANDRA-13658)
+ * Run repair with down replicas (CASSANDRA-10446)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130)
+ * Improve calculation of available disk space for compaction (CASSANDRA-13068)
+ * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579)
+ * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570)
+ * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585)
+ * Fix Randomness of stress values (CASSANDRA-12744)
+ * Allow selecting Map values and Set elements (CASSANDRA-7396)
+ * Fast and garbage-free Streaming Histogram (CASSANDRA-13444)
+ * Update repairTime for keyspaces on completion (CASSANDRA-13539)
+ * Add configurable upper bound for validation executor threads (CASSANDRA-13521)
+ * Bring back maxHintTTL propery (CASSANDRA-12982)
+ * Add testing guidelines (CASSANDRA-13497)
+ * Add more repair metrics (CASSANDRA-13531)
+ * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650)
+ * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367)
+ * Log time elapsed for each incremental repair phase (CASSANDRA-13498)
+ * Add multiple table operation support to cassandra-stress (CASSANDRA-8780)
+ * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262)
+ * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421)
+ * Improve incremental repair logging (CASSANDRA-13468)
+ * Start compaction when incremental repair finishes (CASSANDRA-13454)
+ * Add repair streaming preview (CASSANDRA-13257)
+ * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430)
+ * Change protocol to allow sending key space independent of query string (CASSANDRA-10145)
+ * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661)
+ * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354)
+ * Skip building views during base table streams on range movements (CASSANDRA-13065)
+ * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197)
+ * Remove deprecated repair JMX APIs (CASSANDRA-11530)
+ * Fix version check to enable streaming keep-alive (CASSANDRA-12929)
+ * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289)
+ * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324)
+ * Upgrade junit from 4.6 to 4.12 (CASSANDRA-13360)
+ * Cleanup ParentRepairSession after repairs (CASSANDRA-13359)
+ * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336)
+ * Incremental repair not streaming correct sstables (CASSANDRA-13328)
+ * Upgrade the jna version to 4.3.0 (CASSANDRA-13300)
+ * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132)
+ * Remove config option index_interval (CASSANDRA-10671)
+ * Reduce lock contention for collection types and serializers (CASSANDRA-13271)
+ * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283)
+ * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292)
+ * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520)
+ * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226)
+ * Remove unused method (CASSANDRA-13227)
+ * Fix minor bugs related to #9143 (CASSANDRA-13217)
+ * Output warning if user increases RF (CASSANDRA-13079)
+ * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081)
+ * Add support for + and - operations on dates (CASSANDRA-11936)
+ * Fix consistency of incrementally repaired data (CASSANDRA-9143)
+ * Increase commitlog version (CASSANDRA-13161)
+ * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425)
+ * Refactor ColumnCondition (CASSANDRA-12981)
+ * Parallelize streaming of different keyspaces (CASSANDRA-4663)
+ * Improved compactions metrics (CASSANDRA-13015)
+ * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
+ * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
+ * Thrift removal (CASSANDRA-11115)
+ * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
+ * Add column definition kind to dropped columns in schema (CASSANDRA-12705)
+ * Add (automate) Nodetool Documentation (CASSANDRA-12672)
+ * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+ * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422)
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
+ * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
+ * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720)
+ * Trivial format error in StorageProxy (CASSANDRA-13551)
+ * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480)
+ * Anticompaction can cause noisy log messages (CASSANDRA-13684)
+ * Switch to client init for sstabledump (CASSANDRA-13683)
+ * CQLSH: Don't pause when capturing data (CASSANDRA-13743)
+
+
3.11.2
Merged from 3.0:
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
* Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[07/12] cassandra git commit: Ninja-fix changelog
Posted by sl...@apache.org.
Ninja-fix changelog
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8a424cef
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8a424cef
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8a424cef
Branch: refs/heads/cassandra-3.0
Commit: 8a424cef32270e75b3a4565793b2d436e6df181f
Parents: 5378ba2
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Fri Oct 6 16:18:07 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:18:07 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8a424cef/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index df05f7f..6977be7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,6 @@
3.0.16
- * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
-
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
3.0.15
* Improve TRUNCATE performance (CASSANDRA-13909)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[03/12] cassandra git commit: Fix handling of cells for removed
column when reading legacy sstables
Posted by sl...@apache.org.
Fix handling of cells for removed column when reading legacy sstables
Patch by Sylvain Lebresne; reviewed by Robert Stupp for CASSANDRA-13939
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5378ba27
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5378ba27
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5378ba27
Branch: refs/heads/trunk
Commit: 5378ba27ff66b4f39e34d613f99b0f2eddf7990d
Parents: c074412
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Thu Oct 5 10:30:56 2017 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Fri Oct 6 16:12:56 2017 +0200
----------------------------------------------------------------------
.../org/apache/cassandra/db/LegacyLayout.java | 70 +++++++++++++-------
.../cassandra/db/UnfilteredDeserializer.java | 29 +++++---
2 files changed, 67 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/LegacyLayout.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/LegacyLayout.java b/src/java/org/apache/cassandra/db/LegacyLayout.java
index 40b9fd3..3ba96a6 100644
--- a/src/java/org/apache/cassandra/db/LegacyLayout.java
+++ b/src/java/org/apache/cassandra/db/LegacyLayout.java
@@ -649,7 +649,7 @@ public abstract class LegacyLayout
boolean foundOne = false;
LegacyAtom atom;
- while ((atom = readLegacyAtom(metadata, in, false)) != null)
+ while ((atom = readLegacyAtomSkippingUnknownColumn(metadata,in)) != null)
{
if (atom.isCell())
{
@@ -672,6 +672,23 @@ public abstract class LegacyLayout
return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW;
}
+ private static LegacyAtom readLegacyAtomSkippingUnknownColumn(CFMetaData metadata, DataInputPlus in)
+ throws IOException
+ {
+ while (true)
+ {
+ try
+ {
+ return readLegacyAtom(metadata, in, false);
+ }
+ catch (UnknownColumnException e)
+ {
+ // Simply skip, as the method name implies.
+ }
+ }
+
+ }
+
private static Row getNextRow(CellGrouper grouper, PeekingIterator<? extends LegacyAtom> cells)
{
if (!cells.hasNext())
@@ -1020,29 +1037,36 @@ public abstract class LegacyLayout
};
}
- public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
+ public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic)
+ throws IOException, UnknownColumnException
{
- while (true)
- {
- ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
- if (!cellname.hasRemaining())
- return null; // END_OF_ROW
-
- try
- {
- int b = in.readUnsignedByte();
- return (b & RANGE_TOMBSTONE_MASK) != 0
- ? readLegacyRangeTombstoneBody(metadata, in, cellname)
- : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
- }
- catch (UnknownColumnException e)
- {
- // We can get there if we read a cell for a dropped column, and ff that is the case,
- // then simply ignore the cell is fine. But also not that we ignore if it's the
- // system keyspace because for those table we actually remove columns without registering
- // them in the dropped columns
- assert metadata.ksName.equals(SystemKeyspace.NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
- }
+ ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
+ if (!cellname.hasRemaining())
+ return null; // END_OF_ROW
+
+ try
+ {
+ int b = in.readUnsignedByte();
+ return (b & RANGE_TOMBSTONE_MASK) != 0
+ ? readLegacyRangeTombstoneBody(metadata, in, cellname)
+ : readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
+ }
+ catch (UnknownColumnException e)
+ {
+ // We legitimately can get here in 2 cases:
+ // 1) for system tables, because we've unceremoniously removed columns (without registering them as dropped)
+ // 2) for dropped columns.
+ // In any other case, there is a mismatch between the schema and the data, and we complain loudly in
+ // that case. Note that if we are in a legit case of an unknown column, we want to simply skip that cell,
+ // but we don't do this here and re-throw the exception because the calling code sometimes has to know
+ // about this happening. This does mean code calling this method should handle this case properly.
+ if (!metadata.ksName.equals(SystemKeyspace.NAME) && metadata.getDroppedColumnDefinition(e.columnName) == null)
+ throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
+ "This suggest a problem with the schema which doesn't list " +
+ "this column. Even if that column was dropped, it should have " +
+ "been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
+
+ throw e;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5378ba27/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
index ea65633..0aa5741 100644
--- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
+++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
@@ -274,16 +274,26 @@ public abstract class UnfilteredDeserializer
private LegacyLayout.LegacyAtom readAtom()
{
- try
- {
- long pos = currentPosition();
- LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
- bytesReadForNextAtom = currentPosition() - pos;
- return atom;
- }
- catch (IOException e)
+ while (true)
{
- throw new IOError(e);
+ try
+ {
+ long pos = currentPosition();
+ LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
+ bytesReadForNextAtom = currentPosition() - pos;
+ return atom;
+ }
+ catch (UnknownColumnException e)
+ {
+ // This is ok, see LegacyLayout.readLegacyAtom() for why this only happens in case were we're ok
+ // skipping the cell. We do want to catch this at this level however because when that happen,
+ // we should *not* count the byte of that discarded cell as part of the bytes for the atom
+ // we will eventually return, as doing so could throw the logic bytesReadForNextAtom participates in.
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
}
}
@@ -407,6 +417,7 @@ public abstract class UnfilteredDeserializer
saved = null;
iterator.clearState();
lastConsumedPosition = currentPosition();
+ bytesReadForNextAtom = 0;
}
// Groups atoms from the input into proper Unfiltered.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org