You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by al...@apache.org on 2016/07/11 13:33:08 UTC
[1/6] cassandra git commit: Fix upgrading sparse tables that are
incorrectly marked as dense
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 2fa44cd88 -> f37b6f145
refs/heads/cassandra-3.9 d20c765a7 -> 314f60a44
refs/heads/trunk 997cb663e -> ec02e6a25
Fix upgrading sparse tables that are incorrectly marked as dense
patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-11315
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f37b6f14
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f37b6f14
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f37b6f14
Branch: refs/heads/cassandra-3.0
Commit: f37b6f145d3db2b14de38b2c75f390f7a3113c85
Parents: 2fa44cd
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Wed Jun 29 23:15:45 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:29:55 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 58 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e483f1..f2d11b9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
* Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
2.2.8
* Use dedicated thread for JMX notifications (CASSANDRA-12146)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
index 924bd7a..b6d8d2b 100644
--- a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
+++ b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
@@ -284,11 +284,26 @@ public final class LegacySchemaMigrator
AbstractType<?> subComparator = tableRow.has("subcomparator") ? TypeParser.parse(tableRow.getString("subcomparator")) : null;
boolean isSuper = "super".equals(tableRow.getString("type").toLowerCase());
- boolean isDense = tableRow.has("is_dense")
- ? tableRow.getBoolean("is_dense")
- : calculateIsDense(rawComparator, columnRows);
boolean isCompound = rawComparator instanceof CompositeType;
+ /*
+ * Determine whether or not the table is *really* dense
+ * We cannot trust is_dense value of true (see CASSANDRA-11502, that fixed the issue for 2.2 only, and not retroactively),
+ * but we can trust is_dense value of false.
+ */
+ Boolean rawIsDense = tableRow.has("is_dense") ? tableRow.getBoolean("is_dense") : null;
+ boolean isDense;
+ if (rawIsDense != null && !rawIsDense)
+ isDense = false;
+ else
+ isDense = calculateIsDense(rawComparator, columnRows);
+
+ // now, if switched to sparse, remove redundant compact_value column and the last clustering column,
+ // directly copying CASSANDRA-11502 logic. See CASSANDRA-11315.
+ Iterable<UntypedResultSet.Row> filteredColumnRows = !isDense && (rawIsDense == null || rawIsDense)
+ ? filterOutRedundantRowsForSparse(columnRows, isSuper, isCompound)
+ : columnRows;
+
// We don't really use the default validator but as we have it for backward compatibility, we use it to know if it's a counter table
AbstractType<?> defaultValidator = TypeParser.parse(tableRow.getString("default_validator"));
boolean isCounter = defaultValidator instanceof CounterColumnType;
@@ -311,9 +326,9 @@ public final class LegacySchemaMigrator
// previous versions, they may not have the expected schema, so detect if we need to upgrade and do
// it in createColumnsFromColumnRows.
// We can remove this once we don't support upgrade from versions < 3.0.
- boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(columnRows, isSuper, isStaticCompactTable);
+ boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(filteredColumnRows, isSuper, isStaticCompactTable);
- List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(columnRows,
+ List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -323,7 +338,6 @@ public final class LegacySchemaMigrator
isStaticCompactTable,
needsUpgrade);
-
if (needsUpgrade)
{
addDefinitionForUpgrade(columnDefs,
@@ -348,7 +362,7 @@ public final class LegacySchemaMigrator
DatabaseDescriptor.getPartitioner());
Indexes indexes = createIndexesFromColumnRows(cfm,
- columnRows,
+ filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -374,7 +388,7 @@ public final class LegacySchemaMigrator
* information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this
* method does its best to infer whether the table is dense or not based on other elements.
*/
- public static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
+ private static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
{
/*
* As said above, this method is only here because we need to deal with thrift upgrades.
@@ -395,25 +409,44 @@ public final class LegacySchemaMigrator
* in which case it should not be dense. However, we can limit our margin of error by assuming we are
* in the latter case only if the comparator is exactly CompositeType(UTF8Type).
*/
- boolean hasRegular = false;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("regular".equals(columnRow.getString("type")))
+ return false;
+
int maxClusteringIdx = -1;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("clustering_key".equals(columnRow.getString("type")))
+ maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
+ return maxClusteringIdx >= 0
+ ? maxClusteringIdx == comparator.componentsCount() - 1
+ : !isCQL3OnlyPKComparator(comparator);
+ }
+
+ private static Iterable<UntypedResultSet.Row> filterOutRedundantRowsForSparse(UntypedResultSet columnRows, boolean isSuper, boolean isCompound)
+ {
+ Collection<UntypedResultSet.Row> filteredRows = new ArrayList<>();
for (UntypedResultSet.Row columnRow : columnRows)
{
- switch (columnRow.getString("type"))
+ String kind = columnRow.getString("type");
+
+ if ("compact_value".equals(kind))
+ continue;
+
+ if ("clustering_key".equals(kind))
{
- case "clustering_key":
- maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
- break;
- case "regular":
- hasRegular = true;
- break;
+ int position = columnRow.has("component_index") ? columnRow.getInt("component_index") : 0;
+ if (isSuper && position != 0)
+ continue;
+
+ if (!isSuper && !isCompound)
+ continue;
}
+
+ filteredRows.add(columnRow);
}
- return maxClusteringIdx >= 0
- ? maxClusteringIdx == comparator.componentsCount() - 1
- : !hasRegular && !isCQL3OnlyPKComparator(comparator);
+ return filteredRows;
}
private static boolean isCQL3OnlyPKComparator(AbstractType<?> comparator)
@@ -507,7 +540,7 @@ public final class LegacySchemaMigrator
}
// Should only be called on compact tables
- private static boolean checkNeedsUpgrade(UntypedResultSet defs, boolean isSuper, boolean isStaticCompactTable)
+ private static boolean checkNeedsUpgrade(Iterable<UntypedResultSet.Row> defs, boolean isSuper, boolean isStaticCompactTable)
{
if (isSuper)
{
@@ -527,7 +560,7 @@ public final class LegacySchemaMigrator
return !hasRegularColumns(defs);
}
- private static boolean hasRegularColumns(UntypedResultSet columnRows)
+ private static boolean hasRegularColumns(Iterable<UntypedResultSet.Row> columnRows)
{
for (UntypedResultSet.Row row : columnRows)
{
@@ -581,7 +614,7 @@ public final class LegacySchemaMigrator
}
}
- private static boolean hasKind(UntypedResultSet defs, ColumnDefinition.Kind kind)
+ private static boolean hasKind(Iterable<UntypedResultSet.Row> defs, ColumnDefinition.Kind kind)
{
for (UntypedResultSet.Row row : defs)
if (deserializeKind(row.getString("type")) == kind)
@@ -620,7 +653,7 @@ public final class LegacySchemaMigrator
}
}
- private static List<ColumnDefinition> createColumnsFromColumnRows(UntypedResultSet rows,
+ private static List<ColumnDefinition> createColumnsFromColumnRows(Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,
@@ -689,7 +722,7 @@ public final class LegacySchemaMigrator
}
private static Indexes createIndexesFromColumnRows(CFMetaData cfm,
- UntypedResultSet rows,
+ Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,
[4/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.9
Posted by al...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.9
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/314f60a4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/314f60a4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/314f60a4
Branch: refs/heads/trunk
Commit: 314f60a44e969fb7da62a21275d94524277c4eb3
Parents: d20c765 f37b6f1
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Jul 11 14:32:30 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:32:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 59 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/314f60a4/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9df49f3,f2d11b9..44fe26c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,5 +1,11 @@@
+3.9
+ * Partial revert of CASSANDRA-11971, cannot recycle buffer in SP.sendMessagesToNonlocalDC (CASSANDRA-11950)
+ * Fix hdr logging for single operation workloads (CASSANDRA-12145)
+ * Fix SASI PREFIX search in CONTAINS mode with partial terms (CASSANDRA-12073)
+ * Increase size of flushExecutor thread pool (CASSANDRA-12071)
+Merged from 3.0:
+ 3.0.9
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
* Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
2.2.8
* Use dedicated thread for JMX notifications (CASSANDRA-12146)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/314f60a4/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
----------------------------------------------------------------------
[6/6] cassandra git commit: Merge branch 'cassandra-3.9' into trunk
Posted by al...@apache.org.
Merge branch 'cassandra-3.9' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ec02e6a2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ec02e6a2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ec02e6a2
Branch: refs/heads/trunk
Commit: ec02e6a25704f79ae5fda1aeaf0d515528c38929
Parents: 997cb66 314f60a
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Jul 11 14:32:40 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:32:40 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 59 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ec02e6a2/CHANGES.txt
----------------------------------------------------------------------
[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.9
Posted by al...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.9
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/314f60a4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/314f60a4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/314f60a4
Branch: refs/heads/cassandra-3.9
Commit: 314f60a44e969fb7da62a21275d94524277c4eb3
Parents: d20c765 f37b6f1
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Mon Jul 11 14:32:30 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:32:30 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 59 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/314f60a4/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9df49f3,f2d11b9..44fe26c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,5 +1,11 @@@
+3.9
+ * Partial revert of CASSANDRA-11971, cannot recycle buffer in SP.sendMessagesToNonlocalDC (CASSANDRA-11950)
+ * Fix hdr logging for single operation workloads (CASSANDRA-12145)
+ * Fix SASI PREFIX search in CONTAINS mode with partial terms (CASSANDRA-12073)
+ * Increase size of flushExecutor thread pool (CASSANDRA-12071)
+Merged from 3.0:
+ 3.0.9
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
* Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
2.2.8
* Use dedicated thread for JMX notifications (CASSANDRA-12146)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/314f60a4/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
----------------------------------------------------------------------
[2/6] cassandra git commit: Fix upgrading sparse tables that are
incorrectly marked as dense
Posted by al...@apache.org.
Fix upgrading sparse tables that are incorrectly marked as dense
patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-11315
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f37b6f14
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f37b6f14
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f37b6f14
Branch: refs/heads/cassandra-3.9
Commit: f37b6f145d3db2b14de38b2c75f390f7a3113c85
Parents: 2fa44cd
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Wed Jun 29 23:15:45 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:29:55 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 58 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e483f1..f2d11b9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
* Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
2.2.8
* Use dedicated thread for JMX notifications (CASSANDRA-12146)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
index 924bd7a..b6d8d2b 100644
--- a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
+++ b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
@@ -284,11 +284,26 @@ public final class LegacySchemaMigrator
AbstractType<?> subComparator = tableRow.has("subcomparator") ? TypeParser.parse(tableRow.getString("subcomparator")) : null;
boolean isSuper = "super".equals(tableRow.getString("type").toLowerCase());
- boolean isDense = tableRow.has("is_dense")
- ? tableRow.getBoolean("is_dense")
- : calculateIsDense(rawComparator, columnRows);
boolean isCompound = rawComparator instanceof CompositeType;
+ /*
+ * Determine whether or not the table is *really* dense
+ * We cannot trust is_dense value of true (see CASSANDRA-11502, that fixed the issue for 2.2 only, and not retroactively),
+ * but we can trust is_dense value of false.
+ */
+ Boolean rawIsDense = tableRow.has("is_dense") ? tableRow.getBoolean("is_dense") : null;
+ boolean isDense;
+ if (rawIsDense != null && !rawIsDense)
+ isDense = false;
+ else
+ isDense = calculateIsDense(rawComparator, columnRows);
+
+ // now, if switched to sparse, remove redundant compact_value column and the last clustering column,
+ // directly copying CASSANDRA-11502 logic. See CASSANDRA-11315.
+ Iterable<UntypedResultSet.Row> filteredColumnRows = !isDense && (rawIsDense == null || rawIsDense)
+ ? filterOutRedundantRowsForSparse(columnRows, isSuper, isCompound)
+ : columnRows;
+
// We don't really use the default validator but as we have it for backward compatibility, we use it to know if it's a counter table
AbstractType<?> defaultValidator = TypeParser.parse(tableRow.getString("default_validator"));
boolean isCounter = defaultValidator instanceof CounterColumnType;
@@ -311,9 +326,9 @@ public final class LegacySchemaMigrator
// previous versions, they may not have the expected schema, so detect if we need to upgrade and do
// it in createColumnsFromColumnRows.
// We can remove this once we don't support upgrade from versions < 3.0.
- boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(columnRows, isSuper, isStaticCompactTable);
+ boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(filteredColumnRows, isSuper, isStaticCompactTable);
- List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(columnRows,
+ List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -323,7 +338,6 @@ public final class LegacySchemaMigrator
isStaticCompactTable,
needsUpgrade);
-
if (needsUpgrade)
{
addDefinitionForUpgrade(columnDefs,
@@ -348,7 +362,7 @@ public final class LegacySchemaMigrator
DatabaseDescriptor.getPartitioner());
Indexes indexes = createIndexesFromColumnRows(cfm,
- columnRows,
+ filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -374,7 +388,7 @@ public final class LegacySchemaMigrator
* information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this
* method does its best to infer whether the table is dense or not based on other elements.
*/
- public static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
+ private static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
{
/*
* As said above, this method is only here because we need to deal with thrift upgrades.
@@ -395,25 +409,44 @@ public final class LegacySchemaMigrator
* in which case it should not be dense. However, we can limit our margin of error by assuming we are
* in the latter case only if the comparator is exactly CompositeType(UTF8Type).
*/
- boolean hasRegular = false;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("regular".equals(columnRow.getString("type")))
+ return false;
+
int maxClusteringIdx = -1;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("clustering_key".equals(columnRow.getString("type")))
+ maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
+ return maxClusteringIdx >= 0
+ ? maxClusteringIdx == comparator.componentsCount() - 1
+ : !isCQL3OnlyPKComparator(comparator);
+ }
+
+ private static Iterable<UntypedResultSet.Row> filterOutRedundantRowsForSparse(UntypedResultSet columnRows, boolean isSuper, boolean isCompound)
+ {
+ Collection<UntypedResultSet.Row> filteredRows = new ArrayList<>();
for (UntypedResultSet.Row columnRow : columnRows)
{
- switch (columnRow.getString("type"))
+ String kind = columnRow.getString("type");
+
+ if ("compact_value".equals(kind))
+ continue;
+
+ if ("clustering_key".equals(kind))
{
- case "clustering_key":
- maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
- break;
- case "regular":
- hasRegular = true;
- break;
+ int position = columnRow.has("component_index") ? columnRow.getInt("component_index") : 0;
+ if (isSuper && position != 0)
+ continue;
+
+ if (!isSuper && !isCompound)
+ continue;
}
+
+ filteredRows.add(columnRow);
}
- return maxClusteringIdx >= 0
- ? maxClusteringIdx == comparator.componentsCount() - 1
- : !hasRegular && !isCQL3OnlyPKComparator(comparator);
+ return filteredRows;
}
private static boolean isCQL3OnlyPKComparator(AbstractType<?> comparator)
@@ -507,7 +540,7 @@ public final class LegacySchemaMigrator
}
// Should only be called on compact tables
- private static boolean checkNeedsUpgrade(UntypedResultSet defs, boolean isSuper, boolean isStaticCompactTable)
+ private static boolean checkNeedsUpgrade(Iterable<UntypedResultSet.Row> defs, boolean isSuper, boolean isStaticCompactTable)
{
if (isSuper)
{
@@ -527,7 +560,7 @@ public final class LegacySchemaMigrator
return !hasRegularColumns(defs);
}
- private static boolean hasRegularColumns(UntypedResultSet columnRows)
+ private static boolean hasRegularColumns(Iterable<UntypedResultSet.Row> columnRows)
{
for (UntypedResultSet.Row row : columnRows)
{
@@ -581,7 +614,7 @@ public final class LegacySchemaMigrator
}
}
- private static boolean hasKind(UntypedResultSet defs, ColumnDefinition.Kind kind)
+ private static boolean hasKind(Iterable<UntypedResultSet.Row> defs, ColumnDefinition.Kind kind)
{
for (UntypedResultSet.Row row : defs)
if (deserializeKind(row.getString("type")) == kind)
@@ -620,7 +653,7 @@ public final class LegacySchemaMigrator
}
}
- private static List<ColumnDefinition> createColumnsFromColumnRows(UntypedResultSet rows,
+ private static List<ColumnDefinition> createColumnsFromColumnRows(Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,
@@ -689,7 +722,7 @@ public final class LegacySchemaMigrator
}
private static Indexes createIndexesFromColumnRows(CFMetaData cfm,
- UntypedResultSet rows,
+ Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,
[3/6] cassandra git commit: Fix upgrading sparse tables that are
incorrectly marked as dense
Posted by al...@apache.org.
Fix upgrading sparse tables that are incorrectly marked as dense
patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-11315
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f37b6f14
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f37b6f14
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f37b6f14
Branch: refs/heads/trunk
Commit: f37b6f145d3db2b14de38b2c75f390f7a3113c85
Parents: 2fa44cd
Author: Aleksey Yeschenko <al...@apache.org>
Authored: Wed Jun 29 23:15:45 2016 +0100
Committer: Aleksey Yeschenko <al...@apache.org>
Committed: Mon Jul 11 14:29:55 2016 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/schema/LegacySchemaMigrator.java | 81 ++++++++++++++------
2 files changed, 58 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e483f1..f2d11b9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.9
+ * Fix upgrading sparse tables that are incorrectly marked as dense (CASSANDRA-11315)
* Fix reverse queries ignoring range tombstones (CASSANDRA-11733)
2.2.8
* Use dedicated thread for JMX notifications (CASSANDRA-12146)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f37b6f14/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
index 924bd7a..b6d8d2b 100644
--- a/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
+++ b/src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
@@ -284,11 +284,26 @@ public final class LegacySchemaMigrator
AbstractType<?> subComparator = tableRow.has("subcomparator") ? TypeParser.parse(tableRow.getString("subcomparator")) : null;
boolean isSuper = "super".equals(tableRow.getString("type").toLowerCase());
- boolean isDense = tableRow.has("is_dense")
- ? tableRow.getBoolean("is_dense")
- : calculateIsDense(rawComparator, columnRows);
boolean isCompound = rawComparator instanceof CompositeType;
+ /*
+ * Determine whether or not the table is *really* dense
+ * We cannot trust is_dense value of true (see CASSANDRA-11502, that fixed the issue for 2.2 only, and not retroactively),
+ * but we can trust is_dense value of false.
+ */
+ Boolean rawIsDense = tableRow.has("is_dense") ? tableRow.getBoolean("is_dense") : null;
+ boolean isDense;
+ if (rawIsDense != null && !rawIsDense)
+ isDense = false;
+ else
+ isDense = calculateIsDense(rawComparator, columnRows);
+
+ // now, if switched to sparse, remove redundant compact_value column and the last clustering column,
+ // directly copying CASSANDRA-11502 logic. See CASSANDRA-11315.
+ Iterable<UntypedResultSet.Row> filteredColumnRows = !isDense && (rawIsDense == null || rawIsDense)
+ ? filterOutRedundantRowsForSparse(columnRows, isSuper, isCompound)
+ : columnRows;
+
// We don't really use the default validator but as we have it for backward compatibility, we use it to know if it's a counter table
AbstractType<?> defaultValidator = TypeParser.parse(tableRow.getString("default_validator"));
boolean isCounter = defaultValidator instanceof CounterColumnType;
@@ -311,9 +326,9 @@ public final class LegacySchemaMigrator
// previous versions, they may not have the expected schema, so detect if we need to upgrade and do
// it in createColumnsFromColumnRows.
// We can remove this once we don't support upgrade from versions < 3.0.
- boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(columnRows, isSuper, isStaticCompactTable);
+ boolean needsUpgrade = !isCQLTable && checkNeedsUpgrade(filteredColumnRows, isSuper, isStaticCompactTable);
- List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(columnRows,
+ List<ColumnDefinition> columnDefs = createColumnsFromColumnRows(filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -323,7 +338,6 @@ public final class LegacySchemaMigrator
isStaticCompactTable,
needsUpgrade);
-
if (needsUpgrade)
{
addDefinitionForUpgrade(columnDefs,
@@ -348,7 +362,7 @@ public final class LegacySchemaMigrator
DatabaseDescriptor.getPartitioner());
Indexes indexes = createIndexesFromColumnRows(cfm,
- columnRows,
+ filteredColumnRows,
ksName,
cfName,
rawComparator,
@@ -374,7 +388,7 @@ public final class LegacySchemaMigrator
* information for table just created through thrift, nor for table prior to CASSANDRA-7744, so this
* method does its best to infer whether the table is dense or not based on other elements.
*/
- public static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
+ private static boolean calculateIsDense(AbstractType<?> comparator, UntypedResultSet columnRows)
{
/*
* As said above, this method is only here because we need to deal with thrift upgrades.
@@ -395,25 +409,44 @@ public final class LegacySchemaMigrator
* in which case it should not be dense. However, we can limit our margin of error by assuming we are
* in the latter case only if the comparator is exactly CompositeType(UTF8Type).
*/
- boolean hasRegular = false;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("regular".equals(columnRow.getString("type")))
+ return false;
+
int maxClusteringIdx = -1;
+ for (UntypedResultSet.Row columnRow : columnRows)
+ if ("clustering_key".equals(columnRow.getString("type")))
+ maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
+ return maxClusteringIdx >= 0
+ ? maxClusteringIdx == comparator.componentsCount() - 1
+ : !isCQL3OnlyPKComparator(comparator);
+ }
+
+ private static Iterable<UntypedResultSet.Row> filterOutRedundantRowsForSparse(UntypedResultSet columnRows, boolean isSuper, boolean isCompound)
+ {
+ Collection<UntypedResultSet.Row> filteredRows = new ArrayList<>();
for (UntypedResultSet.Row columnRow : columnRows)
{
- switch (columnRow.getString("type"))
+ String kind = columnRow.getString("type");
+
+ if ("compact_value".equals(kind))
+ continue;
+
+ if ("clustering_key".equals(kind))
{
- case "clustering_key":
- maxClusteringIdx = Math.max(maxClusteringIdx, columnRow.has("component_index") ? columnRow.getInt("component_index") : 0);
- break;
- case "regular":
- hasRegular = true;
- break;
+ int position = columnRow.has("component_index") ? columnRow.getInt("component_index") : 0;
+ if (isSuper && position != 0)
+ continue;
+
+ if (!isSuper && !isCompound)
+ continue;
}
+
+ filteredRows.add(columnRow);
}
- return maxClusteringIdx >= 0
- ? maxClusteringIdx == comparator.componentsCount() - 1
- : !hasRegular && !isCQL3OnlyPKComparator(comparator);
+ return filteredRows;
}
private static boolean isCQL3OnlyPKComparator(AbstractType<?> comparator)
@@ -507,7 +540,7 @@ public final class LegacySchemaMigrator
}
// Should only be called on compact tables
- private static boolean checkNeedsUpgrade(UntypedResultSet defs, boolean isSuper, boolean isStaticCompactTable)
+ private static boolean checkNeedsUpgrade(Iterable<UntypedResultSet.Row> defs, boolean isSuper, boolean isStaticCompactTable)
{
if (isSuper)
{
@@ -527,7 +560,7 @@ public final class LegacySchemaMigrator
return !hasRegularColumns(defs);
}
- private static boolean hasRegularColumns(UntypedResultSet columnRows)
+ private static boolean hasRegularColumns(Iterable<UntypedResultSet.Row> columnRows)
{
for (UntypedResultSet.Row row : columnRows)
{
@@ -581,7 +614,7 @@ public final class LegacySchemaMigrator
}
}
- private static boolean hasKind(UntypedResultSet defs, ColumnDefinition.Kind kind)
+ private static boolean hasKind(Iterable<UntypedResultSet.Row> defs, ColumnDefinition.Kind kind)
{
for (UntypedResultSet.Row row : defs)
if (deserializeKind(row.getString("type")) == kind)
@@ -620,7 +653,7 @@ public final class LegacySchemaMigrator
}
}
- private static List<ColumnDefinition> createColumnsFromColumnRows(UntypedResultSet rows,
+ private static List<ColumnDefinition> createColumnsFromColumnRows(Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,
@@ -689,7 +722,7 @@ public final class LegacySchemaMigrator
}
private static Indexes createIndexesFromColumnRows(CFMetaData cfm,
- UntypedResultSet rows,
+ Iterable<UntypedResultSet.Row> rows,
String keyspace,
String table,
AbstractType<?> rawComparator,