You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2017/08/28 13:42:41 UTC
[1/6] cassandra git commit: Dont throw IOExceptions in when opening
sstables
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.0 95f1b23bf -> dfbe3fabd
refs/heads/cassandra-3.11 26fedcd0b -> 809f3b30e
refs/heads/trunk 68c2d2e6a -> 326f3a7c7
Dont throw IOExceptions in when opening sstables
Patch by marcuse; reviewed by Ariel Weisberg for CASSANDRA-13620
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/dfbe3fab
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/dfbe3fab
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/dfbe3fab
Branch: refs/heads/cassandra-3.0
Commit: dfbe3fabd266493e698c194ef90b4dfc7d62b030
Parents: 95f1b23
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 19 15:22:57 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:37:26 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++++++++++++++---
.../io/sstable/format/SSTableReader.java | 19 +++++++++++++++--
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
4 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ba35152..5ccd5cd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.15
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
* Fix the merging of cells with different user type versions (CASSANDRA-13776)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
* Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f720330..7251244 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -59,8 +59,10 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
@@ -691,7 +693,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
catch (IOException e)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(e, entry.getKey().filenameFor(Component.STATS)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, e);
continue;
}
@@ -718,9 +721,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
}
- catch (IOException e)
+ catch (CorruptSSTableException ex)
+ {
+ FileUtils.handleCorruptSSTable(ex);
+ logger.error("Corrupt sstable {}; skipping table", entry, ex);
+ continue;
+ }
+ catch (FSError ex)
+ {
+ FileUtils.handleFSError(ex);
+ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
+ continue;
+ }
+ catch (IOException ex)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
continue;
}
newSSTables.add(reader);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index cd41b5b..d56b3e7 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -446,7 +446,16 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+
+ Map<MetadataType, MetadataComponent> sstableMetadata;
+ try
+ {
+ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
@@ -488,6 +497,11 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstable;
}
+ catch (IOException e)
+ {
+ sstable.selfRef().release();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
catch (Throwable t)
{
sstable.selfRef().release();
@@ -534,6 +548,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
}
catch (IOException ex)
{
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
return;
}
@@ -1177,7 +1192,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
if (this.first.compareTo(this.last) > 0)
{
- throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
+ throw new CorruptSSTableException(new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)), getFilename());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index f97d9a9..08336a1 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -349,7 +349,7 @@ public class ScrubTest
SSTableReader.open(desc, cfs.metadata);
fail("SSTR validation should have caught the out-of-order rows");
}
- catch (IllegalStateException ise)
+ catch (CorruptSSTableException ise)
{ /* this is expected */ }
// open without validation for scrubbing
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/326f3a7c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/326f3a7c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/326f3a7c
Branch: refs/heads/trunk
Commit: 326f3a7c7d8f20d0389d9cc036b0bb32d37462be
Parents: 68c2d2e 809f3b3
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Aug 28 15:40:11 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:41:04 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 19 ++++--
.../cassandra/io/sstable/SSTableLoader.java | 4 +-
.../io/sstable/format/SSTableReader.java | 66 +++++++++++++-------
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
5 files changed, 62 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 73e7db6,23d0b8e..5aecc9d
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -62,8 -60,10 +62,11 @@@ import org.apache.cassandra.exceptions.
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
+ import org.apache.cassandra.io.FSError;
++import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
@@@ -781,11 -767,24 +785,18 @@@ public class ColumnFamilyStore implemen
{
reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
}
- catch (IOException e)
+ catch (CorruptSSTableException ex)
+ {
+ FileUtils.handleCorruptSSTable(ex);
+ logger.error("Corrupt sstable {}; skipping table", entry, ex);
+ continue;
+ }
+ catch (FSError ex)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleFSError(ex);
+ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
continue;
}
- catch (IOException ex)
- {
- FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
- continue;
- }
newSSTables.add(reader);
}
@@@ -1912,7 -1906,7 +1923,7 @@@
}
}
}
-- catch (IOException | RuntimeException e)
++ catch (FSReadError | RuntimeException e)
{
// In case one of the snapshot sstables fails to open,
// we must release the references to the ones we opened so far
http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index dc56520,043f6fa..69e5c85
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@@ -25,7 -25,7 +25,8 @@@ import java.util.*
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
-import org.apache.cassandra.config.CFMetaData;
++import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.dht.Range;
@@@ -138,8 -138,8 +139,9 @@@ public class SSTableLoader implements S
// to conserve heap space when bulk loading
sstable.releaseSummary();
}
-- catch (IOException e)
++ catch (FSError e)
{
++ // todo: should we really continue if we can't open all sstables?
outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
}
return false;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index bb1eb0d,03af2bb..2801e36
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -344,9 -352,9 +344,9 @@@ public abstract class SSTableReader ext
return base;
}
-- public static SSTableReader open(Descriptor descriptor) throws IOException
++ public static SSTableReader open(Descriptor descriptor)
{
- CFMetaData metadata;
+ TableMetadataRef metadata;
if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
{
int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
@@@ -362,24 -374,24 +362,24 @@@
return open(descriptor, metadata);
}
- public static SSTableReader open(Descriptor desc, TableMetadataRef metadata) throws IOException
- public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException
++ public static SSTableReader open(Descriptor desc, TableMetadataRef metadata)
{
return open(desc, componentsFor(desc), metadata);
}
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) throws IOException
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
++ public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
return open(descriptor, components, metadata, true, true);
}
// use only for offline or "Standalone" operations
-- public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs) throws IOException
++ public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs)
{
return open(descriptor, components, cfs.metadata, false, false); // do not track hotness
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata) throws IOException
- public static SSTableReader openNoValidation(Descriptor descriptor, CFMetaData metadata) throws IOException
++ public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata)
{
return open(descriptor, componentsFor(descriptor), metadata, false, false); // do not track hotness
}
@@@ -393,14 -405,14 +393,22 @@@
* @return opened SSTableReader
* @throws IOException
*/
- public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata) throws IOException
- public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
++ public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
// Minimum components without which we can't do anything
assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
-- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
++ Map<MetadataType, MetadataComponent> sstableMetadata;
++ try
++ {
++ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
++ }
++ catch (IOException e)
++ {
++ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
++ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
@@@ -435,7 -447,7 +443,16 @@@
.withChunkCache(ChunkCache.instance))
{
if (!sstable.loadSummary())
-- sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
++ {
++ try
++ {
++ sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL);
++ }
++ catch(IOException e)
++ {
++ throw new CorruptSSTableException(e, sstable.getFilename());
++ }
++ }
long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length();
int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size());
@@@ -448,20 -460,29 +465,29 @@@
}
public static SSTableReader open(Descriptor descriptor,
- Set<Component> components,
- CFMetaData metadata,
- boolean validate,
- boolean trackHotness) throws IOException
+ Set<Component> components,
+ TableMetadataRef metadata,
+ boolean validate,
- boolean trackHotness) throws IOException
++ boolean trackHotness)
{
// Minimum components without which we can't do anything
assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
// For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content
- assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
+ assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+
+ Map<MetadataType, MetadataComponent> sstableMetadata;
+ try
+ {
+ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
@@@ -511,16 -537,16 +542,8 @@@
}
}
-- public static void logOpenException(Descriptor descriptor, IOException e)
-- {
-- if (e instanceof FileNotFoundException)
-- logger.error("Missing sstable component in {}; skipped because of {}", descriptor, e.getMessage());
-- else
-- logger.error("Corrupt sstable {}; skipped", descriptor, e);
-- }
--
public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries,
- final CFMetaData metadata)
+ final TableMetadataRef metadata)
{
final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>();
@@@ -548,11 -574,12 +571,6 @@@
logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
return;
}
-- catch (IOException ex)
-- {
- FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
-- logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
-- return;
-- }
sstables.add(sstable);
}
};
http://git-wip-us.apache.org/repos/asf/cassandra/blob/326f3a7c/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/809f3b30
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/809f3b30
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/809f3b30
Branch: refs/heads/cassandra-3.11
Commit: 809f3b30e88b57f2c0dcf71db400710233120143
Parents: 26fedcd dfbe3fa
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Aug 28 15:39:49 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:39:49 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++++++++++++++---
.../io/sstable/format/SSTableReader.java | 19 +++++++++++++++--
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
4 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8fb8e2f,5ccd5cd..b0dbd60
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,5 +1,12 @@@
-3.0.15
+3.11.1
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
* Fix the merging of cells with different user type versions (CASSANDRA-13776)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
* Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[2/6] cassandra git commit: Dont throw IOExceptions in when opening
sstables
Posted by ma...@apache.org.
Dont throw IOExceptions in when opening sstables
Patch by marcuse; reviewed by Ariel Weisberg for CASSANDRA-13620
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/dfbe3fab
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/dfbe3fab
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/dfbe3fab
Branch: refs/heads/cassandra-3.11
Commit: dfbe3fabd266493e698c194ef90b4dfc7d62b030
Parents: 95f1b23
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 19 15:22:57 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:37:26 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++++++++++++++---
.../io/sstable/format/SSTableReader.java | 19 +++++++++++++++--
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
4 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ba35152..5ccd5cd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.15
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
* Fix the merging of cells with different user type versions (CASSANDRA-13776)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
* Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f720330..7251244 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -59,8 +59,10 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
@@ -691,7 +693,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
catch (IOException e)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(e, entry.getKey().filenameFor(Component.STATS)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, e);
continue;
}
@@ -718,9 +721,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
}
- catch (IOException e)
+ catch (CorruptSSTableException ex)
+ {
+ FileUtils.handleCorruptSSTable(ex);
+ logger.error("Corrupt sstable {}; skipping table", entry, ex);
+ continue;
+ }
+ catch (FSError ex)
+ {
+ FileUtils.handleFSError(ex);
+ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
+ continue;
+ }
+ catch (IOException ex)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
continue;
}
newSSTables.add(reader);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index cd41b5b..d56b3e7 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -446,7 +446,16 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+
+ Map<MetadataType, MetadataComponent> sstableMetadata;
+ try
+ {
+ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
@@ -488,6 +497,11 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstable;
}
+ catch (IOException e)
+ {
+ sstable.selfRef().release();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
catch (Throwable t)
{
sstable.selfRef().release();
@@ -534,6 +548,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
}
catch (IOException ex)
{
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
return;
}
@@ -1177,7 +1192,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
if (this.first.compareTo(this.last) > 0)
{
- throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
+ throw new CorruptSSTableException(new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)), getFilename());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index f97d9a9..08336a1 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -349,7 +349,7 @@ public class ScrubTest
SSTableReader.open(desc, cfs.metadata);
fail("SSTR validation should have caught the out-of-order rows");
}
- catch (IllegalStateException ise)
+ catch (CorruptSSTableException ise)
{ /* this is expected */ }
// open without validation for scrubbing
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[3/6] cassandra git commit: Dont throw IOExceptions in when opening
sstables
Posted by ma...@apache.org.
Dont throw IOExceptions in when opening sstables
Patch by marcuse; reviewed by Ariel Weisberg for CASSANDRA-13620
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/dfbe3fab
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/dfbe3fab
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/dfbe3fab
Branch: refs/heads/trunk
Commit: dfbe3fabd266493e698c194ef90b4dfc7d62b030
Parents: 95f1b23
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Jun 19 15:22:57 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:37:26 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++++++++++++++---
.../io/sstable/format/SSTableReader.java | 19 +++++++++++++++--
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
4 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ba35152..5ccd5cd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.15
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
* Fix the merging of cells with different user type versions (CASSANDRA-13776)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
* Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index f720330..7251244 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -59,8 +59,10 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
@@ -691,7 +693,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
catch (IOException e)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(e, entry.getKey().filenameFor(Component.STATS)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, e);
continue;
}
@@ -718,9 +721,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
reader = SSTableReader.open(newDescriptor, entry.getValue(), metadata);
}
- catch (IOException e)
+ catch (CorruptSSTableException ex)
+ {
+ FileUtils.handleCorruptSSTable(ex);
+ logger.error("Corrupt sstable {}; skipping table", entry, ex);
+ continue;
+ }
+ catch (FSError ex)
+ {
+ FileUtils.handleFSError(ex);
+ logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex);
+ continue;
+ }
+ catch (IOException ex)
{
- SSTableReader.logOpenException(entry.getKey(), e);
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
+ logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
continue;
}
newSSTables.add(reader);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index cd41b5b..d56b3e7 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -446,7 +446,16 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+
+ Map<MetadataType, MetadataComponent> sstableMetadata;
+ try
+ {
+ sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS));
+ }
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
@@ -488,6 +497,11 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return sstable;
}
+ catch (IOException e)
+ {
+ sstable.selfRef().release();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
catch (Throwable t)
{
sstable.selfRef().release();
@@ -534,6 +548,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
}
catch (IOException ex)
{
+ FileUtils.handleCorruptSSTable(new CorruptSSTableException(ex, entry.getKey().filenameFor(Component.DATA)));
logger.error("Cannot read sstable {}; other IO error, skipping table", entry, ex);
return;
}
@@ -1177,7 +1192,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
if (this.first.compareTo(this.last) > 0)
{
- throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
+ throw new CorruptSSTableException(new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)), getFilename());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/dfbe3fab/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index f97d9a9..08336a1 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -349,7 +349,7 @@ public class ScrubTest
SSTableReader.open(desc, cfs.metadata);
fail("SSTR validation should have caught the out-of-order rows");
}
- catch (IllegalStateException ise)
+ catch (CorruptSSTableException ise)
{ /* this is expected */ }
// open without validation for scrubbing
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[4/6] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Posted by ma...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/809f3b30
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/809f3b30
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/809f3b30
Branch: refs/heads/trunk
Commit: 809f3b30e88b57f2c0dcf71db400710233120143
Parents: 26fedcd dfbe3fa
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Aug 28 15:39:49 2017 +0200
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Aug 28 15:39:49 2017 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++++++++++++++---
.../io/sstable/format/SSTableReader.java | 19 +++++++++++++++--
.../unit/org/apache/cassandra/db/ScrubTest.java | 2 +-
4 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8fb8e2f,5ccd5cd..b0dbd60
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,5 +1,12 @@@
-3.0.15
+3.11.1
+ * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
+ * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
+ * Don't skip corrupted sstables on startup (CASSANDRA-13620)
* Fix the merging of cells with different user type versions (CASSANDRA-13776)
* Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
* Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/809f3b30/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org