You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2014/12/17 12:37:07 UTC
[1/2] cassandra git commit: Make sstablescrub check both the repaired
and unrepaired leveled manifests
Repository: cassandra
Updated Branches:
refs/heads/trunk e6c5982fa -> 417563a59
Make sstablescrub check both the repaired and unrepaired leveled manifests
Patch by marcuse; reviewed by cyeksigian for CASSANDRA-8432
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6f98c6c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6f98c6c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6f98c6c4
Branch: refs/heads/trunk
Commit: 6f98c6c4ed2e2b3d427cd10c2b2ada9c60b35acd
Parents: d5e5f98
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Dec 8 11:45:59 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Dec 17 12:31:40 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/StandaloneScrubber.java | 52 ++++++++++++++------
2 files changed, 38 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f98c6c4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 410d49a..9ec3585 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.3
+ * Make sstablescrub check leveled manifest again (CASSANDRA-8432)
* Check first/last keys in sstable when giving out positions (CASSANDRA-8458)
* Disable mmap on Windows (CASSANDRA-6993)
* Add missing ConsistencyLevels to cassandra-stress (CASSANDRA-8253)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f98c6c4/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 42799a5..2a9763b 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -21,6 +21,10 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.util.*;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -28,9 +32,11 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.compaction.Scrubber;
+import org.apache.cassandra.db.compaction.WrappingCompactionStrategy;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
@@ -95,14 +101,6 @@ public class StandaloneScrubber
}
System.out.println(String.format("Pre-scrub sstables snapshotted into snapshot %s", snapshotName));
- LeveledManifest manifest = null;
- // If leveled, load the manifest
- if (cfs.getCompactionStrategy() instanceof LeveledCompactionStrategy)
- {
- int maxSizeInMB = (int)((cfs.getCompactionStrategy().getMaxSSTableBytes()) / (1024L * 1024L));
- manifest = LeveledManifest.create(cfs, maxSizeInMB, sstables);
- }
-
if (!options.manifestCheckOnly)
{
for (SSTableReader sstable : sstables)
@@ -131,9 +129,8 @@ public class StandaloneScrubber
}
}
- // Check (and repair) manifest
- if (manifest != null)
- checkManifest(manifest);
+ // Check (and repair) manifests
+ checkManifest(cfs.getCompactionStrategy(), cfs, sstables);
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
@@ -147,11 +144,36 @@ public class StandaloneScrubber
}
}
- private static void checkManifest(LeveledManifest manifest)
+ private static void checkManifest(AbstractCompactionStrategy strategy, ColumnFamilyStore cfs, Collection<SSTableReader> sstables)
{
- System.out.println(String.format("Checking leveled manifest"));
- for (int i = 1; i <= manifest.getLevelCount(); ++i)
- manifest.repairOverlappingSSTables(i);
+ WrappingCompactionStrategy wrappingStrategy = (WrappingCompactionStrategy)strategy;
+ int maxSizeInMB = (int)((cfs.getCompactionStrategy().getMaxSSTableBytes()) / (1024L * 1024L));
+ if (wrappingStrategy.getWrappedStrategies().size() == 2 && wrappingStrategy.getWrappedStrategies().iterator().next() instanceof LeveledCompactionStrategy)
+ {
+ System.out.println("Checking leveled manifest");
+ Predicate<SSTableReader> repairedPredicate = new Predicate<SSTableReader>()
+ {
+ @Override
+ public boolean apply(SSTableReader sstable)
+ {
+ return sstable.isRepaired();
+ }
+ };
+
+ List<SSTableReader> repaired = Lists.newArrayList(Iterables.filter(sstables, repairedPredicate));
+ List<SSTableReader> unRepaired = Lists.newArrayList(Iterables.filter(sstables, Predicates.not(repairedPredicate)));
+
+ LeveledManifest repairedManifest = LeveledManifest.create(cfs, maxSizeInMB, repaired);
+ for (int i = 1; i < repairedManifest.getLevelCount(); i++)
+ {
+ repairedManifest.repairOverlappingSSTables(i);
+ }
+ LeveledManifest unRepairedManifest = LeveledManifest.create(cfs, maxSizeInMB, unRepaired);
+ for (int i = 1; i < unRepairedManifest.getLevelCount(); i++)
+ {
+ unRepairedManifest.repairOverlappingSSTables(i);
+ }
+ }
}
private static class Options
[2/2] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Conflicts:
src/java/org/apache/cassandra/tools/StandaloneScrubber.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/417563a5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/417563a5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/417563a5
Branch: refs/heads/trunk
Commit: 417563a59b9106aa1e225ef0e6412351bf4f2c9c
Parents: e6c5982 6f98c6c
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Dec 17 12:34:30 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Wed Dec 17 12:34:30 2014 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/StandaloneScrubber.java | 54 ++++++++++++++------
2 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/417563a5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 6330e4b,9ec3585..9cea513
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,46 -1,5 +1,47 @@@
+3.0
+ * Modernize schema tables (CASSANDRA-8261)
+ * Support for user-defined aggregation functions (CASSANDRA-8053)
+ * Fix NPE in SelectStatement with empty IN values (CASSANDRA-8419)
+ * Refactor SelectStatement, return IN results in natural order instead
+ of IN value list order (CASSANDRA-7981)
+ * Support UDTs, tuples, and collections in user-defined
+ functions (CASSANDRA-7563)
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer apis (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support pure user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063, 7813)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
+ * Improve concurrency of repair (CASSANDRA-6455, 8208)
+
+
2.1.3
+ * Make sstablescrub check leveled manifest again (CASSANDRA-8432)
* Check first/last keys in sstable when giving out positions (CASSANDRA-8458)
* Disable mmap on Windows (CASSANDRA-6993)
* Add missing ConsistencyLevels to cassandra-stress (CASSANDRA-8253)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/417563a5/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index b6e2bf8,2a9763b..80640d0
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@@ -21,9 -21,13 +21,12 @@@ package org.apache.cassandra.tools
import java.io.File;
import java.util.*;
- import org.apache.cassandra.io.sstable.format.SSTableReader;
+ import com.google.common.base.Predicate;
+ import com.google.common.base.Predicates;
+ import com.google.common.collect.Iterables;
+ import com.google.common.collect.Lists;
import org.apache.commons.cli.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
@@@ -31,6 -36,7 +35,8 @@@ import org.apache.cassandra.db.compacti
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.compaction.Scrubber;
+ import org.apache.cassandra.db.compaction.WrappingCompactionStrategy;
++import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;