You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/01/29 10:27:31 UTC
[1/2] cassandra git commit: Stop compactions before exiting offline
tools
Repository: cassandra
Updated Branches:
refs/heads/trunk 48a7e645d -> 25b18b758
Stop compactions before exiting offline tools
Patch by marcuse; reviewed by yukim for CASSANDRA-8623
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3ad0d3a8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3ad0d3a8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3ad0d3a8
Branch: refs/heads/trunk
Commit: 3ad0d3a8f2ff4a2f5207b47620b52c192f4e5d82
Parents: 2bff135
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jan 29 10:20:09 2015 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jan 29 10:21:39 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 46 +++++++++++++++++++-
.../cassandra/tools/StandaloneScrubber.java | 4 +-
.../cassandra/tools/StandaloneSplitter.java | 4 +-
.../cassandra/tools/StandaloneUpgrader.java | 4 +-
5 files changed, 55 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3ad0d3a8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d273350..fce4898 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.3
+ * Stop compactions before exiting offline tools (CASSANDRA-8623)
* Update tools/stress/README.txt to match current behaviour (CASSANDRA-7933)
* Fix schema from Thrift conversion with empty metadata (CASSANDRA-8695)
* Safer Resource Management (CASSANDRA-7705)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3ad0d3a8/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index f59938f..68313a3 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -50,6 +50,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,9 +179,13 @@ public class CompactionManager implements CompactionManagerMBean
cfs.name,
cfs.getCompactionStrategy().getName());
List<Future<?>> futures = new ArrayList<Future<?>>();
-
// we must schedule it at least once, otherwise compaction will stop for a CF until next flush
do {
+ if (executor.isShutdown())
+ {
+ logger.info("Executor has shut down, not submitting background task");
+ return Collections.emptyList();
+ }
compactingCF.add(cfs);
futures.add(executor.submit(new BackgroundCompactionTask(cfs)));
// if we have room for more compactions, then fill up executor
@@ -197,6 +202,12 @@ public class CompactionManager implements CompactionManagerMBean
return false;
}
+ public void finishCompactionsAndShutdown(long timeout, TimeUnit unit) throws InterruptedException
+ {
+ executor.shutdown();
+ executor.awaitTermination(timeout, unit);
+ }
+
// the actual sstables to compact are not determined until we run the BCT; that way, if new sstables
// are created between task submission and execution, we execute against the most up-to-date information
class BackgroundCompactionTask implements Runnable
@@ -256,6 +267,12 @@ public class CompactionManager implements CompactionManagerMBean
for (final SSTableReader sstable : sstables)
{
+ if (executor.isShutdown())
+ {
+ logger.info("Executor has shut down, not submitting task");
+ return AllSSTableOpStatus.ABORTED;
+ }
+
futures.add(executor.submit(new Callable<Object>()
{
@Override
@@ -394,6 +411,12 @@ public class CompactionManager implements CompactionManagerMBean
performAnticompaction(cfs, ranges, sstables, repairedAt);
}
};
+ if (executor.isShutdown())
+ {
+ logger.info("Compaction executor has shut down, not submitting anticompaction");
+ return Futures.immediateCancelledFuture();
+ }
+
return executor.submit(runnable);
}
@@ -489,6 +512,11 @@ public class CompactionManager implements CompactionManagerMBean
task.execute(metrics);
}
};
+ if (executor.isShutdown())
+ {
+ logger.info("Compaction executor has shut down, not submitting task");
+ return Collections.emptyList();
+ }
futures.add(executor.submit(runnable));
}
return futures;
@@ -554,6 +582,12 @@ public class CompactionManager implements CompactionManagerMBean
}
}
};
+ if (executor.isShutdown())
+ {
+ logger.info("Compaction executor has shut down, not submitting task");
+ return Futures.immediateCancelledFuture();
+ }
+
return executor.submit(runnable);
}
@@ -1090,6 +1124,11 @@ public class CompactionManager implements CompactionManagerMBean
}
}
};
+ if (executor.isShutdown())
+ {
+ logger.info("Compaction executor has shut down, not submitting index build");
+ return null;
+ }
return executor.submit(runnable);
}
@@ -1123,6 +1162,11 @@ public class CompactionManager implements CompactionManagerMBean
}
}
};
+ if (executor.isShutdown())
+ {
+ logger.info("Executor has shut down, not submitting background task");
+ Futures.immediateCancelledFuture();
+ }
return executor.submit(runnable);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3ad0d3a8/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 63a3727..1bc2674 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
@@ -33,6 +34,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.compaction.Scrubber;
@@ -131,7 +133,7 @@ public class StandaloneScrubber
// Check (and repair) manifests
checkManifest(cfs.getCompactionStrategy(), cfs, sstables);
-
+ CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3ad0d3a8/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index 5ed1543..242b1c0 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.*;
@@ -28,6 +29,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.SSTableSplitter;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -108,7 +110,6 @@ public class StandaloneSplitter
// Do not load sstables since they might be broken
Keyspace keyspace = Keyspace.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
@@ -159,6 +160,7 @@ public class StandaloneSplitter
e.printStackTrace(System.err);
}
}
+ CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3ad0d3a8/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 92b6445..8fa5b60 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.tools;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.*;
@@ -27,6 +28,7 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.Upgrader;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -107,7 +109,7 @@ public class StandaloneUpgrader
e.printStackTrace(System.err);
}
}
-
+ CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0);
}
[2/2] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ma...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/25b18b75
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/25b18b75
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/25b18b75
Branch: refs/heads/trunk
Commit: 25b18b75839e148638c716a0311b56347b7aa9b2
Parents: 48a7e64 3ad0d3a
Author: Marcus Eriksson <ma...@apache.org>
Authored: Thu Jan 29 10:26:23 2015 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Thu Jan 29 10:26:23 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../db/compaction/CompactionManager.java | 46 +++++++++++++++++++-
.../cassandra/tools/StandaloneScrubber.java | 4 +-
.../cassandra/tools/StandaloneSplitter.java | 4 +-
.../cassandra/tools/StandaloneUpgrader.java | 4 +-
5 files changed, 55 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/25b18b75/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ab4621d,fce4898..157550a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,61 -1,5 +1,62 @@@
+3.0
+ * Upgrade Metrics library and remove depricated metrics (CASSANDRA-5657)
+ * Serializing Row cache alternative, fully off heap (CASSANDRA-7438)
+ * Duplicate rows returned when in clause has repeated values (CASSANDRA-6707)
+ * Make CassandraException unchecked, extend RuntimeException (CASSANDRA-8560)
+ * Support direct buffer decompression for reads (CASSANDRA-8464)
+ * DirectByteBuffer compatible LZ4 methods (CASSANDRA-7039)
+ * Add role based access control (CASSANDRA-7653)
+ * Group sstables for anticompaction correctly (CASSANDRA-8578)
+ * Add ReadFailureException to native protocol, respond
+ immediately when replicas encounter errors while handling
+ a read request (CASSANDRA-7886)
+ * Switch CommitLogSegment from RandomAccessFile to nio (CASSANDRA-8308)
+ * Allow mixing token and partition key restrictions (CASSANDRA-7016)
+ * Support index key/value entries on map collections (CASSANDRA-8473)
+ * Modernize schema tables (CASSANDRA-8261)
+ * Support for user-defined aggregation functions (CASSANDRA-8053)
+ * Fix NPE in SelectStatement with empty IN values (CASSANDRA-8419)
+ * Refactor SelectStatement, return IN results in natural order instead
+ of IN value list order and ignore duplicate values in partition key IN restrictions (CASSANDRA-7981)
+ * Support UDTs, tuples, and collections in user-defined
+ functions (CASSANDRA-7563)
+ * Fix aggregate fn results on empty selection, result column name,
+ and cqlsh parsing (CASSANDRA-8229)
+ * Mark sstables as repaired after full repair (CASSANDRA-7586)
+ * Extend Descriptor to include a format value and refactor reader/writer
+ APIs (CASSANDRA-7443)
+ * Integrate JMH for microbenchmarks (CASSANDRA-8151)
+ * Keep sstable levels when bootstrapping (CASSANDRA-7460)
+ * Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
+ * Support for aggregation functions (CASSANDRA-4914)
+ * Remove cassandra-cli (CASSANDRA-7920)
+ * Accept dollar quoted strings in CQL (CASSANDRA-7769)
+ * Make assassinate a first class command (CASSANDRA-7935)
+ * Support IN clause on any clustering column (CASSANDRA-4762)
+ * Improve compaction logging (CASSANDRA-7818)
+ * Remove YamlFileNetworkTopologySnitch (CASSANDRA-7917)
+ * Do anticompaction in groups (CASSANDRA-6851)
+ * Support user-defined functions (CASSANDRA-7395, 7526, 7562, 7740, 7781, 7929,
+ 7924, 7812, 8063, 7813, 7708)
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+ * Fail on very large batch sizes (CASSANDRA-8011)
+ * Improve concurrency of repair (CASSANDRA-6455, 8208)
+
+
2.1.3
+ * Stop compactions before exiting offline tools (CASSANDRA-8623)
* Update tools/stress/README.txt to match current behaviour (CASSANDRA-7933)
* Fix schema from Thrift conversion with empty metadata (CASSANDRA-8695)
* Safer Resource Management (CASSANDRA-7705)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/25b18b75/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index a1672a2,68313a3..8b96c3a
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@@ -50,10 -50,8 +50,11 @@@ import com.google.common.collect.Lists
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;
+ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/25b18b75/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/25b18b75/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index 28f9dd7,242b1c0..0504491
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@@ -19,12 -19,13 +19,13 @@@
package org.apache.cassandra.tools;
import java.io.File;
-import java.io.IOException;
import java.util.*;
+ import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
@@@ -108,10 -110,9 +110,9 @@@ public class StandaloneSplitte
// Do not load sstables since they might be broken
Keyspace keyspace = Keyspace.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
-
String snapshotName = "pre-split-" + System.currentTimeMillis();
- List<SSTableReader> sstables = new ArrayList<SSTableReader>();
+ List<SSTableReader> sstables = new ArrayList<>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet())
{
try
http://git-wip-us.apache.org/repos/asf/cassandra/blob/25b18b75/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 06cc98e,8fa5b60..98eaaf7
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@@ -17,9 -17,10 +17,10 @@@
*/
package org.apache.cassandra.tools;
-import java.io.IOException;
import java.util.*;
+ import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;