You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2014/12/05 00:30:27 UTC
[1/6] cassandra git commit: Use live sstables in snapshot repair if
possible
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.0 4030088ec -> ceed3a20e
refs/heads/cassandra-2.1 587657d37 -> b7a0cd9e6
refs/heads/trunk f5866ca2b -> a7208383f
Use live sstables in snapshot repair if possible
patch by Jimmy Mårdell; reviewed by yukim for CASSANDRA-8312
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ceed3a20
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ceed3a20
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ceed3a20
Branch: refs/heads/cassandra-2.0
Commit: ceed3a20ef78b402a7a734e63d758aff105fa2de
Parents: 4030088
Author: Jimmy Mårdell <ya...@spotify.com>
Authored: Thu Dec 4 09:59:34 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:00:53 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 36 ++++++++++++++++++--
.../db/compaction/CompactionManager.java | 13 +++----
3 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dc3896d..79c2d81 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -30,6 +30,7 @@
* Fix totalDiskSpaceUsed calculation (CASSANDRA-8205)
* Add DC-aware sequential repair (CASSANDRA-8193)
* Improve JBOD disk utilization (CASSANDRA-7386)
+ * Use live sstables in snapshot repair if possible (CASSANDRA-8312)
2.0.11:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 6cdf9e9..b5c6c98 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1840,10 +1840,40 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
+ Map<Integer, SSTableReader> active = new HashMap<>();
+ for (SSTableReader sstable : data.getView().sstables)
+ active.put(sstable.descriptor.generation, sstable);
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
- for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
- readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
+ try
+ {
+ for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
+ {
+ // Try acquire reference to an active sstable instead of snapshot if it exists,
+ // to avoid opening new sstables. If it fails, use the snapshot reference instead.
+ SSTableReader sstable = active.get(entries.getKey().generation);
+ if (sstable == null || !sstable.acquireReference())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("using snapshot sstable " + entries.getKey());
+ sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner);
+ // This is technically not necessary since it's a snapshot but makes things easier
+ sstable.acquireReference();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("using active sstable " + entries.getKey());
+ }
+ readers.add(sstable);
+ }
+ }
+ catch (IOException | RuntimeException e)
+ {
+ // In case one of the snapshot sstables fails to open,
+ // we must release the references to the ones we opened so far
+ SSTableReader.releaseReferences(readers);
+ throw e;
+ }
return readers;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index d298e72..19dedb0 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -765,8 +765,8 @@ public class CompactionManager implements CompactionManagerMBean
sstables = cfs.getSnapshotSSTableReader(snapshotName);
// Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
- // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
- // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case
+ // this at a different time (that's the whole purpose of repair with snapshot). So instead we take the creation
+ // time of the snapshot, which should give us roughly the same time on each replica (roughly being in that case
// 'as good as in the non-snapshot' case)
gcBefore = cfs.gcBefore(cfs.getSnapshotCreationTime(snapshotName));
}
@@ -803,16 +803,11 @@ public class CompactionManager implements CompactionManagerMBean
finally
{
iter.close();
+ SSTableReader.releaseReferences(sstables);
if (isSnapshotValidation)
{
- for (SSTableReader sstable : sstables)
- FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(snapshotName);
}
- else
- {
- SSTableReader.releaseReferences(sstables);
- }
metrics.finishCompaction(ci);
}
@@ -956,7 +951,7 @@ public class CompactionManager implements CompactionManagerMBean
public void afterExecute(Runnable r, Throwable t)
{
DebuggableThreadPoolExecutor.maybeResetTraceSessionWrapper(r);
-
+
if (t == null)
t = DebuggableThreadPoolExecutor.extractThrowable(r);
[6/6] cassandra git commit: Merge branch 'cassandra-2.1' into trunk
Posted by yu...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Conflicts:
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a7208383
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a7208383
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a7208383
Branch: refs/heads/trunk
Commit: a7208383fba67fda025d354c66491c668887602a
Parents: f5866ca b7a0cd9
Author: Yuki Morishita <yu...@apache.org>
Authored: Thu Dec 4 17:30:09 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:30:09 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 34 ++++++++++++++++++--
2 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a7208383/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a7208383/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
[2/6] cassandra git commit: Use live sstables in snapshot repair if
possible
Posted by yu...@apache.org.
Use live sstables in snapshot repair if possible
patch by Jimmy Mårdell; reviewed by yukim for CASSANDRA-8312
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ceed3a20
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ceed3a20
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ceed3a20
Branch: refs/heads/cassandra-2.1
Commit: ceed3a20ef78b402a7a734e63d758aff105fa2de
Parents: 4030088
Author: Jimmy Mårdell <ya...@spotify.com>
Authored: Thu Dec 4 09:59:34 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:00:53 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 36 ++++++++++++++++++--
.../db/compaction/CompactionManager.java | 13 +++----
3 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dc3896d..79c2d81 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -30,6 +30,7 @@
* Fix totalDiskSpaceUsed calculation (CASSANDRA-8205)
* Add DC-aware sequential repair (CASSANDRA-8193)
* Improve JBOD disk utilization (CASSANDRA-7386)
+ * Use live sstables in snapshot repair if possible (CASSANDRA-8312)
2.0.11:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 6cdf9e9..b5c6c98 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1840,10 +1840,40 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
+ Map<Integer, SSTableReader> active = new HashMap<>();
+ for (SSTableReader sstable : data.getView().sstables)
+ active.put(sstable.descriptor.generation, sstable);
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
- for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
- readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
+ try
+ {
+ for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
+ {
+ // Try acquire reference to an active sstable instead of snapshot if it exists,
+ // to avoid opening new sstables. If it fails, use the snapshot reference instead.
+ SSTableReader sstable = active.get(entries.getKey().generation);
+ if (sstable == null || !sstable.acquireReference())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("using snapshot sstable " + entries.getKey());
+ sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner);
+ // This is technically not necessary since it's a snapshot but makes things easier
+ sstable.acquireReference();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("using active sstable " + entries.getKey());
+ }
+ readers.add(sstable);
+ }
+ }
+ catch (IOException | RuntimeException e)
+ {
+ // In case one of the snapshot sstables fails to open,
+ // we must release the references to the ones we opened so far
+ SSTableReader.releaseReferences(readers);
+ throw e;
+ }
return readers;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index d298e72..19dedb0 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -765,8 +765,8 @@ public class CompactionManager implements CompactionManagerMBean
sstables = cfs.getSnapshotSSTableReader(snapshotName);
// Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
- // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
- // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case
+ // this at a different time (that's the whole purpose of repair with snapshot). So instead we take the creation
+ // time of the snapshot, which should give us roughly the same time on each replica (roughly being in that case
// 'as good as in the non-snapshot' case)
gcBefore = cfs.gcBefore(cfs.getSnapshotCreationTime(snapshotName));
}
@@ -803,16 +803,11 @@ public class CompactionManager implements CompactionManagerMBean
finally
{
iter.close();
+ SSTableReader.releaseReferences(sstables);
if (isSnapshotValidation)
{
- for (SSTableReader sstable : sstables)
- FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(snapshotName);
}
- else
- {
- SSTableReader.releaseReferences(sstables);
- }
metrics.finishCompaction(ci);
}
@@ -956,7 +951,7 @@ public class CompactionManager implements CompactionManagerMBean
public void afterExecute(Runnable r, Throwable t)
{
DebuggableThreadPoolExecutor.maybeResetTraceSessionWrapper(r);
-
+
if (t == null)
t = DebuggableThreadPoolExecutor.extractThrowable(r);
[5/6] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by yu...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b7a0cd9e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b7a0cd9e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b7a0cd9e
Branch: refs/heads/cassandra-2.1
Commit: b7a0cd9e6037a0fb21a5fb64310c50cd39e35496
Parents: 587657d ceed3a2
Author: Yuki Morishita <yu...@apache.org>
Authored: Thu Dec 4 17:26:59 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:26:59 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 36 ++++++++++++++++++--
2 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7a0cd9e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 041c1e1,79c2d81..145347b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -33,35 -16,7 +33,36 @@@ Merged from 2.0
* Avoid overlap in L1 when L0 contains many nonoverlapping
sstables (CASSANDRA-8211)
* Improve PropertyFileSnitch logging (CASSANDRA-8183)
- * Abort liveRatio calculation if the memtable is flushed (CASSANDRA-8164)
+ * Add DC-aware sequential repair (CASSANDRA-8193)
++ * Use live sstables in snapshot repair if possible (CASSANDRA-8312)
+
+
+2.1.2
+ * (cqlsh) parse_for_table_meta errors out on queries with undefined
+ grammars (CASSANDRA-8262)
+ * (cqlsh) Fix SELECT ... TOKEN() function broken in C* 2.1.1 (CASSANDRA-8258)
+ * Fix Cassandra crash when running on JDK8 update 40 (CASSANDRA-8209)
+ * Optimize partitioner tokens (CASSANDRA-8230)
+ * Improve compaction of repaired/unrepaired sstables (CASSANDRA-8004)
+ * Make cache serializers pluggable (CASSANDRA-8096)
+ * Fix issues with CONTAINS (KEY) queries on secondary indexes
+ (CASSANDRA-8147)
+ * Fix read-rate tracking of sstables for some queries (CASSANDRA-8239)
+ * Fix default timestamp in QueryOptions (CASSANDRA-8246)
+ * Set socket timeout when reading remote version (CASSANDRA-8188)
+ * Refactor how we track live size (CASSANDRA-7852)
+ * Make sure unfinished compaction files are removed (CASSANDRA-8124)
+ * Fix shutdown when run as Windows service (CASSANDRA-8136)
+ * Fix DESCRIBE TABLE with custom indexes (CASSANDRA-8031)
+ * Fix race in RecoveryManagerTest (CASSANDRA-8176)
+ * Avoid IllegalArgumentException while sorting sstables in
+ IndexSummaryManager (CASSANDRA-8182)
+ * Shutdown JVM on file descriptor exhaustion (CASSANDRA-7579)
+ * Add 'die' policy for commit log and disk failure (CASSANDRA-7927)
+ * Fix installing as service on Windows (CASSANDRA-8115)
+ * Fix CREATE TABLE for CQL2 (CASSANDRA-8144)
+ * Avoid boxing in ColumnStats min/max trackers (CASSANDRA-8109)
+Merged from 2.0:
* Correctly handle non-text column names in cql3 (CASSANDRA-8178)
* Fix deletion for indexes on primary key columns (CASSANDRA-8206)
* Add 'nodetool statusgossip' (CASSANDRA-8125)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7a0cd9e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 0507973,b5c6c98..be89318
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -2165,32 -1838,42 +2165,62 @@@ public class ColumnFamilyStore implemen
}
}
+ private void writeSnapshotManifest(final JSONArray filesJSONArr, final String snapshotName)
+ {
+ final File manifestFile = directories.getSnapshotManifestFile(snapshotName);
+ final JSONObject manifestJSON = new JSONObject();
+ manifestJSON.put("files", filesJSONArr);
+
+ try
+ {
+ if (!manifestFile.getParentFile().exists())
+ manifestFile.getParentFile().mkdirs();
+ PrintStream out = new PrintStream(manifestFile);
+ out.println(manifestJSON.toJSONString());
+ out.close();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, manifestFile);
+ }
+ }
+
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
+ Map<Integer, SSTableReader> active = new HashMap<>();
+ for (SSTableReader sstable : data.getView().sstables)
+ active.put(sstable.descriptor.generation, sstable);
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
- for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
- readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
+ try
+ {
+ for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
+ {
+ // Try acquire reference to an active sstable instead of snapshot if it exists,
+ // to avoid opening new sstables. If it fails, use the snapshot reference instead.
+ SSTableReader sstable = active.get(entries.getKey().generation);
+ if (sstable == null || !sstable.acquireReference())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("using snapshot sstable " + entries.getKey());
+ sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner);
+ // This is technically not necessary since it's a snapshot but makes things easier
+ sstable.acquireReference();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("using active sstable " + entries.getKey());
+ }
+ readers.add(sstable);
+ }
+ }
+ catch (IOException | RuntimeException e)
+ {
+ // In case one of the snapshot sstables fails to open,
+ // we must release the references to the ones we opened so far
+ SSTableReader.releaseReferences(readers);
+ throw e;
+ }
return readers;
}
[4/6] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Posted by yu...@apache.org.
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b7a0cd9e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b7a0cd9e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b7a0cd9e
Branch: refs/heads/trunk
Commit: b7a0cd9e6037a0fb21a5fb64310c50cd39e35496
Parents: 587657d ceed3a2
Author: Yuki Morishita <yu...@apache.org>
Authored: Thu Dec 4 17:26:59 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:26:59 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 36 ++++++++++++++++++--
2 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7a0cd9e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 041c1e1,79c2d81..145347b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -33,35 -16,7 +33,36 @@@ Merged from 2.0
* Avoid overlap in L1 when L0 contains many nonoverlapping
sstables (CASSANDRA-8211)
* Improve PropertyFileSnitch logging (CASSANDRA-8183)
- * Abort liveRatio calculation if the memtable is flushed (CASSANDRA-8164)
+ * Add DC-aware sequential repair (CASSANDRA-8193)
++ * Use live sstables in snapshot repair if possible (CASSANDRA-8312)
+
+
+2.1.2
+ * (cqlsh) parse_for_table_meta errors out on queries with undefined
+ grammars (CASSANDRA-8262)
+ * (cqlsh) Fix SELECT ... TOKEN() function broken in C* 2.1.1 (CASSANDRA-8258)
+ * Fix Cassandra crash when running on JDK8 update 40 (CASSANDRA-8209)
+ * Optimize partitioner tokens (CASSANDRA-8230)
+ * Improve compaction of repaired/unrepaired sstables (CASSANDRA-8004)
+ * Make cache serializers pluggable (CASSANDRA-8096)
+ * Fix issues with CONTAINS (KEY) queries on secondary indexes
+ (CASSANDRA-8147)
+ * Fix read-rate tracking of sstables for some queries (CASSANDRA-8239)
+ * Fix default timestamp in QueryOptions (CASSANDRA-8246)
+ * Set socket timeout when reading remote version (CASSANDRA-8188)
+ * Refactor how we track live size (CASSANDRA-7852)
+ * Make sure unfinished compaction files are removed (CASSANDRA-8124)
+ * Fix shutdown when run as Windows service (CASSANDRA-8136)
+ * Fix DESCRIBE TABLE with custom indexes (CASSANDRA-8031)
+ * Fix race in RecoveryManagerTest (CASSANDRA-8176)
+ * Avoid IllegalArgumentException while sorting sstables in
+ IndexSummaryManager (CASSANDRA-8182)
+ * Shutdown JVM on file descriptor exhaustion (CASSANDRA-7579)
+ * Add 'die' policy for commit log and disk failure (CASSANDRA-7927)
+ * Fix installing as service on Windows (CASSANDRA-8115)
+ * Fix CREATE TABLE for CQL2 (CASSANDRA-8144)
+ * Avoid boxing in ColumnStats min/max trackers (CASSANDRA-8109)
+Merged from 2.0:
* Correctly handle non-text column names in cql3 (CASSANDRA-8178)
* Fix deletion for indexes on primary key columns (CASSANDRA-8206)
* Add 'nodetool statusgossip' (CASSANDRA-8125)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b7a0cd9e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 0507973,b5c6c98..be89318
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -2165,32 -1838,42 +2165,62 @@@ public class ColumnFamilyStore implemen
}
}
+ private void writeSnapshotManifest(final JSONArray filesJSONArr, final String snapshotName)
+ {
+ final File manifestFile = directories.getSnapshotManifestFile(snapshotName);
+ final JSONObject manifestJSON = new JSONObject();
+ manifestJSON.put("files", filesJSONArr);
+
+ try
+ {
+ if (!manifestFile.getParentFile().exists())
+ manifestFile.getParentFile().mkdirs();
+ PrintStream out = new PrintStream(manifestFile);
+ out.println(manifestJSON.toJSONString());
+ out.close();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, manifestFile);
+ }
+ }
+
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
+ Map<Integer, SSTableReader> active = new HashMap<>();
+ for (SSTableReader sstable : data.getView().sstables)
+ active.put(sstable.descriptor.generation, sstable);
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
- for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
- readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
+ try
+ {
+ for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
+ {
+ // Try acquire reference to an active sstable instead of snapshot if it exists,
+ // to avoid opening new sstables. If it fails, use the snapshot reference instead.
+ SSTableReader sstable = active.get(entries.getKey().generation);
+ if (sstable == null || !sstable.acquireReference())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("using snapshot sstable " + entries.getKey());
+ sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner);
+ // This is technically not necessary since it's a snapshot but makes things easier
+ sstable.acquireReference();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("using active sstable " + entries.getKey());
+ }
+ readers.add(sstable);
+ }
+ }
+ catch (IOException | RuntimeException e)
+ {
+ // In case one of the snapshot sstables fails to open,
+ // we must release the references to the ones we opened so far
+ SSTableReader.releaseReferences(readers);
+ throw e;
+ }
return readers;
}
[3/6] cassandra git commit: Use live sstables in snapshot repair if
possible
Posted by yu...@apache.org.
Use live sstables in snapshot repair if possible
patch by Jimmy Mårdell; reviewed by yukim for CASSANDRA-8312
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ceed3a20
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ceed3a20
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ceed3a20
Branch: refs/heads/trunk
Commit: ceed3a20ef78b402a7a734e63d758aff105fa2de
Parents: 4030088
Author: Jimmy Mårdell <ya...@spotify.com>
Authored: Thu Dec 4 09:59:34 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Thu Dec 4 17:00:53 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 36 ++++++++++++++++++--
.../db/compaction/CompactionManager.java | 13 +++----
3 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dc3896d..79c2d81 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -30,6 +30,7 @@
* Fix totalDiskSpaceUsed calculation (CASSANDRA-8205)
* Add DC-aware sequential repair (CASSANDRA-8193)
* Improve JBOD disk utilization (CASSANDRA-7386)
+ * Use live sstables in snapshot repair if possible (CASSANDRA-8312)
2.0.11:
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 6cdf9e9..b5c6c98 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -1840,10 +1840,40 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
+ Map<Integer, SSTableReader> active = new HashMap<>();
+ for (SSTableReader sstable : data.getView().sstables)
+ active.put(sstable.descriptor.generation, sstable);
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
- for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
- readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
+ try
+ {
+ for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
+ {
+ // Try acquire reference to an active sstable instead of snapshot if it exists,
+ // to avoid opening new sstables. If it fails, use the snapshot reference instead.
+ SSTableReader sstable = active.get(entries.getKey().generation);
+ if (sstable == null || !sstable.acquireReference())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("using snapshot sstable " + entries.getKey());
+ sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner);
+ // This is technically not necessary since it's a snapshot but makes things easier
+ sstable.acquireReference();
+ }
+ else if (logger.isDebugEnabled())
+ {
+ logger.debug("using active sstable " + entries.getKey());
+ }
+ readers.add(sstable);
+ }
+ }
+ catch (IOException | RuntimeException e)
+ {
+ // In case one of the snapshot sstables fails to open,
+ // we must release the references to the ones we opened so far
+ SSTableReader.releaseReferences(readers);
+ throw e;
+ }
return readers;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ceed3a20/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index d298e72..19dedb0 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -765,8 +765,8 @@ public class CompactionManager implements CompactionManagerMBean
sstables = cfs.getSnapshotSSTableReader(snapshotName);
// Computing gcbefore based on the current time wouldn't be very good because we know each replica will execute
- // this at a different time (that's the whole purpose of repair with snaphsot). So instead we take the creation
- // time of the snapshot, which should give us roughtly the same time on each replica (roughtly being in that case
+ // this at a different time (that's the whole purpose of repair with snapshot). So instead we take the creation
+ // time of the snapshot, which should give us roughly the same time on each replica (roughly being in that case
// 'as good as in the non-snapshot' case)
gcBefore = cfs.gcBefore(cfs.getSnapshotCreationTime(snapshotName));
}
@@ -803,16 +803,11 @@ public class CompactionManager implements CompactionManagerMBean
finally
{
iter.close();
+ SSTableReader.releaseReferences(sstables);
if (isSnapshotValidation)
{
- for (SSTableReader sstable : sstables)
- FileUtils.closeQuietly(sstable);
cfs.clearSnapshot(snapshotName);
}
- else
- {
- SSTableReader.releaseReferences(sstables);
- }
metrics.finishCompaction(ci);
}
@@ -956,7 +951,7 @@ public class CompactionManager implements CompactionManagerMBean
public void afterExecute(Runnable r, Throwable t)
{
DebuggableThreadPoolExecutor.maybeResetTraceSessionWrapper(r);
-
+
if (t == null)
t = DebuggableThreadPoolExecutor.extractThrowable(r);