You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/05/07 23:53:47 UTC
[1/3] git commit: Add snapshot "manifest" describing files included
patch by Sankalp Kohli; reviewed by jbellis for CASSANDRA-6326
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 311c27642 -> ce7bf5e99
refs/heads/trunk c37cceef5 -> d6eb9e8bd
Add snapshot "manifest" describing files included
patch by Sankalp Kohli; reviewed by jbellis for CASSANDRA-6326
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ce7bf5e9
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ce7bf5e9
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ce7bf5e9
Branch: refs/heads/cassandra-2.1
Commit: ce7bf5e99405ce08dbd2b4955fd76582c27db403
Parents: 311c276
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed May 7 16:53:16 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed May 7 16:53:32 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 23 +++++++++++++++++++-
.../org/apache/cassandra/db/Directories.java | 5 +++++
.../apache/cassandra/io/sstable/Descriptor.java | 13 +++++++++++
4 files changed, 41 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5ecd19d..fc5786b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.0-rc1
+ * Add snapshot "manifest" describing files included (CASSANDRA-6326)
* Parallel streaming for sstableloader (CASSANDRA-3668)
* Fix bugs in supercolumns handling (CASSANDRA-7138)
* Fix ClassClassException on composite dense tables (CASSANDRA-7112)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1fdcb73..c5afb25 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -35,6 +35,8 @@ import com.google.common.collect.*;
import com.google.common.util.concurrent.*;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.io.FSWriteError;
+import org.json.simple.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2139,7 +2141,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (ColumnFamilyStore cfs : concatWithIndexes())
{
DataTracker.View currentView = cfs.markCurrentViewReferenced();
-
+ final JSONArray filesJSONArr = new JSONArray();
try
{
for (SSTableReader ssTable : currentView.sstables)
@@ -2151,9 +2153,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName);
ssTable.createLinks(snapshotDirectory.getPath()); // hard links
+ filesJSONArr.add(ssTable.descriptor.relativeFilenameFor(Component.DATA));
if (logger.isDebugEnabled())
logger.debug("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
}
+
+ writeSnapshotManifest(filesJSONArr, snapshotName);
}
finally
{
@@ -2162,6 +2167,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ private void writeSnapshotManifest(final JSONArray filesJSONArr, final String snapshotName)
+ {
+ final File manifestFile = directories.getSnapshotManifestFile(snapshotName);
+ final JSONObject manifestJSON = new JSONObject();
+ manifestJSON.put("files", filesJSONArr);
+
+ try
+ {
+ org.apache.commons.io.FileUtils.writeStringToFile(manifestFile, manifestJSON.toJSONString());
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, manifestFile);
+ }
+ }
+
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 1350be2..a146855 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -358,6 +358,11 @@ public class Directories
return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
}
+ public File getSnapshotManifestFile(String snapshotName)
+ {
+ return new File(getDirectoryForCompactedSSTables(), join(SNAPSHOT_SUBDIR, snapshotName, "manifest.json"));
+ }
+
public static File getBackupsDirectory(Descriptor desc)
{
return getOrCreate(desc.directory, BACKUPS_SUBDIR);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/io/sstable/Descriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index b42abf4..4415db4 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -182,12 +182,25 @@ public class Descriptor
{
StringBuilder buff = new StringBuilder();
buff.append(directory).append(File.separatorChar);
+ appendFileName(buff);
+ return buff.toString();
+ }
+
+ private void appendFileName(StringBuilder buff)
+ {
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
if (type.isTemporary)
buff.append(type.marker).append(separator);
buff.append(version).append(separator);
buff.append(generation);
+ }
+
+ public String relativeFilenameFor(Component component)
+ {
+ final StringBuilder buff = new StringBuilder();
+ appendFileName(buff);
+ buff.append(separator).append(component.name());
return buff.toString();
}
[2/3] git commit: Add snapshot "manifest" describing files included
patch by Sankalp Kohli; reviewed by jbellis for CASSANDRA-6326
Posted by jb...@apache.org.
Add snapshot "manifest" describing files included
patch by Sankalp Kohli; reviewed by jbellis for CASSANDRA-6326
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ce7bf5e9
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ce7bf5e9
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ce7bf5e9
Branch: refs/heads/trunk
Commit: ce7bf5e99405ce08dbd2b4955fd76582c27db403
Parents: 311c276
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed May 7 16:53:16 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed May 7 16:53:32 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 23 +++++++++++++++++++-
.../org/apache/cassandra/db/Directories.java | 5 +++++
.../apache/cassandra/io/sstable/Descriptor.java | 13 +++++++++++
4 files changed, 41 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5ecd19d..fc5786b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.0-rc1
+ * Add snapshot "manifest" describing files included (CASSANDRA-6326)
* Parallel streaming for sstableloader (CASSANDRA-3668)
* Fix bugs in supercolumns handling (CASSANDRA-7138)
* Fix ClassClassException on composite dense tables (CASSANDRA-7112)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1fdcb73..c5afb25 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -35,6 +35,8 @@ import com.google.common.collect.*;
import com.google.common.util.concurrent.*;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.io.FSWriteError;
+import org.json.simple.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2139,7 +2141,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (ColumnFamilyStore cfs : concatWithIndexes())
{
DataTracker.View currentView = cfs.markCurrentViewReferenced();
-
+ final JSONArray filesJSONArr = new JSONArray();
try
{
for (SSTableReader ssTable : currentView.sstables)
@@ -2151,9 +2153,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName);
ssTable.createLinks(snapshotDirectory.getPath()); // hard links
+ filesJSONArr.add(ssTable.descriptor.relativeFilenameFor(Component.DATA));
if (logger.isDebugEnabled())
logger.debug("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
}
+
+ writeSnapshotManifest(filesJSONArr, snapshotName);
}
finally
{
@@ -2162,6 +2167,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
}
+ private void writeSnapshotManifest(final JSONArray filesJSONArr, final String snapshotName)
+ {
+ final File manifestFile = directories.getSnapshotManifestFile(snapshotName);
+ final JSONObject manifestJSON = new JSONObject();
+ manifestJSON.put("files", filesJSONArr);
+
+ try
+ {
+ org.apache.commons.io.FileUtils.writeStringToFile(manifestFile, manifestJSON.toJSONString());
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, manifestFile);
+ }
+ }
+
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 1350be2..a146855 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -358,6 +358,11 @@ public class Directories
return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
}
+ public File getSnapshotManifestFile(String snapshotName)
+ {
+ return new File(getDirectoryForCompactedSSTables(), join(SNAPSHOT_SUBDIR, snapshotName, "manifest.json"));
+ }
+
public static File getBackupsDirectory(Descriptor desc)
{
return getOrCreate(desc.directory, BACKUPS_SUBDIR);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce7bf5e9/src/java/org/apache/cassandra/io/sstable/Descriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index b42abf4..4415db4 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -182,12 +182,25 @@ public class Descriptor
{
StringBuilder buff = new StringBuilder();
buff.append(directory).append(File.separatorChar);
+ appendFileName(buff);
+ return buff.toString();
+ }
+
+ private void appendFileName(StringBuilder buff)
+ {
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
if (type.isTemporary)
buff.append(type.marker).append(separator);
buff.append(version).append(separator);
buff.append(generation);
+ }
+
+ public String relativeFilenameFor(Component component)
+ {
+ final StringBuilder buff = new StringBuilder();
+ appendFileName(buff);
+ buff.append(separator).append(component.name());
return buff.toString();
}
[3/3] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d6eb9e8b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d6eb9e8b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d6eb9e8b
Branch: refs/heads/trunk
Commit: d6eb9e8bd4dd523bae757840042d3194456e5748
Parents: c37ccee ce7bf5e
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed May 7 16:53:40 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed May 7 16:53:40 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 23 +++++++++++++++++++-
.../org/apache/cassandra/db/Directories.java | 5 +++++
.../apache/cassandra/io/sstable/Descriptor.java | 13 +++++++++++
4 files changed, 41 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/d6eb9e8b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 067632f,fc5786b..0f7012c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,12 -1,5 +1,13 @@@
+3.0
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7208)
+
2.1.0-rc1
+ * Add snapshot "manifest" describing files included (CASSANDRA-6326)
* Parallel streaming for sstableloader (CASSANDRA-3668)
* Fix bugs in supercolumns handling (CASSANDRA-7138)
* Fix ClassClassException on composite dense tables (CASSANDRA-7112)