You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2011/03/07 18:30:08 UTC
svn commit: r1078858 - in /cassandra/branches/cassandra-0.7: ./ conf/
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/io/sstable/ test/conf/
test/unit/org/apache/cassandra/db/
Author: jbellis
Date: Mon Mar 7 17:30:07 2011
New Revision: 1078858
URL: http://svn.apache.org/viewvc?rev=1078858&view=rev
Log:
add incremental_backups option
patch by Yang Yang and jbellis for CASSANDRA-1872
Modified:
cassandra/branches/cassandra-0.7/CHANGES.txt
cassandra/branches/cassandra-0.7/conf/cassandra.yaml
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/Config.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/branches/cassandra-0.7/test/conf/cassandra.yaml
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Modified: cassandra/branches/cassandra-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7/CHANGES.txt Mon Mar 7 17:30:07 2011
@@ -8,6 +8,8 @@
* avoid exporting an un-requested row in sstable2json, when exporting
a key that does not exist (CASSANDRA-2168)
* track and migrate cached pages during compaction (CASSANDRA-1902)
+ * add incremental_backups option (CASSANDRA-1872)
+
0.7.3
* Keep endpoint state until aVeryLongTime (CASSANDRA-2115)
Modified: cassandra/branches/cassandra-0.7/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/conf/cassandra.yaml?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/conf/cassandra.yaml (original)
+++ cassandra/branches/cassandra-0.7/conf/cassandra.yaml Mon Mar 7 17:30:07 2011
@@ -212,6 +212,12 @@ thrift_framed_transport_size_in_mb: 15
# internal thrift overhead.
thrift_max_message_length_in_mb: 16
+# Set to true to have Cassandra create a hard link to each sstable
+# flushed or streamed locally in a backups/ subdirectory of the
+# Keyspace data. Removing these links is the operator's
+# responsibility.
+incremental_backups: false
+
# Whether or not to take a snapshot before each compaction. Be
# careful using this option, since Cassandra won't clean up the
# snapshots for you. Mostly useful if you're paranoid when there
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/Config.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/Config.java?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/Config.java Mon Mar 7 17:30:07 2011
@@ -115,6 +115,8 @@ public class Config
// TODO: remove in 0.8
public Boolean enable_page_cache_migration = false;
+ public boolean incremental_backups = false;
+
public static enum CommitLogSync {
periodic,
batch
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Mon Mar 7 17:30:07 2011
@@ -37,7 +37,6 @@ import org.apache.cassandra.config.Confi
import org.apache.cassandra.db.ColumnFamilyType;
import org.apache.cassandra.db.DefsTable;
import org.apache.cassandra.db.Table;
-import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.dht.IPartitioner;
@@ -1218,4 +1217,9 @@ public class DatabaseDescriptor
{
return conf.enable_page_cache_migration;
}
+
+ public static boolean incrementalBackupsEnabled()
+ {
+ return conf.incremental_backups;
+ }
}
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Mar 7 17:30:07 2011
@@ -919,6 +919,21 @@ public class ColumnFamilyStore implement
{
assert sstable.getColumnFamilyName().equals(columnFamily);
ssTables.add(Arrays.asList(sstable));
+ if (DatabaseDescriptor.incrementalBackupsEnabled())
+ {
+ File keyspaceDir = new File(sstable.getFilename()).getParentFile();
+ File backupsDir = new File(keyspaceDir, "backups");
+ try
+ {
+ if (!backupsDir.exists() && !backupsDir.mkdirs())
+ throw new IOException("Unable to create " + backupsDir);
+ sstable.createLinks(backupsDir.getCanonicalPath());
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+ }
CompactionManager.instance.submitMinorIfNeeded(this);
}
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTable.java Mon Mar 7 17:30:07 2011
@@ -183,7 +183,7 @@ public abstract class SSTable
/**
* Discovers existing components for the descriptor. Slow: only intended for use outside the critical path.
*/
- static Set<Component> componentsFor(final Descriptor desc) throws IOException
+ static Set<Component> componentsFor(final Descriptor desc)
{
final Set<Component> components = new HashSet<Component>();
desc.directory.list(new FilenameFilter()
Modified: cassandra/branches/cassandra-0.7/test/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/conf/cassandra.yaml?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/conf/cassandra.yaml (original)
+++ cassandra/branches/cassandra-0.7/test/conf/cassandra.yaml Mon Mar 7 17:30:07 2011
@@ -24,6 +24,7 @@ endpoint_snitch: org.apache.cassandra.lo
dynamic_snitch: true
request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
request_scheduler_id: keyspace
+incremental_backups: true
keyspaces:
- name: Keyspace1
replica_placement_strategy: org.apache.cassandra.locator.SimpleStrategy
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=1078858&r1=1078857&r2=1078858&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java Mon Mar 7 17:30:07 2011
@@ -18,6 +18,7 @@
*/
package org.apache.cassandra.db;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -39,6 +40,7 @@ import org.apache.cassandra.CleanupHelpe
import org.apache.cassandra.Util;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.ConfigurationException;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.dht.IPartitioner;
@@ -52,6 +54,7 @@ import org.apache.cassandra.thrift.Index
import org.apache.cassandra.utils.WrappedRunnable;
import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
import static org.apache.cassandra.Util.column;
import static org.apache.cassandra.Util.getBytes;
import static org.junit.Assert.assertNull;
@@ -515,4 +518,17 @@ public class ColumnFamilyStoreTest exten
rms.add(rm);
return Util.writeColumnFamily(rms);
}
+
+ @Test
+ public void testBackupAfterFlush() throws Throwable
+ {
+ insertKey1Key2();
+
+ File backupDir = new File(DatabaseDescriptor.getDataFileLocationForTable("Keyspace2", 0), "backups");
+ for (String f : Arrays.asList("Standard1-f-1-Data.db", "Standard1-f-1-Index.db", "Standard1-f-2-Data.db", "Standard1-f-2-Index.db",
+ "Standard1-f-1-Filter.db", "Standard1-f-1-Statistics.db", "Standard1-f-2-Filter.db", "Standard1-f-2-Statistics.db"))
+ {
+ assertTrue("can not find backedup file:" + f, new File(backupDir, f).exists());
+ }
+ }
}