You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/11/12 18:03:52 UTC
svn commit: r835446 - in /incubator/cassandra/trunk: conf/
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/io/ test/unit/org/apache/cassandra/
Author: jbellis
Date: Thu Nov 12 17:03:52 2009
New Revision: 835446
URL: http://svn.apache.org/viewvc?rev=835446&view=rev
Log:
r/m unused bootstrap directory and ensure streaming files live to be streamed
patch by jbellis; reviewed by Jun Rao for CASSANDRA-538
Modified:
incubator/cassandra/trunk/conf/storage-conf.xml
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/Streaming.java
incubator/cassandra/trunk/test/unit/org/apache/cassandra/CleanupHelper.java
Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=835446&r1=835445&r2=835446&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Thu Nov 12 17:03:52 2009
@@ -164,7 +164,6 @@
<DataFileDirectory>/var/lib/cassandra/data</DataFileDirectory>
</DataFileDirectories>
<CalloutLocation>/var/lib/cassandra/callouts</CalloutLocation>
- <BootstrapFileDirectory>/var/lib/cassandra/bootstrap</BootstrapFileDirectory>
<StagingFileDirectory>/var/lib/cassandra/staging</StagingFileDirectory>
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=835446&r1=835445&r2=835446&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Thu Nov 12 17:03:52 2009
@@ -65,7 +65,6 @@
/* Current index into the above list of directories */
private static int currentIndex_ = 0;
private static String logFileDirectory_;
- private static String bootstrapFileDirectory_;
private static int consistencyThreads_ = 4; // not configurable
private static int concurrentReaders_ = 8;
private static int concurrentWriters_ = 32;
@@ -387,14 +386,6 @@
for ( String dataFileDirectory : dataFileDirectories_ )
FileUtils.createDirectory(dataFileDirectory);
- /* bootstrap file directory */
- bootstrapFileDirectory_ = xmlUtils.getNodeValue("/Storage/BootstrapFileDirectory");
- if (bootstrapFileDirectory_ == null)
- {
- throw new ConfigurationException("BootstrapFileDirectory must be specified");
- }
- FileUtils.createDirectory(bootstrapFileDirectory_);
-
/* commit log directory */
logFileDirectory_ = xmlUtils.getNodeValue("/Storage/CommitLogDirectory");
if (logFileDirectory_ == null)
@@ -839,16 +830,6 @@
return dataFileDirectory;
}
- public static String getBootstrapFileLocation()
- {
- return bootstrapFileDirectory_;
- }
-
- public static void setBootstrapFileLocation(String bfLocation)
- {
- bootstrapFileDirectory_ = bfLocation;
- }
-
public static String getLogFileLocation()
{
return logFileDirectory_;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=835446&r1=835445&r2=835446&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Thu Nov 12 17:03:52 2009
@@ -804,25 +804,25 @@
SSTableWriter writer = null;
CompactionIterator ci = new CompactionIterator(sstables, getDefaultGCBefore());
+ Iterator nni = new FilterIterator(ci, PredicateUtils.notNullPredicate());
try
{
- if (!ci.hasNext())
+ if (!nni.hasNext())
{
- logger_.warn("Nothing to compact (all files empty or corrupt). This should not happen.");
return results;
}
- while (ci.hasNext())
+ while (nni.hasNext())
{
- CompactionIterator.CompactedRow row = ci.next();
+ CompactionIterator.CompactedRow row = (CompactionIterator.CompactedRow) nni.next();
if (Range.isTokenInRanges(row.key.token, ranges))
{
if (writer == null)
{
if (target != null)
{
- compactionFileLocation = compactionFileLocation + File.separator + "bootstrap";
+ compactionFileLocation = compactionFileLocation + File.separator + "stream";
}
FileUtils.createDirectory(compactionFileLocation);
String newFilename = new File(compactionFileLocation, getTempSSTableFileName()).getAbsolutePath();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/Streaming.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/Streaming.java?rev=835446&r1=835445&r2=835446&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/Streaming.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/Streaming.java Thu Nov 12 17:03:52 2009
@@ -56,12 +56,7 @@
if (logger.isDebugEnabled())
logger.debug("Performing anticompaction ...");
/* Get the list of files that need to be streamed */
- List<String> fileList = new ArrayList<String>();
- for (SSTableReader sstable : table.forceAntiCompaction(ranges, target))
- {
- fileList.addAll(sstable.getAllFilenames());
- }
- transferOneTable(target, fileList, tName); // also deletes the file, so no further cleanup needed
+ transferOneTable(target, table.forceAntiCompaction(ranges, target), tName); // SSTR GC deletes the file when done
}
catch (IOException e)
{
@@ -72,17 +67,20 @@
callback.run();
}
- private static void transferOneTable(InetAddress target, List<String> fileList, String table) throws IOException
+ private static void transferOneTable(InetAddress target, List<SSTableReader> sstables, String table) throws IOException
{
- if (fileList.isEmpty())
+ if (sstables.isEmpty())
return;
- StreamContextManager.StreamContext[] streamContexts = new StreamContextManager.StreamContext[fileList.size()];
+ StreamContextManager.StreamContext[] streamContexts = new StreamContextManager.StreamContext[sstables.size()];
int i = 0;
- for (String filename : fileList)
+ for (SSTableReader sstable : sstables)
{
- File file = new File(filename);
- streamContexts[i++] = new StreamContextManager.StreamContext(file.getAbsolutePath(), file.length(), table);
+ for (String filename : sstable.getAllFilenames())
+ {
+ File file = new File(filename);
+ streamContexts[i++] = new StreamContextManager.StreamContext(file.getAbsolutePath(), file.length(), table);
+ }
}
if (logger.isDebugEnabled())
logger.debug("Stream context metadata " + StringUtils.join(streamContexts, ", "));
@@ -96,8 +94,9 @@
if (logger.isDebugEnabled())
logger.debug("Waiting for transfer to " + target + " to complete");
StreamManager.instance(target).waitForStreamCompletion();
+ // reference sstables one more time to make sure it doesn't get GC'd early (causing delete of its files)
if (logger.isDebugEnabled())
- logger.debug("Done with transfer to " + target);
+ logger.debug("Done with transfer to " + target + " of " + StringUtils.join(sstables, ", "));
}
public static class StreamInitiateVerbHandler implements IVerbHandler
Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/CleanupHelper.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/CleanupHelper.java?rev=835446&r1=835445&r2=835446&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/CleanupHelper.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/CleanupHelper.java Thu Nov 12 17:03:52 2009
@@ -35,7 +35,6 @@
// we clean the fs twice, once to start with (so old data files don't get stored by anything static if this is the first run)
// and once after flushing stuff (to try to clean things out if it is not.) part #2 seems to be less than perfect.
String[] directoryNames = {
- DatabaseDescriptor.getBootstrapFileLocation(),
DatabaseDescriptor.getLogFileLocation(),
};