You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/05/20 16:44:41 UTC
svn commit: r776716 - in /incubator/cassandra/trunk: ./
interface/gen-java/org/apache/cassandra/service/ lib/ src/java/org/
src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/
test/unit/org/ test/unit/org/apache/cassandra/db/
Author: jbellis
Date: Wed May 20 14:44:41 2009
New Revision: 776716
URL: http://svn.apache.org/viewvc?rev=776716&view=rev
Log:
merge from 0.3 branch
Removed:
incubator/cassandra/trunk/lib/activation.jar
Modified:
incubator/cassandra/trunk/ (props changed)
incubator/cassandra/trunk/build.xml
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java (props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java (props changed)
incubator/cassandra/trunk/src/java/org/ (props changed)
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
incubator/cassandra/trunk/test/unit/org/ (props changed)
incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java
Propchange: incubator/cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1 +1 @@
-/incubator/cassandra/branches/cassandra-0.3:774578-775981
+/incubator/cassandra/branches/cassandra-0.3:774578-776715
Modified: incubator/cassandra/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/build.xml?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/build.xml (original)
+++ incubator/cassandra/trunk/build.xml Wed May 20 14:44:41 2009
@@ -178,7 +178,7 @@
</copy>
<tar compression="gzip" longfile="gnu"
- destfile="${build.dir}/${final.name}.tgz">
+ destfile="${build.dir}/${final.name}-bin.tar.gz">
<tarfileset dir="${dist.dir}"
prefix="${final.name}">
@@ -191,6 +191,15 @@
<include name="bin/cassandra"/>
</tarfileset>
</tar>
+ <tar compression="gzip" longfile="gnu"
+ destfile="${build.dir}/${final.name}-src.tar.gz">
+
+ <tarfileset dir="${basedir}"
+ prefix="${final.name}-src">
+ <include name="**"/>
+ <exclude name="build/**" />
+ </tarfileset>
+ </tar>
</target>
<target name="build-test" depends="build" description="Build the Cassandra classes">
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java:749219-768588
Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-776715
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:749219-768588
Propchange: incubator/cassandra/trunk/src/java/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-776715
/incubator/cassandra/trunk/src/java/org:749219-769885
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed May 20 14:44:41 2009
@@ -56,7 +56,6 @@
{
private static Logger logger_ = Logger.getLogger(ColumnFamilyStore.class);
- private static int COMPACTION_THRESHOLD = 4; // compact this many sstables at a time
private static final int BUFSIZE = 128 * 1024 * 1024;
private static final int COMPACTION_MEMORY_THRESHOLD = 1 << 30;
@@ -770,8 +769,8 @@
lock_.writeLock().unlock();
}
- if ((ssTableSize >= COMPACTION_THRESHOLD && !isCompacting_.get())
- || (isCompacting_.get() && ssTableSize % COMPACTION_THRESHOLD == 0))
+ if ((ssTableSize >= MinorCompactionManager.COMPACTION_THRESHOLD && !isCompacting_.get())
+ || (isCompacting_.get() && ssTableSize % MinorCompactionManager.COMPACTION_THRESHOLD == 0))
{
logger_.debug("Submitting for compaction ...");
MinorCompactionManager.instance().submit(ColumnFamilyStore.this);
@@ -860,11 +859,6 @@
return buckets.keySet();
}
- public int doCompaction() throws IOException
- {
- return doCompaction(COMPACTION_THRESHOLD);
- }
-
/*
* Break the files into buckets and then compact.
*/
@@ -1294,6 +1288,7 @@
*/
private int doFileCompaction(List<String> files, int minBufferSize) throws IOException
{
+ logger_.info("Compacting [" + StringUtils.join(files, ",") + "]");
String compactionFileLocation = DatabaseDescriptor.getCompactionFileLocation(getExpectedCompactedFileSize(files));
// If the compaction file path is null that means we have no space left for this compaction.
// try again w/o the largest one.
@@ -1314,6 +1309,7 @@
if (pq.isEmpty())
{
+ logger_.warn("Nothing to compact (all files empty or corrupt)");
// TODO clean out bad files, if any
return 0;
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java Wed May 20 14:44:41 2009
@@ -46,7 +46,8 @@
private static MinorCompactionManager instance_;
private static Lock lock_ = new ReentrantLock();
private static Logger logger_ = Logger.getLogger(MinorCompactionManager.class);
- final static long intervalInMins_ = 5;
+ private static final long intervalInMins_ = 5;
+ static final int COMPACTION_THRESHOLD = 4; // compact this many sstables at a time
public static MinorCompactionManager instance()
{
@@ -66,33 +67,6 @@
return instance_;
}
- class FileCompactor implements Callable<Integer>
- {
- private ColumnFamilyStore columnFamilyStore_;
-
- FileCompactor(ColumnFamilyStore columnFamilyStore)
- {
- columnFamilyStore_ = columnFamilyStore;
- }
-
- public Integer call()
- {
- logger_.debug("Started compaction ..." + columnFamilyStore_.columnFamily_);
- try
- {
- return columnFamilyStore_.doCompaction();
- }
- catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- finally
- {
- logger_.debug("Finished compaction ..." + columnFamilyStore_.columnFamily_);
- }
- }
- }
-
class FileCompactor2 implements Callable<Boolean>
{
private ColumnFamilyStore columnFamilyStore_;
@@ -138,9 +112,9 @@
public void run()
{
- logger_.debug("Started Major compaction ..."+columnFamilyStore_.columnFamily_);
+ logger_.debug("Started Major compaction for " + columnFamilyStore_.columnFamily_);
columnFamilyStore_.doMajorCompaction(skip_);
- logger_.debug("Finished Major compaction ..."+columnFamilyStore_.columnFamily_);
+ logger_.debug("Finished Major compaction for " + columnFamilyStore_.columnFamily_);
}
}
@@ -183,22 +157,41 @@
public void submitPeriodicCompaction(final ColumnFamilyStore columnFamilyStore)
{
- Runnable runnable = new Runnable() // having to wrap Callable in Runnable is retarded but that's what the API insists on.
+ Runnable runnable = new Runnable()
{
public void run()
{
- new FileCompactor(columnFamilyStore).call();
+ try
+ {
+ columnFamilyStore.doCompaction(COMPACTION_THRESHOLD);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
}
};
compactor_.scheduleWithFixedDelay(runnable, MinorCompactionManager.intervalInMins_,
MinorCompactionManager.intervalInMins_, TimeUnit.MINUTES);
}
- public Future<Integer> submit(ColumnFamilyStore columnFamilyStore)
+ public Future<Integer> submit(final ColumnFamilyStore columnFamilyStore)
{
- return compactor_.submit(new FileCompactor(columnFamilyStore));
+ return submit(columnFamilyStore, COMPACTION_THRESHOLD);
}
-
+
+ Future<Integer> submit(final ColumnFamilyStore columnFamilyStore, final int threshold)
+ {
+ Callable<Integer> callable = new Callable<Integer>()
+ {
+ public Integer call() throws IOException
+ {
+ return columnFamilyStore.doCompaction(threshold);
+ }
+ };
+ return compactor_.submit(callable);
+ }
+
public void submitCleanup(ColumnFamilyStore columnFamilyStore)
{
compactor_.submit(new CleanupCompactor(columnFamilyStore));
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java Wed May 20 14:44:41 2009
@@ -220,7 +220,7 @@
}
File file = new File(dataFile);
- assert file.exists();
+ assert file.exists() : "attempted to delete non-existing file " + dataFile;
/* delete the data file */
if (!file.delete())
{
Propchange: incubator/cassandra/trunk/test/unit/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-776715
/incubator/cassandra/trunk/test/unit/org:749219-768583
Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java Wed May 20 14:44:41 2009
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import java.util.Set;
import java.util.HashSet;
@@ -44,7 +45,9 @@
store.forceBlockingFlush();
assertEquals(table.getKeyRange("", "", 10000).size(), inserted.size());
}
- store.doCompaction(2);
+ Future<Integer> ft = MinorCompactionManager.instance().submit(store, 2);
+ ft.get();
+ assertEquals(1, store.getSSTableFilenames().size());
assertEquals(table.getKeyRange("", "", 10000).size(), inserted.size());
}
@@ -57,6 +60,6 @@
@Test
public void testCompaction2() throws IOException, ExecutionException, InterruptedException
{
- testCompaction("Standard2", 500);
+ testCompaction("Standard2", 5);
}
}