You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@cassandra.apache.org by Prashant Malik <pm...@gmail.com> on 2009/04/03 00:57:35 UTC

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Hey Jonathan ,

  Are you testing these changes ?
  If we are going to make changes like this without much of a reason we
better test it as this code needs to run in
  our production environment.
  You need to test the changes at a reasonable scale.

- Prashant




On Thu, Apr 2, 2009 at 1:48 PM, <jb...@apache.org> wrote:

> Author: jbellis
> Date: Thu Apr  2 20:48:02 2009
> New Revision: 761423
>
> URL: http://svn.apache.org/viewvc?rev=761423&view=rev
> Log:
> rename stageOrderedCompaction -> getCompactionBuckets.  clean up code to
> make the algorithm more clear, and to allow grouping with more than just the
> previous bucket.  add getCompactionBucketsTest
>
> Modified:
>
>  incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
>
>  incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
>
> Modified:
> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
> URL:
> http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java?rev=761423&r1=761422&r2=761423&view=diff
>
> ==============================================================================
> ---
> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
> (original)
> +++
> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
> Thu Apr  2 20:48:02 2009
> @@ -30,6 +30,7 @@
>  import java.util.PriorityQueue;
>  import java.util.Set;
>  import java.util.StringTokenizer;
> +import java.util.Arrays;
>  import java.util.concurrent.ExecutionException;
>  import java.util.concurrent.Future;
>  import java.util.concurrent.atomic.AtomicBoolean;
> @@ -707,48 +708,47 @@
>     }
>
>     /*
> -     * Stage the compactions , compact similar size files.
> -     * This fn figures out the files close enough by size and if they
> -     * are greater than the threshold then compacts.
> +     * Group files of similar size into buckets.
>      */
> -    Map<Integer, List<String>> stageOrderedCompaction(List<String> files)
> +    static Set<List<String>> getCompactionBuckets(List<String> files, long
> min)
>     {
> -        // Sort the files based on the generation ID
> -        Collections.sort(files, new
> FileNameComparator(FileNameComparator.Ascending));
> -       Map<Integer, List<String>>  buckets = new HashMap<Integer,
> List<String>>();
> -       int maxBuckets = 1000;
> -       long averages[] = new long[maxBuckets];
> -       long min = 50L*1024L*1024L;
> -       Integer i = 0;
> -       for(String file : files)
> +       Map<List<String>, Long> buckets = new HashMap<List<String>,
> Long>();
> +       for(String fname : files)
>        {
> -               File f = new File(file);
> +               File f = new File(fname);
>                long size = f.length();
> -                       if ( (size > averages[i]/2 && size <
> 3*averages[i]/2) || ( size < min && averages[i] < min ))
> -                       {
> -                               averages[i] = (averages[i] + size) / 2 ;
> -                               List<String> fileList = buckets.get(i);
> -                               if(fileList == null)
> -                               {
> -                                       fileList = new ArrayList<String>();
> -                                       buckets.put(i, fileList);
> -                               }
> -                               fileList.add(file);
> -                       }
> -                       else
> +
> +               boolean bFound = false;
> +            // look for a bucket containing similar-sized files:
> +            // group in the same bucket if it's w/in 50% of the average
> for this bucket,
> +            // or this file and the bucket are all considered "small"
> (less than `min`)
> +            for (List<String> bucket : new
> ArrayList<List<String>>(buckets.keySet()))
> +               {
> +                long averageSize = buckets.get(bucket);
> +                if ((size > averageSize/2 && size < 3*averageSize/2)
> +                    || ( size < min && averageSize < min))
> +                       {
> +                    // remove and re-add because adding changes the hash
> +                    buckets.remove(bucket);
> +                               averageSize = (averageSize + size) / 2 ;
> +                    bucket.add(fname);
> +                    buckets.put(bucket, averageSize);
> +                               bFound = true;
> +                               break;
> +                       }
> +               }
> +            // no similar bucket found; put it in a new one
> +               if(!bFound)
>                {
> -                               if( i >= maxBuckets )
> -                                       break;
> -                               i++;
> -                               List<String> fileList = new
> ArrayList<String>();
> -                               buckets.put(i, fileList);
> -                               fileList.add(file);
> -                       averages[i] = size;
> +                ArrayList<String> bucket = new ArrayList<String>();
> +                bucket.add(fname);
> +                buckets.put(bucket, size);
>                }
>        }
> -       return buckets;
> +
> +        return buckets.keySet();
>     }
> -
> +
>     /*
>      * Break the files into buckets and then compact.
>      */
> @@ -759,11 +759,8 @@
>         try
>         {
>                int count = 0;
> -               Map<Integer, List<String>> buckets =
> stageOrderedCompaction(files);
> -               Set<Integer> keySet = buckets.keySet();
> -               for(Integer key : keySet)
> -               {
> -                       List<String> fileList = buckets.get(key);
> +               for(List<String> fileList : getCompactionBuckets(files,
> 50L*1024L*1024L))
> +            {
>                        Collections.sort( fileList , new FileNameComparator(
> FileNameComparator.Ascending));
>                        if(fileList.size() >= threshHold_ )
>                        {
>
> Modified:
> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
> URL:
> http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=761423&r1=761422&r2=761423&view=diff
>
> ==============================================================================
> ---
> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
> (original)
> +++
> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
> Thu Apr  2 20:48:02 2009
> @@ -4,17 +4,24 @@
>  import org.testng.annotations.Test;
>
>  import java.io.IOException;
> +import java.io.File;
> +import java.io.FileOutputStream;
>  import java.util.Collection;
>  import java.util.List;
>  import java.util.Random;
>  import java.util.Arrays;
> +import java.util.ArrayList;
> +import java.util.Set;
>  import java.util.concurrent.FutureTask;
>  import java.util.concurrent.Future;
>  import java.util.concurrent.ExecutionException;
>
> -public class ColumnFamilyStoreTest extends ServerTest {
> +public class ColumnFamilyStoreTest extends ServerTest
> +{
>     static byte[] bytes1, bytes2;
> -    static {
> +
> +    static
> +    {
>         Random random = new Random();
>         bytes1 = new byte[1024];
>         bytes2 = new byte[128];
> @@ -31,14 +38,14 @@
>         {
>             String key = Integer.toString(i);
>             RowMutation rm;
> -            for ( int j = 0; j < 8; ++j )
> +            for (int j = 0; j < 8; ++j)
>             {
>                 byte[] bytes = j % 2 == 0 ? bytes1 : bytes2;
>                 rm = new RowMutation("Table1", key);
>                 rm.add("Standard1:" + "Column-" + j, bytes, j);
>                 rm.apply();
>
> -                for ( int k = 0; k < 4; ++k )
> +                for (int k = 0; k < 4; ++k)
>                 {
>                     bytes = (j + k) % 2 == 0 ? bytes1 : bytes2;
>                     rm = new RowMutation("Table1", key);
> @@ -55,7 +62,9 @@
>         // wait for flush to finish
>         Future f = MemtableManager.instance().flusher_.submit(new
> Runnable()
>         {
> -            public void run() {}
> +            public void run()
> +            {
> +            }
>         });
>         f.get();
>
> @@ -65,7 +74,7 @@
>     private void validateBytes(Table table)
>             throws ColumnFamilyNotDefinedException, IOException
>     {
> -        for ( int i = 900; i < 1000; ++i )
> +        for (int i = 900; i < 1000; ++i)
>         {
>             String key = Integer.toString(i);
>             ColumnFamily cf;
> @@ -99,7 +108,8 @@
>     }
>
>     @Test
> -    public void testRemove() throws IOException,
> ColumnFamilyNotDefinedException {
> +    public void testRemove() throws IOException,
> ColumnFamilyNotDefinedException
> +    {
>         Table table = Table.open("Table1");
>         ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
>         RowMutation rm;
> @@ -122,7 +132,8 @@
>     }
>
>     @Test
> -    public void testRemoveSuperColumn() throws IOException,
> ColumnFamilyNotDefinedException {
> +    public void testRemoveSuperColumn() throws IOException,
> ColumnFamilyNotDefinedException
> +    {
>         Table table = Table.open("Table1");
>         ColumnFamilyStore store = table.getColumnFamilyStore("Super1");
>         RowMutation rm;
> @@ -152,4 +163,42 @@
>         assert subColumns.iterator().next().timestamp() == 0;
>         assert ColumnFamilyStore.removeDeleted(resolved).getColumnCount()
> == 0;
>     }
> +
> +    @Test
> +    public void testGetCompactionBuckets() throws IOException
> +    {
> +        // create files 20 40 60 ... 180
> +        List<String> small = new ArrayList<String>();
> +        List<String> med = new ArrayList<String>();
> +        List<String> all = new ArrayList<String>();
> +
> +        String fname;
> +        fname = createFile(20);
> +        small.add(fname);
> +        all.add(fname);
> +        fname = createFile(40);
> +        small.add(fname);
> +        all.add(fname);
> +
> +        for (int i = 60; i <= 140; i += 20)
> +        {
> +            fname = createFile(i);
> +            med.add(fname);
> +            all.add(fname);
> +        }
> +
> +        Set<List<String>> buckets =
> ColumnFamilyStore.getCompactionBuckets(all, 50);
> +        assert buckets.contains(small);
> +        assert buckets.contains(med);
> +    }
> +
> +    private String createFile(int nBytes) throws IOException
> +    {
> +        File f = File.createTempFile("bucket_test", "");
> +        FileOutputStream fos = new FileOutputStream(f);
> +        byte[] bytes = new byte[nBytes];
> +        fos.write(bytes);
> +        fos.close();
> +        return f.getAbsolutePath();
> +    }
>  }
>
>
>

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Posted by Torsten Curdt <tc...@apache.org>.
Then maybe you guys should discuss the option of using a branch.
Either a development or a stable branch. Otherwise I fear this will be
a problem. For the project and for you guys.

cheers
--
Torsten

On Fri, Apr 3, 2009 at 08:58, Prashant Malik <pm...@gmail.com> wrote:
> So far we were doing that internally as it was more controlled development
> but looks like now we will have to come up with a process.
>
> - Prashant
>
> On 4/2/09, Torsten Curdt <tc...@vafer.org> wrote:
>>
>> Prashant, you guys run trunk in production?? ...just wondering
>>
>> cheers
>> --
>>
>> Torsten
>>
>>
>> On Fri, Apr 3, 2009 at 00:57, Prashant Malik <pm...@gmail.com> wrote:
>> > Hey Jonathan ,
>> >
>> >  Are you testing these changes ?
>> >  If we are going to make changes like this without much of a reason we
>> > better test it as this code needs to run in
>> >  our production environment.
>> >  You need to test the changes at a reasonable scale.
>> >
>> > - Prashant
>>
>

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Posted by Prashant Malik <pm...@gmail.com>.
So far we were doing that internally as it was more controlled development
but looks like now we will have to come up with a process.

- Prashant

On 4/2/09, Torsten Curdt <tc...@vafer.org> wrote:
>
> Prashant, you guys run trunk in production?? ...just wondering
>
> cheers
> --
>
> Torsten
>
>
> On Fri, Apr 3, 2009 at 00:57, Prashant Malik <pm...@gmail.com> wrote:
> > Hey Jonathan ,
> >
> >  Are you testing these changes ?
> >  If we are going to make changes like this without much of a reason we
> > better test it as this code needs to run in
> >  our production environment.
> >  You need to test the changes at a reasonable scale.
> >
> > - Prashant
>

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Posted by Torsten Curdt <tc...@vafer.org>.
Prashant, you guys run trunk in production?? ...just wondering

cheers
--
Torsten

On Fri, Apr 3, 2009 at 00:57, Prashant Malik <pm...@gmail.com> wrote:
> Hey Jonathan ,
>
>  Are you testing these changes ?
>  If we are going to make changes like this without much of a reason we
> better test it as this code needs to run in
>  our production environment.
>  You need to test the changes at a reasonable scale.
>
> - Prashant

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Posted by Jonathan Ellis <jb...@gmail.com>.
That would be awesome.

On Thu, Apr 2, 2009 at 7:09 PM, Sandeep Tata <sa...@gmail.com> wrote:
> This brings up the issue of unit tests.I know Jonathan is working on
> building up some tests.
> I have some that I can clean up and contribute (they're all Junit4)

Re: svn commit: r761423 - in /incubator/cassandra/trunk: src/org/apache/cassandra/db/ColumnFamilyStore.java test/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Posted by Sandeep Tata <sa...@gmail.com>.
This brings up the issue of unit tests.I know Jonathan is working on
building up some tests.
I have some that I can clean up and contribute (they're all Junit4)
A & P : Any suite of tests you can share ?

We should probably put this as our highest in the list of "Cassandra Priorities"

This'll probably not catch performance bugs, but at least prevent semantic bugs.

On Thu, Apr 2, 2009 at 3:57 PM, Prashant Malik <pm...@gmail.com> wrote:
> Hey Jonathan ,
>
>  Are you testing these changes ?
>  If we are going to make changes like this without much of a reason we
> better test it as this code needs to run in
>  our production environment.
>  You need to test the changes at a reasonable scale.
>
> - Prashant
>
>
>
>
> On Thu, Apr 2, 2009 at 1:48 PM, <jb...@apache.org> wrote:
>
>> Author: jbellis
>> Date: Thu Apr  2 20:48:02 2009
>> New Revision: 761423
>>
>> URL: http://svn.apache.org/viewvc?rev=761423&view=rev
>> Log:
>> rename stageOrderedCompaction -> getCompactionBuckets.  clean up code to
>> make the algorithm more clear, and to allow grouping with more than just the
>> previous bucket.  add getCompactionBucketsTest
>>
>> Modified:
>>
>>  incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
>>
>>  incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
>>
>> Modified:
>> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java?rev=761423&r1=761422&r2=761423&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
>> (original)
>> +++
>> incubator/cassandra/trunk/src/org/apache/cassandra/db/ColumnFamilyStore.java
>> Thu Apr  2 20:48:02 2009
>> @@ -30,6 +30,7 @@
>>  import java.util.PriorityQueue;
>>  import java.util.Set;
>>  import java.util.StringTokenizer;
>> +import java.util.Arrays;
>>  import java.util.concurrent.ExecutionException;
>>  import java.util.concurrent.Future;
>>  import java.util.concurrent.atomic.AtomicBoolean;
>> @@ -707,48 +708,47 @@
>>     }
>>
>>     /*
>> -     * Stage the compactions , compact similar size files.
>> -     * This fn figures out the files close enough by size and if they
>> -     * are greater than the threshold then compacts.
>> +     * Group files of similar size into buckets.
>>      */
>> -    Map<Integer, List<String>> stageOrderedCompaction(List<String> files)
>> +    static Set<List<String>> getCompactionBuckets(List<String> files, long
>> min)
>>     {
>> -        // Sort the files based on the generation ID
>> -        Collections.sort(files, new
>> FileNameComparator(FileNameComparator.Ascending));
>> -       Map<Integer, List<String>>  buckets = new HashMap<Integer,
>> List<String>>();
>> -       int maxBuckets = 1000;
>> -       long averages[] = new long[maxBuckets];
>> -       long min = 50L*1024L*1024L;
>> -       Integer i = 0;
>> -       for(String file : files)
>> +       Map<List<String>, Long> buckets = new HashMap<List<String>,
>> Long>();
>> +       for(String fname : files)
>>        {
>> -               File f = new File(file);
>> +               File f = new File(fname);
>>                long size = f.length();
>> -                       if ( (size > averages[i]/2 && size <
>> 3*averages[i]/2) || ( size < min && averages[i] < min ))
>> -                       {
>> -                               averages[i] = (averages[i] + size) / 2 ;
>> -                               List<String> fileList = buckets.get(i);
>> -                               if(fileList == null)
>> -                               {
>> -                                       fileList = new ArrayList<String>();
>> -                                       buckets.put(i, fileList);
>> -                               }
>> -                               fileList.add(file);
>> -                       }
>> -                       else
>> +
>> +               boolean bFound = false;
>> +            // look for a bucket containing similar-sized files:
>> +            // group in the same bucket if it's w/in 50% of the average
>> for this bucket,
>> +            // or this file and the bucket are all considered "small"
>> (less than `min`)
>> +            for (List<String> bucket : new
>> ArrayList<List<String>>(buckets.keySet()))
>> +               {
>> +                long averageSize = buckets.get(bucket);
>> +                if ((size > averageSize/2 && size < 3*averageSize/2)
>> +                    || ( size < min && averageSize < min))
>> +                       {
>> +                    // remove and re-add because adding changes the hash
>> +                    buckets.remove(bucket);
>> +                               averageSize = (averageSize + size) / 2 ;
>> +                    bucket.add(fname);
>> +                    buckets.put(bucket, averageSize);
>> +                               bFound = true;
>> +                               break;
>> +                       }
>> +               }
>> +            // no similar bucket found; put it in a new one
>> +               if(!bFound)
>>                {
>> -                               if( i >= maxBuckets )
>> -                                       break;
>> -                               i++;
>> -                               List<String> fileList = new
>> ArrayList<String>();
>> -                               buckets.put(i, fileList);
>> -                               fileList.add(file);
>> -                       averages[i] = size;
>> +                ArrayList<String> bucket = new ArrayList<String>();
>> +                bucket.add(fname);
>> +                buckets.put(bucket, size);
>>                }
>>        }
>> -       return buckets;
>> +
>> +        return buckets.keySet();
>>     }
>> -
>> +
>>     /*
>>      * Break the files into buckets and then compact.
>>      */
>> @@ -759,11 +759,8 @@
>>         try
>>         {
>>                int count = 0;
>> -               Map<Integer, List<String>> buckets =
>> stageOrderedCompaction(files);
>> -               Set<Integer> keySet = buckets.keySet();
>> -               for(Integer key : keySet)
>> -               {
>> -                       List<String> fileList = buckets.get(key);
>> +               for(List<String> fileList : getCompactionBuckets(files,
>> 50L*1024L*1024L))
>> +            {
>>                        Collections.sort( fileList , new FileNameComparator(
>> FileNameComparator.Ascending));
>>                        if(fileList.size() >= threshHold_ )
>>                        {
>>
>> Modified:
>> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=761423&r1=761422&r2=761423&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
>> (original)
>> +++
>> incubator/cassandra/trunk/test/org/apache/cassandra/db/ColumnFamilyStoreTest.java
>> Thu Apr  2 20:48:02 2009
>> @@ -4,17 +4,24 @@
>>  import org.testng.annotations.Test;
>>
>>  import java.io.IOException;
>> +import java.io.File;
>> +import java.io.FileOutputStream;
>>  import java.util.Collection;
>>  import java.util.List;
>>  import java.util.Random;
>>  import java.util.Arrays;
>> +import java.util.ArrayList;
>> +import java.util.Set;
>>  import java.util.concurrent.FutureTask;
>>  import java.util.concurrent.Future;
>>  import java.util.concurrent.ExecutionException;
>>
>> -public class ColumnFamilyStoreTest extends ServerTest {
>> +public class ColumnFamilyStoreTest extends ServerTest
>> +{
>>     static byte[] bytes1, bytes2;
>> -    static {
>> +
>> +    static
>> +    {
>>         Random random = new Random();
>>         bytes1 = new byte[1024];
>>         bytes2 = new byte[128];
>> @@ -31,14 +38,14 @@
>>         {
>>             String key = Integer.toString(i);
>>             RowMutation rm;
>> -            for ( int j = 0; j < 8; ++j )
>> +            for (int j = 0; j < 8; ++j)
>>             {
>>                 byte[] bytes = j % 2 == 0 ? bytes1 : bytes2;
>>                 rm = new RowMutation("Table1", key);
>>                 rm.add("Standard1:" + "Column-" + j, bytes, j);
>>                 rm.apply();
>>
>> -                for ( int k = 0; k < 4; ++k )
>> +                for (int k = 0; k < 4; ++k)
>>                 {
>>                     bytes = (j + k) % 2 == 0 ? bytes1 : bytes2;
>>                     rm = new RowMutation("Table1", key);
>> @@ -55,7 +62,9 @@
>>         // wait for flush to finish
>>         Future f = MemtableManager.instance().flusher_.submit(new
>> Runnable()
>>         {
>> -            public void run() {}
>> +            public void run()
>> +            {
>> +            }
>>         });
>>         f.get();
>>
>> @@ -65,7 +74,7 @@
>>     private void validateBytes(Table table)
>>             throws ColumnFamilyNotDefinedException, IOException
>>     {
>> -        for ( int i = 900; i < 1000; ++i )
>> +        for (int i = 900; i < 1000; ++i)
>>         {
>>             String key = Integer.toString(i);
>>             ColumnFamily cf;
>> @@ -99,7 +108,8 @@
>>     }
>>
>>     @Test
>> -    public void testRemove() throws IOException,
>> ColumnFamilyNotDefinedException {
>> +    public void testRemove() throws IOException,
>> ColumnFamilyNotDefinedException
>> +    {
>>         Table table = Table.open("Table1");
>>         ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
>>         RowMutation rm;
>> @@ -122,7 +132,8 @@
>>     }
>>
>>     @Test
>> -    public void testRemoveSuperColumn() throws IOException,
>> ColumnFamilyNotDefinedException {
>> +    public void testRemoveSuperColumn() throws IOException,
>> ColumnFamilyNotDefinedException
>> +    {
>>         Table table = Table.open("Table1");
>>         ColumnFamilyStore store = table.getColumnFamilyStore("Super1");
>>         RowMutation rm;
>> @@ -152,4 +163,42 @@
>>         assert subColumns.iterator().next().timestamp() == 0;
>>         assert ColumnFamilyStore.removeDeleted(resolved).getColumnCount()
>> == 0;
>>     }
>> +
>> +    @Test
>> +    public void testGetCompactionBuckets() throws IOException
>> +    {
>> +        // create files 20 40 60 ... 180
>> +        List<String> small = new ArrayList<String>();
>> +        List<String> med = new ArrayList<String>();
>> +        List<String> all = new ArrayList<String>();
>> +
>> +        String fname;
>> +        fname = createFile(20);
>> +        small.add(fname);
>> +        all.add(fname);
>> +        fname = createFile(40);
>> +        small.add(fname);
>> +        all.add(fname);
>> +
>> +        for (int i = 60; i <= 140; i += 20)
>> +        {
>> +            fname = createFile(i);
>> +            med.add(fname);
>> +            all.add(fname);
>> +        }
>> +
>> +        Set<List<String>> buckets =
>> ColumnFamilyStore.getCompactionBuckets(all, 50);
>> +        assert buckets.contains(small);
>> +        assert buckets.contains(med);
>> +    }
>> +
>> +    private String createFile(int nBytes) throws IOException
>> +    {
>> +        File f = File.createTempFile("bucket_test", "");
>> +        FileOutputStream fos = new FileOutputStream(f);
>> +        byte[] bytes = new byte[nBytes];
>> +        fos.write(bytes);
>> +        fos.close();
>> +        return f.getAbsolutePath();
>> +    }
>>  }
>>
>>
>>
>