You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2009/09/15 20:17:05 UTC
svn commit: r815434 - in
/incubator/cassandra/trunk/src/java/org/apache/cassandra:
db/BinaryMemtable.java dht/CollatingOrderPreservingPartitioner.java
dht/IPartitioner.java dht/OrderPreservingPartitioner.java
dht/RandomPartitioner.java
Author: jbellis
Date: Tue Sep 15 18:17:05 2009
New Revision: 815434
URL: http://svn.apache.org/viewvc?rev=815434&view=rev
Log:
Use DecoratedKey objects in BMt to avoid lots of expensive String parsing operations when sorting. patch by johano; reviewed by jbellis for CASSANDRA-420
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java?rev=815434&r1=815433&r2=815434&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java Tue Sep 15 18:17:05 2009
@@ -34,7 +34,6 @@
import org.apache.log4j.Logger;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
-import java.util.*;
import org.apache.cassandra.dht.IPartitioner;
public class BinaryMemtable
@@ -47,10 +46,11 @@
private String table_;
private String cfName_;
private boolean isFrozen_ = false;
- private Map<String, byte[]> columnFamilies_ = new NonBlockingHashMap<String, byte[]>();
+ private Map<DecoratedKey, byte[]> columnFamilies_ = new NonBlockingHashMap<DecoratedKey, byte[]>();
/* Lock and Condition for notifying new clients about Memtable switches */
Lock lock_ = new ReentrantLock();
Condition condition_;
+ private final IPartitioner partitioner_ = StorageService.getPartitioner();
BinaryMemtable(String table, String cfName) throws IOException
{
@@ -123,8 +123,8 @@
private void resolve(String key, byte[] buffer)
{
- columnFamilies_.put(key, buffer);
- currentSize_.addAndGet(buffer.length + key.length());
+ columnFamilies_.put(partitioner_.decorateKeyObj(key), buffer);
+ currentSize_.addAndGet(buffer.length + key.length());
}
@@ -144,7 +144,7 @@
String path;
SSTableWriter writer;
ColumnFamilyStore cfStore = Table.open(table_).getColumnFamilyStore(cfName_);
- List<String> keys = new ArrayList<String>( columnFamilies_.keySet() );
+ List<DecoratedKey> keys = new ArrayList<DecoratedKey>( columnFamilies_.keySet() );
/*
Adding a lock here so data directories are evenly used. By default currentIndex
is incremented, not an AtomicInteger. Let's fix this!
@@ -160,19 +160,16 @@
lock_.unlock();
}
- final IPartitioner partitioner = StorageService.getPartitioner();
- final Comparator<String> dc = partitioner.getDecoratedKeyComparator();
- Collections.sort(keys, dc);
-
+ Collections.sort(keys, partitioner_.getDecoratedKeyObjComparator());
/* Use this BloomFilter to decide if a key exists in a SSTable */
- for ( String key : keys )
- {
+ for (DecoratedKey key : keys)
+ {
byte[] bytes = columnFamilies_.get(key);
- if ( bytes.length > 0 )
- {
+ if (bytes.length > 0)
+ {
/* Now write the key and value to disk */
- writer.append(key, bytes);
+ writer.append(key.toString(), bytes);
}
}
cfStore.addSSTable(writer.closeAndOpenReader());
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java?rev=815434&r1=815433&r2=815434&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java Tue Sep 15 18:17:05 2009
@@ -18,8 +18,6 @@
package org.apache.cassandra.dht;
-import java.io.UnsupportedEncodingException;
-import java.math.BigInteger;
import java.text.Collator;
import java.util.Arrays;
import java.util.Comparator;
@@ -27,6 +25,7 @@
import java.util.Random;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.utils.FBUtilities;
public class CollatingOrderPreservingPartitioner implements IPartitioner<BytesToken>
@@ -44,18 +43,29 @@
return collator.compare(o1, o2);
}
};
+ private static final Comparator<DecoratedKey<BytesToken>> objComparator = new Comparator<DecoratedKey<BytesToken>>() {
+ public int compare(DecoratedKey<BytesToken> o1, DecoratedKey<BytesToken> o2)
+ {
+ return FBUtilities.compareByteArrays(o1.getToken().token, o2.getToken().token);
+ }
+ };
private static final Comparator<String> reverseComparator = new Comparator<String>() {
public int compare(String o1, String o2)
{
return -comparator.compare(o1, o2);
}
};
-
+
public String decorateKey(String key)
{
return key;
}
+ public DecoratedKey<BytesToken> decorateKeyObj(String key)
+ {
+ return new DecoratedKey<BytesToken>(getToken(key), key);
+ }
+
public String undecorateKey(String decoratedKey)
{
return decoratedKey;
@@ -66,6 +76,11 @@
return comparator;
}
+ public Comparator<DecoratedKey<BytesToken>> getDecoratedKeyObjComparator()
+ {
+ return objComparator;
+ }
+
public Comparator<String> getReverseDecoratedKeyComparator()
{
return reverseComparator;
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java?rev=815434&r1=815433&r2=815434&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java Tue Sep 15 18:17:05 2009
@@ -20,6 +20,8 @@
import java.util.Comparator;
+import org.apache.cassandra.db.DecoratedKey;
+
public interface IPartitioner<T extends Token>
{
/**
@@ -30,10 +32,23 @@
* @return decorated on-disk version of key
*/
public String decorateKey(String key);
+
+ /**
+ * Transform key to object representation of the on-disk format.
+ *
+ * @param key the raw, client-facing key
+ * @return decorated version of key
+ */
+ public DecoratedKey<T> decorateKeyObj(String key);
public String undecorateKey(String decoratedKey);
public Comparator<String> getDecoratedKeyComparator();
+
+ /**
+ * @return a comparator for decorated key objects, not strings
+ */
+ public Comparator<DecoratedKey<T>> getDecoratedKeyObjComparator();
public Comparator<String> getReverseDecoratedKeyComparator();
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java?rev=815434&r1=815433&r2=815434&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java Tue Sep 15 18:17:05 2009
@@ -24,6 +24,7 @@
import java.util.Random;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
public class OrderPreservingPartitioner implements IPartitioner<StringToken>
{
@@ -38,6 +39,13 @@
return o1.compareTo(o2);
}
};
+ private static final Comparator<DecoratedKey<StringToken>> objComparator =
+ new Comparator<DecoratedKey<StringToken>>() {
+ public int compare(DecoratedKey<StringToken> o1, DecoratedKey<StringToken> o2)
+ {
+ return o1.getKey().compareTo(o2.getKey());
+ }
+ };
private static final Comparator<String> reverseComparator = new Comparator<String>() {
public int compare(String o1, String o2)
{
@@ -50,6 +58,11 @@
return key;
}
+ public DecoratedKey<StringToken> decorateKeyObj(String key)
+ {
+ return new DecoratedKey<StringToken>(null, key);
+ }
+
public String undecorateKey(String decoratedKey)
{
return decoratedKey;
@@ -60,6 +73,11 @@
return comparator;
}
+ public Comparator<DecoratedKey<StringToken>> getDecoratedKeyObjComparator()
+ {
+ return objComparator;
+ }
+
public Comparator<String> getReverseDecoratedKeyComparator()
{
return reverseComparator;
@@ -234,4 +252,5 @@
{
return new StringToken(key);
}
+
}
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java?rev=815434&r1=815433&r2=815434&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java Tue Sep 15 18:17:05 2009
@@ -23,6 +23,7 @@
import java.util.StringTokenizer;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.GuidGenerator;
@@ -36,6 +37,21 @@
public static final BigIntegerToken MINIMUM = new BigIntegerToken("0");
+ private static final Comparator<DecoratedKey<BigIntegerToken>> objComparator =
+ new Comparator<DecoratedKey<BigIntegerToken>>() {
+ public int compare(DecoratedKey<BigIntegerToken> o1, DecoratedKey<BigIntegerToken> o2)
+ {
+ // first, compare on the bigint hash "decoration". usually this will be enough.
+ int v = o1.getToken().compareTo(o2.getToken());
+ if (v != 0) {
+ return v;
+ }
+
+ // if the hashes are equal, compare the strings
+ return o1.getKey().compareTo(o2.getKey());
+ }
+ };
+
private static final Comparator<String> comparator = new Comparator<String>()
{
public int compare(String o1, String o2)
@@ -69,6 +85,11 @@
return FBUtilities.hash(key).toString() + ":" + key;
}
+ public DecoratedKey<BigIntegerToken> decorateKeyObj(String key)
+ {
+ return new DecoratedKey<BigIntegerToken>(getToken(key), key);
+ }
+
public String undecorateKey(String decoratedKey)
{
return decoratedKey.split(":", 2)[1];
@@ -78,6 +99,11 @@
{
return comparator;
}
+
+ public Comparator<DecoratedKey<BigIntegerToken>> getDecoratedKeyObjComparator()
+ {
+ return objComparator;
+ }
public Comparator<String> getReverseDecoratedKeyComparator()
{