You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/09/23 16:13:38 UTC
[1/3] git commit: Improve memory usage of metadata min/max column
names patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6077
Updated Branches:
refs/heads/cassandra-2.0 05952e7c6 -> 7e59db22b
refs/heads/trunk 494cc32d2 -> 8f5eb84b6
Improve memory usage of metadata min/max column names
patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6077
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7e59db22
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7e59db22
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7e59db22
Branch: refs/heads/cassandra-2.0
Commit: 7e59db22bdba93850a9877e6be021e29be9b9fef
Parents: 05952e7
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:13:10 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 09:13:10 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 4 +++
.../cassandra/io/sstable/ColumnNameHelper.java | 26 ++++++++++++++------
.../apache/cassandra/utils/ByteBufferUtil.java | 6 +++++
3 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f239b5a..6df5559 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.0.2
+ * Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+
+
2.0.1
* Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025)
* Improve memory use defaults (CASSANDRA-5069)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
index 2ad1cff..aaed765 100644
--- a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
+++ b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
@@ -27,6 +27,8 @@ import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ColumnToCollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
+import static org.apache.cassandra.utils.ByteBufferUtil.minimalBufferFor;
+
public class ColumnNameHelper
{
/**
@@ -174,7 +176,7 @@ public class ColumnNameHelper
public static List<ByteBuffer> mergeMin(List<ByteBuffer> minColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator)
{
if (minColumnNames.isEmpty())
- return candidates;
+ return minimalBuffersFor(candidates);
if (candidates.isEmpty())
return minColumnNames;
@@ -190,18 +192,26 @@ public class ColumnNameHelper
List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize);
for (int i = 0; i < minSize; i++)
- retList.add(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i)));
+ retList.add(minimalBufferFor(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i))));
for (int i = minSize; i < maxSize; i++)
- retList.add(biggest.get(i));
+ retList.add(minimalBufferFor(biggest.get(i)));
return retList;
}
else
{
- return Collections.singletonList(min(minColumnNames.get(0), candidates.get(0), columnNameComparator));
+ return Collections.singletonList(minimalBufferFor(min(minColumnNames.get(0), candidates.get(0), columnNameComparator)));
}
}
+ private static List<ByteBuffer> minimalBuffersFor(List<ByteBuffer> candidates)
+ {
+ List<ByteBuffer> minimalBuffers = new ArrayList<ByteBuffer>(candidates.size());
+ for (ByteBuffer byteBuffer : candidates)
+ minimalBuffers.add(minimalBufferFor(byteBuffer));
+ return minimalBuffers;
+ }
+
/**
* if columnNameComparator is CompositeType the columns are compared by components using the subcomparator
* on the same position.
@@ -217,7 +227,7 @@ public class ColumnNameHelper
public static List<ByteBuffer> mergeMax(List<ByteBuffer> maxColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator)
{
if (maxColumnNames.isEmpty())
- return candidates;
+ return minimalBuffersFor(candidates);
if (candidates.isEmpty())
return maxColumnNames;
@@ -232,15 +242,15 @@ public class ColumnNameHelper
List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize);
for (int i = 0; i < minSize; i++)
- retList.add(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i)));
+ retList.add(minimalBufferFor(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i))));
for (int i = minSize; i < maxSize; i++)
- retList.add(biggest.get(i));
+ retList.add(minimalBufferFor(biggest.get(i)));
return retList;
}
else
{
- return Collections.singletonList(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator));
+ return Collections.singletonList(minimalBufferFor(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator)));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index da8b958..4471fb9 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -552,4 +552,10 @@ public class ByteBufferUtil
int diff = value.remaining() - prefix.remaining();
return prefix.equals(value.duplicate().limit(value.remaining() - diff));
}
+
+ /** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */
+ public static ByteBuffer minimalBufferFor(ByteBuffer buf)
+ {
+ return buf.capacity() > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf;
+ }
}
[3/3] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8f5eb84b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8f5eb84b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8f5eb84b
Branch: refs/heads/trunk
Commit: 8f5eb84b66e217c0d74ac83764beda35b279ef8e
Parents: 494cc32 7e59db2
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:13:34 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 09:13:34 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 4 +++
.../cassandra/io/sstable/ColumnNameHelper.java | 26 ++++++++++++++------
.../apache/cassandra/utils/ByteBufferUtil.java | 6 +++++
3 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8f5eb84b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index da728d7,6df5559..fe81e20
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,11 -1,7 +1,15 @@@
+2.1
+ * change logging from log4j to logback (CASSANDRA-5883)
+ * switch to LZ4 compression for internode communication (CASSANDRA-5887)
+ * Stop using Thrift-generated Index* classes internally (CASSANDRA-5971)
+ * Remove 1.2 network compatibility code (CASSANDRA-5960)
+ * Remove leveled json manifest migration code (CASSANDRA-5996)
+
+
+ 2.0.2
+ * Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+
+
2.0.1
* Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025)
* Improve memory use defaults (CASSANDRA-5069)
[2/3] git commit: Improve memory usage of metadata min/max column
names patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6077
Posted by jb...@apache.org.
Improve memory usage of metadata min/max column names
patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6077
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7e59db22
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7e59db22
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7e59db22
Branch: refs/heads/trunk
Commit: 7e59db22bdba93850a9877e6be021e29be9b9fef
Parents: 05952e7
Author: Jonathan Ellis <jb...@apache.org>
Authored: Mon Sep 23 09:13:10 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Mon Sep 23 09:13:10 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 4 +++
.../cassandra/io/sstable/ColumnNameHelper.java | 26 ++++++++++++++------
.../apache/cassandra/utils/ByteBufferUtil.java | 6 +++++
3 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f239b5a..6df5559 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+2.0.2
+ * Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+
+
2.0.1
* Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025)
* Improve memory use defaults (CASSANDRA-5069)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
index 2ad1cff..aaed765 100644
--- a/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
+++ b/src/java/org/apache/cassandra/io/sstable/ColumnNameHelper.java
@@ -27,6 +27,8 @@ import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ColumnToCollectionType;
import org.apache.cassandra.db.marshal.CompositeType;
+import static org.apache.cassandra.utils.ByteBufferUtil.minimalBufferFor;
+
public class ColumnNameHelper
{
/**
@@ -174,7 +176,7 @@ public class ColumnNameHelper
public static List<ByteBuffer> mergeMin(List<ByteBuffer> minColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator)
{
if (minColumnNames.isEmpty())
- return candidates;
+ return minimalBuffersFor(candidates);
if (candidates.isEmpty())
return minColumnNames;
@@ -190,18 +192,26 @@ public class ColumnNameHelper
List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize);
for (int i = 0; i < minSize; i++)
- retList.add(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i)));
+ retList.add(minimalBufferFor(min(minColumnNames.get(i), candidates.get(i), ct.types.get(i))));
for (int i = minSize; i < maxSize; i++)
- retList.add(biggest.get(i));
+ retList.add(minimalBufferFor(biggest.get(i)));
return retList;
}
else
{
- return Collections.singletonList(min(minColumnNames.get(0), candidates.get(0), columnNameComparator));
+ return Collections.singletonList(minimalBufferFor(min(minColumnNames.get(0), candidates.get(0), columnNameComparator)));
}
}
+ private static List<ByteBuffer> minimalBuffersFor(List<ByteBuffer> candidates)
+ {
+ List<ByteBuffer> minimalBuffers = new ArrayList<ByteBuffer>(candidates.size());
+ for (ByteBuffer byteBuffer : candidates)
+ minimalBuffers.add(minimalBufferFor(byteBuffer));
+ return minimalBuffers;
+ }
+
/**
* if columnNameComparator is CompositeType the columns are compared by components using the subcomparator
* on the same position.
@@ -217,7 +227,7 @@ public class ColumnNameHelper
public static List<ByteBuffer> mergeMax(List<ByteBuffer> maxColumnNames, List<ByteBuffer> candidates, AbstractType<?> columnNameComparator)
{
if (maxColumnNames.isEmpty())
- return candidates;
+ return minimalBuffersFor(candidates);
if (candidates.isEmpty())
return maxColumnNames;
@@ -232,15 +242,15 @@ public class ColumnNameHelper
List<ByteBuffer> retList = new ArrayList<ByteBuffer>(maxSize);
for (int i = 0; i < minSize; i++)
- retList.add(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i)));
+ retList.add(minimalBufferFor(max(maxColumnNames.get(i), candidates.get(i), ct.types.get(i))));
for (int i = minSize; i < maxSize; i++)
- retList.add(biggest.get(i));
+ retList.add(minimalBufferFor(biggest.get(i)));
return retList;
}
else
{
- return Collections.singletonList(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator));
+ return Collections.singletonList(minimalBufferFor(max(maxColumnNames.get(0), candidates.get(0), columnNameComparator)));
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7e59db22/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index da8b958..4471fb9 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -552,4 +552,10 @@ public class ByteBufferUtil
int diff = value.remaining() - prefix.remaining();
return prefix.equals(value.duplicate().limit(value.remaining() - diff));
}
+
+ /** trims size of bytebuffer to exactly number of bytes in it, to do not hold too much memory */
+ public static ByteBuffer minimalBufferFor(ByteBuffer buf)
+ {
+ return buf.capacity() > buf.remaining() ? ByteBuffer.wrap(getArray(buf)) : buf;
+ }
}