You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by bd...@apache.org on 2019/11/11 23:33:37 UTC
[cassandra] branch cassandra-3.11 updated (0d5ccb9 -> 1cc9413)
This is an automated email from the ASF dual-hosted git repository.
bdeggleston pushed a change to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 0d5ccb9 Merge branch 'cassandra-3.0' into cassandra-3.11
new 9382186 Minimize clustering values in metadata collector
new 1cc9413 Merge branch 'cassandra-3.0' into cassandra-3.11
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
.../org/apache/cassandra/db/BufferClustering.java | 8 +++++
.../org/apache/cassandra/db/ClusteringBound.java | 8 +++++
.../apache/cassandra/db/ClusteringBoundary.java | 8 +++++
.../org/apache/cassandra/db/ClusteringPrefix.java | 6 ++++
.../org/apache/cassandra/db/NativeClustering.java | 5 ++++
.../org/apache/cassandra/db/RangeTombstone.java | 1 -
src/java/org/apache/cassandra/db/Slice.java | 1 +
.../io/sstable/metadata/MetadataCollector.java | 35 ++--------------------
.../org/apache/cassandra/utils/ByteBufferUtil.java | 23 ++++++++++++++
.../cassandra/io/sstable/SSTableMetadataTest.java | 6 ++++
11 files changed, 68 insertions(+), 34 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-3.0' into cassandra-3.11
Posted by bd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bdeggleston pushed a commit to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 1cc941361a2613dce600695f844b84f2dd3670a8
Merge: 0d5ccb9 9382186
Author: Blake Eggleston <bd...@gmail.com>
AuthorDate: Mon Nov 11 15:25:22 2019 -0800
Merge branch 'cassandra-3.0' into cassandra-3.11
CHANGES.txt | 1 +
.../org/apache/cassandra/db/BufferClustering.java | 8 +++++
.../org/apache/cassandra/db/ClusteringBound.java | 8 +++++
.../apache/cassandra/db/ClusteringBoundary.java | 8 +++++
.../org/apache/cassandra/db/ClusteringPrefix.java | 6 ++++
.../org/apache/cassandra/db/NativeClustering.java | 5 ++++
.../org/apache/cassandra/db/RangeTombstone.java | 1 -
src/java/org/apache/cassandra/db/Slice.java | 1 +
.../io/sstable/metadata/MetadataCollector.java | 35 ++--------------------
.../org/apache/cassandra/utils/ByteBufferUtil.java | 23 ++++++++++++++
.../cassandra/io/sstable/SSTableMetadataTest.java | 6 ++++
11 files changed, 68 insertions(+), 34 deletions(-)
diff --cc CHANGES.txt
index 6781c4e,6dd6d0a..c1480a4
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,5 -1,5 +1,6 @@@
-3.0.20
+3.11.6
+Merged from 3.0:
+ * Minimize clustering values in metadata collector (CASSANDRA-15400)
* Avoid over-trimming of results in mixed mode clusters (CASSANDRA-15405)
* validate value sizes in LegacyLayout (CASSANDRA-15373)
* Ensure that tracing doesn't break connections in 3.x/4.0 mixed mode by default (CASSANDRA-15385)
diff --cc src/java/org/apache/cassandra/db/BufferClustering.java
index df6a473,0000000..7ca9132
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/BufferClustering.java
+++ b/src/java/org/apache/cassandra/db/BufferClustering.java
@@@ -1,40 -1,0 +1,48 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+
++import org.apache.cassandra.utils.ByteBufferUtil;
++
+/**
+ * The clustering column values for a row.
+ * <p>
+ * A {@code Clustering} is a {@code ClusteringPrefix} that must always be "complete", i.e. have
+ * as many values as there is clustering columns in the table it is part of. It is the clustering
+ * prefix used by rows.
+ * <p>
+ * Note however that while it's size must be equal to the table clustering size, a clustering can have
+ * {@code null} values, and this mostly for thrift backward compatibility (in practice, if a value is null,
+ * all of the following ones will be too because that's what thrift allows, but it's never assumed by the
+ * code so we could start generally allowing nulls for clustering columns if we wanted to).
+ */
+public class BufferClustering extends AbstractBufferClusteringPrefix implements Clustering
+{
+ BufferClustering(ByteBuffer... values)
+ {
+ super(Kind.CLUSTERING, values);
+ }
++
++ public ClusteringPrefix minimize()
++ {
++ if (!ByteBufferUtil.canMinimize(values))
++ return this;
++ return new BufferClustering(ByteBufferUtil.minimizeBuffers(values)); }
+}
diff --cc src/java/org/apache/cassandra/db/ClusteringBound.java
index c45f7ba,0000000..8bfeb32
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/ClusteringBound.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBound.java
@@@ -1,171 -1,0 +1,179 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
++import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.memory.AbstractAllocator;
+
+/**
+ * The start or end of a range of clusterings, either inclusive or exclusive.
+ */
+public class ClusteringBound extends ClusteringBoundOrBoundary
+{
+ /** The smallest start bound, i.e. the one that starts before any row. */
+ public static final ClusteringBound BOTTOM = new ClusteringBound(Kind.INCL_START_BOUND, EMPTY_VALUES_ARRAY);
+ /** The biggest end bound, i.e. the one that ends after any row. */
+ public static final ClusteringBound TOP = new ClusteringBound(Kind.INCL_END_BOUND, EMPTY_VALUES_ARRAY);
+
+ protected ClusteringBound(Kind kind, ByteBuffer[] values)
+ {
+ super(kind, values);
+ }
+
+ public static ClusteringBound create(Kind kind, ByteBuffer[] values)
+ {
+ assert !kind.isBoundary();
+ return new ClusteringBound(kind, values);
+ }
+
+ public static Kind boundKind(boolean isStart, boolean isInclusive)
+ {
+ return isStart
+ ? (isInclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND)
+ : (isInclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND);
+ }
+
+ public static ClusteringBound inclusiveStartOf(ByteBuffer... values)
+ {
+ return create(Kind.INCL_START_BOUND, values);
+ }
+
+ public static ClusteringBound inclusiveEndOf(ByteBuffer... values)
+ {
+ return create(Kind.INCL_END_BOUND, values);
+ }
+
+ public static ClusteringBound exclusiveStartOf(ByteBuffer... values)
+ {
+ return create(Kind.EXCL_START_BOUND, values);
+ }
+
+ public static ClusteringBound exclusiveEndOf(ByteBuffer... values)
+ {
+ return create(Kind.EXCL_END_BOUND, values);
+ }
+
+ public static ClusteringBound inclusiveStartOf(ClusteringPrefix prefix)
+ {
+ ByteBuffer[] values = new ByteBuffer[prefix.size()];
+ for (int i = 0; i < prefix.size(); i++)
+ values[i] = prefix.get(i);
+ return inclusiveStartOf(values);
+ }
+
+ public static ClusteringBound exclusiveStartOf(ClusteringPrefix prefix)
+ {
+ ByteBuffer[] values = new ByteBuffer[prefix.size()];
+ for (int i = 0; i < prefix.size(); i++)
+ values[i] = prefix.get(i);
+ return exclusiveStartOf(values);
+ }
+
+ public static ClusteringBound inclusiveEndOf(ClusteringPrefix prefix)
+ {
+ ByteBuffer[] values = new ByteBuffer[prefix.size()];
+ for (int i = 0; i < prefix.size(); i++)
+ values[i] = prefix.get(i);
+ return inclusiveEndOf(values);
+ }
+
+ public static ClusteringBound create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values)
+ {
+ CBuilder builder = CBuilder.create(comparator);
+ for (Object val : values)
+ {
+ if (val instanceof ByteBuffer)
+ builder.add((ByteBuffer) val);
+ else
+ builder.add(val);
+ }
+ return builder.buildBound(isStart, isInclusive);
+ }
+
+ @Override
+ public ClusteringBound invert()
+ {
+ return create(kind().invert(), values);
+ }
+
+ public ClusteringBound copy(AbstractAllocator allocator)
+ {
+ return (ClusteringBound) super.copy(allocator);
+ }
+
++ public ClusteringPrefix minimize()
++ {
++ if (!ByteBufferUtil.canMinimize(values))
++ return this;
++ return new ClusteringBound(kind, ByteBufferUtil.minimizeBuffers(values));
++ }
++
+ public boolean isStart()
+ {
+ return kind().isStart();
+ }
+
+ public boolean isEnd()
+ {
+ return !isStart();
+ }
+
+ public boolean isInclusive()
+ {
+ return kind == Kind.INCL_START_BOUND || kind == Kind.INCL_END_BOUND;
+ }
+
+ public boolean isExclusive()
+ {
+ return kind == Kind.EXCL_START_BOUND || kind == Kind.EXCL_END_BOUND;
+ }
+
+ // For use by intersects, it's called with the sstable bound opposite to the slice bound
+ // (so if the slice bound is a start, it's call with the max sstable bound)
+ int compareTo(ClusteringComparator comparator, List<ByteBuffer> sstableBound)
+ {
+ for (int i = 0; i < sstableBound.size(); i++)
+ {
+ // Say the slice bound is a start. It means we're in the case where the max
+ // sstable bound is say (1:5) while the slice start is (1). So the start
+ // does start before the sstable end bound (and intersect it). It's the exact
+ // inverse with a end slice bound.
+ if (i >= size())
+ return isStart() ? -1 : 1;
+
+ int cmp = comparator.compareComponent(i, get(i), sstableBound.get(i));
+ if (cmp != 0)
+ return cmp;
+ }
+
+ // Say the slice bound is a start. I means we're in the case where the max
+ // sstable bound is say (1), while the slice start is (1:5). This again means
+ // that the slice start before the end bound.
+ if (size() > sstableBound.size())
+ return isStart() ? -1 : 1;
+
+ // The slice bound is equal to the sstable bound. Results depends on whether the slice is inclusive or not
+ return isInclusive() ? 0 : (isStart() ? 1 : -1);
+ }
+}
diff --cc src/java/org/apache/cassandra/db/ClusteringBoundary.java
index 37b3210,0000000..7a6842b
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/ClusteringBoundary.java
+++ b/src/java/org/apache/cassandra/db/ClusteringBoundary.java
@@@ -1,65 -1,0 +1,73 @@@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+
++import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.memory.AbstractAllocator;
+
+/**
+ * The threshold between two different ranges, i.e. a shortcut for the combination of two ClusteringBounds -- one
+ * specifying the end of one of the ranges, and its (implicit) complement specifying the beginning of the other.
+ */
+public class ClusteringBoundary extends ClusteringBoundOrBoundary
+{
+ protected ClusteringBoundary(Kind kind, ByteBuffer[] values)
+ {
+ super(kind, values);
+ }
+
+ public static ClusteringBoundary create(Kind kind, ByteBuffer[] values)
+ {
+ assert kind.isBoundary();
+ return new ClusteringBoundary(kind, values);
+ }
+
+ @Override
+ public ClusteringBoundary invert()
+ {
+ return create(kind().invert(), values);
+ }
+
+ @Override
+ public ClusteringBoundary copy(AbstractAllocator allocator)
+ {
+ return (ClusteringBoundary) super.copy(allocator);
+ }
+
++ public ClusteringPrefix minimize()
++ {
++ if (!ByteBufferUtil.canMinimize(values))
++ return this;
++ return new ClusteringBoundary(kind, ByteBufferUtil.minimizeBuffers(values));
++ }
++
+ public ClusteringBound openBound(boolean reversed)
+ {
+ return ClusteringBound.create(kind.openBoundOfBoundary(reversed), values);
+ }
+
+ public ClusteringBound closeBound(boolean reversed)
+ {
+ return ClusteringBound.create(kind.closeBoundOfBoundary(reversed), values);
+ }
+}
diff --cc src/java/org/apache/cassandra/db/NativeClustering.java
index 1943b71,0000000..e96435b
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/NativeClustering.java
+++ b/src/java/org/apache/cassandra/db/NativeClustering.java
@@@ -1,125 -1,0 +1,130 @@@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.MemoryUtil;
+import org.apache.cassandra.utils.memory.NativeAllocator;
+
+public class NativeClustering extends AbstractClusteringPrefix implements Clustering
+{
+ private static final long EMPTY_SIZE = ObjectSizes.measure(new NativeClustering());
+
+ private final long peer;
+
+ private NativeClustering() { peer = 0; }
+
+ public NativeClustering(NativeAllocator allocator, OpOrder.Group writeOp, Clustering clustering)
+ {
+ int count = clustering.size();
+ int metadataSize = (count * 2) + 4;
+ int dataSize = clustering.dataSize();
+ int bitmapSize = ((count + 7) >>> 3);
+
+ assert count < 64 << 10;
+ assert dataSize < 64 << 10;
+
+ peer = allocator.allocate(metadataSize + dataSize + bitmapSize, writeOp);
+ long bitmapStart = peer + metadataSize;
+ MemoryUtil.setShort(peer, (short) count);
+ MemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets
+
+ MemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
+ long dataStart = peer + metadataSize + bitmapSize;
+ int dataOffset = 0;
+ for (int i = 0 ; i < count ; i++)
+ {
+ MemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);
+
+ ByteBuffer value = clustering.get(i);
+ if (value == null)
+ {
+ long boffset = bitmapStart + (i >>> 3);
+ int b = MemoryUtil.getByte(boffset);
+ b |= 1 << (i & 7);
+ MemoryUtil.setByte(boffset, (byte) b);
+ continue;
+ }
+
+ assert value.order() == ByteOrder.BIG_ENDIAN;
+
+ int size = value.remaining();
+ MemoryUtil.setBytes(dataStart + dataOffset, value);
+ dataOffset += size;
+ }
+ }
+
+ public Kind kind()
+ {
+ return Kind.CLUSTERING;
+ }
+
+ public int size()
+ {
+ return MemoryUtil.getShort(peer);
+ }
+
+ public ByteBuffer get(int i)
+ {
+ // offset at which we store the dataOffset
+ int size = size();
+ if (i >= size)
+ throw new IndexOutOfBoundsException();
+
+ int metadataSize = (size * 2) + 4;
+ int bitmapSize = ((size + 7) >>> 3);
+ long bitmapStart = peer + metadataSize;
+ int b = MemoryUtil.getByte(bitmapStart + (i >>> 3));
+ if ((b & (1 << (i & 7))) != 0)
+ return null;
+
+ int startOffset = MemoryUtil.getShort(peer + 2 + i * 2);
+ int endOffset = MemoryUtil.getShort(peer + 4 + i * 2);
+ return MemoryUtil.getByteBuffer(bitmapStart + bitmapSize + startOffset,
+ endOffset - startOffset,
+ ByteOrder.BIG_ENDIAN);
+ }
+
+ public ByteBuffer[] getRawValues()
+ {
+ ByteBuffer[] values = new ByteBuffer[size()];
+ for (int i = 0 ; i < values.length ; i++)
+ values[i] = get(i);
+ return values;
+ }
+
+ public long unsharedHeapSize()
+ {
+ return EMPTY_SIZE;
+ }
+
+ public long unsharedHeapSizeExcludingData()
+ {
+ return EMPTY_SIZE;
+ }
++
++ public ClusteringPrefix minimize()
++ {
++ return this;
++ }
+}
diff --cc src/java/org/apache/cassandra/db/RangeTombstone.java
index 8e01b8e,4a26581..eb9725a
--- a/src/java/org/apache/cassandra/db/RangeTombstone.java
+++ b/src/java/org/apache/cassandra/db/RangeTombstone.java
@@@ -17,11 -17,19 +17,10 @@@
*/
package org.apache.cassandra.db;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
import java.util.Objects;
-import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.RangeTombstoneMarker;
-import org.apache.cassandra.io.util.DataInputPlus;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.memory.AbstractAllocator;
-
-
/**
* A range tombstone is a tombstone that covers a slice/range of rows.
* <p>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org