You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2018/11/29 14:54:51 UTC
[8/9] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1084ad9e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1084ad9e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1084ad9e
Branch: refs/heads/trunk
Commit: 1084ad9ee62403b663e0f1b5d353ddbe318a4315
Parents: 4a531c2 7bdea44
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Thu Nov 29 14:38:01 2018 +0000
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Thu Nov 29 14:38:01 2018 +0000
----------------------------------------------------------------------
CHANGES.txt | 2 +
src/java/org/apache/cassandra/db/Columns.java | 2 +
.../cassandra/db/SerializationHeader.java | 45 +++---
.../org/apache/cassandra/db/ColumnsTest.java | 29 +++-
.../cassandra/db/SerializationHeaderTest.java | 136 +++++++++++++++++++
5 files changed, 194 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1084ad9e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ebcf1b2,9ba36fa..c228e8f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,8 -1,8 +1,10 @@@
-3.0.18
+3.11.4
+ * Correct sstable sorting for garbagecollect and levelled compaction (CASSANDRA-14870)
+Merged from 3.0:
* RangeTombstoneList doesn't properly clean up mergeable or superseded rts in some cases (CASSANDRA-14894)
* Fix handling of collection tombstones for dropped columns from legacy sstables (CASSANDRA-14912)
+ * Throw exception if Columns serialized subset encode more columns than possible (CASSANDRA-14591)
+ * Drop/add column name with different Kind can result in corruption (CASSANDRA-14843)
* Fix missing rows when reading 2.1 SSTables with static columns in 3.0 (CASSANDRA-14873)
* Move TWCS message 'No compaction necessary for bucket size' to Trace level (CASSANDRA-14884)
* Sstable min/max metadata can cause data loss (CASSANDRA-14861)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1084ad9e/src/java/org/apache/cassandra/db/Columns.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1084ad9e/src/java/org/apache/cassandra/db/SerializationHeader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/SerializationHeader.java
index 4e1d60e,5c4f518..8e2844b
--- a/src/java/org/apache/cassandra/db/SerializationHeader.java
+++ b/src/java/org/apache/cassandra/db/SerializationHeader.java
@@@ -21,6 -21,9 +21,7 @@@ import java.io.IOException
import java.nio.ByteBuffer;
import java.util.*;
+ import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.filter.ColumnFilter;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1084ad9e/test/unit/org/apache/cassandra/db/ColumnsTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1084ad9e/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
index 0000000,3e9f3bc..84fb51c
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
@@@ -1,0 -1,129 +1,136 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.cassandra.db;
+
+ import com.google.common.io.Files;
+ import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.ColumnDefinition;
++import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.cql3.ColumnIdentifier;
+ import org.apache.cassandra.db.compaction.OperationType;
+ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+ import org.apache.cassandra.db.marshal.Int32Type;
+ import org.apache.cassandra.db.partitions.PartitionUpdate;
+ import org.apache.cassandra.db.rows.BTreeRow;
+ import org.apache.cassandra.db.rows.BufferCell;
+ import org.apache.cassandra.db.rows.Cell;
+ import org.apache.cassandra.db.rows.Row;
+ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.ISSTableScanner;
+ import org.apache.cassandra.io.sstable.format.SSTableFormat;
+ import org.apache.cassandra.io.sstable.format.SSTableReader;
+ import org.apache.cassandra.io.sstable.format.SSTableWriter;
+ import org.apache.cassandra.io.sstable.format.big.BigFormat;
+ import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
+ import org.apache.cassandra.io.util.FileUtils;
+ import org.junit.Assert;
+ import org.junit.Test;
+
+ import java.io.File;
+ import java.nio.ByteBuffer;
++import java.util.Collections;
+ import java.util.concurrent.Callable;
+ import java.util.concurrent.atomic.AtomicInteger;
+ import java.util.function.BiFunction;
+ import java.util.function.Function;
+
+ public class SerializationHeaderTest
+ {
+ private static String KEYSPACE = "SerializationHeaderTest";
+
++ static
++ {
++ DatabaseDescriptor.daemonInitialization();
++ }
++
+ @Test
+ public void testWrittenAsDifferentKind() throws Exception
+ {
+ final String tableName = "testWrittenAsDifferentKind";
+ final String schemaCqlWithStatic = String.format("CREATE TABLE %s (k int, c int, v int static, PRIMARY KEY(k, c))", tableName);
+ final String schemaCqlWithRegular = String.format("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))", tableName);
+ ColumnIdentifier v = ColumnIdentifier.getInterned("v", false);
+ CFMetaData schemaWithStatic = CFMetaData.compile(schemaCqlWithStatic, KEYSPACE);
+ CFMetaData schemaWithRegular = CFMetaData.compile(schemaCqlWithRegular, KEYSPACE);
+ ColumnDefinition columnStatic = schemaWithStatic.getColumnDefinition(v);
+ ColumnDefinition columnRegular = schemaWithRegular.getColumnDefinition(v);
+ schemaWithStatic.recordColumnDrop(columnRegular, 0L);
+ schemaWithRegular.recordColumnDrop(columnStatic, 0L);
+
+ final AtomicInteger generation = new AtomicInteger();
+ File dir = Files.createTempDir();
+ try
+ {
+ BiFunction<CFMetaData, Function<ByteBuffer, Clustering>, Callable<Descriptor>> writer = (schema, clusteringFunction) -> () -> {
+ Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.ksName, schema.cfName, generation.incrementAndGet(), SSTableFormat.Type.BIG, Component.DIGEST_CRC32);
+
+ SerializationHeader header = SerializationHeader.makeWithoutStats(schema);
+ try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
- SSTableWriter sstableWriter = BigTableWriter.create(schema, descriptor, 1, 0L, 0, header, txn))
++ SSTableWriter sstableWriter = BigTableWriter.create(schema, descriptor, 1, 0L, 0, header, Collections.emptyList(), txn))
+ {
+ ColumnDefinition cd = schema.getColumnDefinition(v);
+ for (int i = 0 ; i < 5 ; ++i) {
+ final ByteBuffer value = Int32Type.instance.decompose(i);
- Cell cell = BufferCell.live(schema, cd, 1L, value);
++ Cell cell = BufferCell.live(cd, 1L, value);
+ Clustering clustering = clusteringFunction.apply(value);
+ Row row = BTreeRow.singleCellRow(clustering, cell);
+ sstableWriter.append(PartitionUpdate.singleRowUpdate(schema, value, row).unfilteredIterator());
+ }
+ sstableWriter.finish(false);
+ txn.finish();
+ }
+ return descriptor;
+ };
+
- Descriptor sstableWithRegular = writer.apply(schemaWithRegular, Clustering::new).call();
++ Descriptor sstableWithRegular = writer.apply(schemaWithRegular, BufferClustering::new).call();
+ Descriptor sstableWithStatic = writer.apply(schemaWithStatic, value -> Clustering.STATIC_CLUSTERING).call();
+ SSTableReader readerWithStatic = SSTableReader.openNoValidation(sstableWithStatic, schemaWithRegular);
+ SSTableReader readerWithRegular = SSTableReader.openNoValidation(sstableWithRegular, schemaWithStatic);
+
+ try (ISSTableScanner partitions = readerWithStatic.getScanner()) {
+ for (int i = 0 ; i < 5 ; ++i)
+ {
+ UnfilteredRowIterator partition = partitions.next();
+ Assert.assertFalse(partition.hasNext());
+ long value = Int32Type.instance.compose(partition.staticRow().getCell(columnStatic).value());
+ Assert.assertEquals(value, (long)i);
+ }
+ Assert.assertFalse(partitions.hasNext());
+ }
+ try (ISSTableScanner partitions = readerWithRegular.getScanner()) {
+ for (int i = 0 ; i < 5 ; ++i)
+ {
+ UnfilteredRowIterator partition = partitions.next();
+ long value = Int32Type.instance.compose(((Row)partition.next()).getCell(columnRegular).value());
+ Assert.assertEquals(value, (long)i);
+ Assert.assertTrue(partition.staticRow().isEmpty());
+ Assert.assertFalse(partition.hasNext());
+ }
+ Assert.assertFalse(partitions.hasNext());
+ }
+ }
+ finally
+ {
+ FileUtils.deleteRecursive(dir);
+ }
+ }
+
+ }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org