You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2017/08/08 14:27:41 UTC
[5/8] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.11
Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/47a2839b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/47a2839b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/47a2839b
Branch: refs/heads/trunk
Commit: 47a2839bf094a7fa9bad6de140fba486756c49bf
Parents: 15abe2d 3960260
Author: Andrés de la Peña <a....@gmail.com>
Authored: Tue Aug 8 15:01:04 2017 +0100
Committer: Andrés de la Peña <a....@gmail.com>
Committed: Tue Aug 8 15:01:04 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/config/ColumnDefinition.java | 19 ++++++++---
.../apache/cassandra/db/rows/AbstractCell.java | 2 +-
.../org/apache/cassandra/db/rows/BTreeRow.java | 2 +-
.../apache/cassandra/tools/JsonTransformer.java | 31 ++++++++++++++++--
.../org/apache/cassandra/cql3/ViewTest.java | 33 ++++++++++++++++++++
.../cassandra/index/sasi/SASIIndexTest.java | 14 +++++++++
7 files changed, 94 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b8c4bde,1525289..b778df6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,5 +1,10 @@@
-3.0.15
+3.11.1
+ * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
+ * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
+ * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
+ * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
+Merged from 3.0:
+ * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump to use type.toJSONString() (CASSANDRA-13573)
* Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
* Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
* Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/config/ColumnDefinition.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/ColumnDefinition.java
index ea508d2,6a0f530..159ea0c
--- a/src/java/org/apache/cassandra/config/ColumnDefinition.java
+++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java
@@@ -446,197 -398,17 +448,206 @@@ public class ColumnDefinition extends C
*/
public AbstractType<?> cellValueType()
{
- return type instanceof CollectionType
- ? ((CollectionType)type).valueComparator()
- : type;
+ assert !(type instanceof UserType && type.isMultiCell());
+ return type instanceof CollectionType && type.isMultiCell()
+ ? ((CollectionType)type).valueComparator()
+ : type;
+ }
+
+
+ public boolean isCounterColumn()
+ {
+ if (type instanceof CollectionType) // for thrift
+ return ((CollectionType) type).valueComparator().isCounter();
+ return type.isCounter();
}
+
+ public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType, List<ColumnDefinition> defs, VariableSpecifications boundNames) throws InvalidRequestException
+ {
+ return SimpleSelector.newFactory(this, addAndGetIndex(this, defs));
+ }
+
+ public AbstractType<?> getExactTypeIfKnown(String keyspace)
+ {
+ return type;
+ }
+
+ /**
+ * Because Thrift-created tables may have a non-text comparator, we cannot determine the proper 'key' until
+ * we know the comparator. ColumnDefinition.Raw is a placeholder that can be converted to a real ColumnIdentifier
+ * once the comparator is known with prepare(). This should only be used with identifiers that are actual
+ * column names. See CASSANDRA-8178 for more background.
+ */
+ public static abstract class Raw extends Selectable.Raw
+ {
+ /**
+ * Creates a {@code ColumnDefinition.Raw} from an unquoted identifier string.
+ */
+ public static Raw forUnquoted(String text)
+ {
+ return new Literal(text, false);
+ }
+
+ /**
+ * Creates a {@code ColumnDefinition.Raw} from a quoted identifier string.
+ */
+ public static Raw forQuoted(String text)
+ {
+ return new Literal(text, true);
+ }
+
+ /**
+ * Creates a {@code ColumnDefinition.Raw} from a pre-existing {@code ColumnDefinition}
+ * (useful in the rare cases where we already have the column but need
+ * a {@code ColumnDefinition.Raw} for typing purposes).
+ */
+ public static Raw forColumn(ColumnDefinition column)
+ {
+ return new ForColumn(column);
+ }
+
+ /**
+ * Get the identifier corresponding to this raw column, without assuming this is an
+ * existing column (unlike {@link #prepare}).
+ */
+ public abstract ColumnIdentifier getIdentifier(CFMetaData cfm);
+
+ public abstract String rawText();
+
+ @Override
+ public abstract ColumnDefinition prepare(CFMetaData cfm);
+
+ @Override
+ public boolean processesSelection()
+ {
+ return false;
+ }
+
+ @Override
+ public final int hashCode()
+ {
+ return toString().hashCode();
+ }
+
+ @Override
+ public final boolean equals(Object o)
+ {
+ if(!(o instanceof Raw))
+ return false;
+
+ Raw that = (Raw)o;
+ return this.toString().equals(that.toString());
+ }
+
+ private static class Literal extends Raw
+ {
+ private final String text;
+
+ public Literal(String rawText, boolean keepCase)
+ {
+ this.text = keepCase ? rawText : rawText.toLowerCase(Locale.US);
+ }
+
+ public ColumnIdentifier getIdentifier(CFMetaData cfm)
+ {
+ if (!cfm.isStaticCompactTable())
+ return ColumnIdentifier.getInterned(text, true);
+
+ AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
+ if (thriftColumnNameType instanceof UTF8Type)
+ return ColumnIdentifier.getInterned(text, true);
+
+ // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use
+ // thriftColumnNameType
+ ByteBuffer bufferName = ByteBufferUtil.bytes(text);
+ for (ColumnDefinition def : cfm.allColumns())
+ {
+ if (def.name.bytes.equals(bufferName))
+ return def.name;
+ }
+ return ColumnIdentifier.getInterned(thriftColumnNameType, thriftColumnNameType.fromString(text), text);
+ }
+
+ public ColumnDefinition prepare(CFMetaData cfm)
+ {
+ if (!cfm.isStaticCompactTable())
+ return find(cfm);
+
+ AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
+ if (thriftColumnNameType instanceof UTF8Type)
+ return find(cfm);
+
+ // We have a Thrift-created table with a non-text comparator. Check if we have a match column, otherwise assume we should use
+ // thriftColumnNameType
+ ByteBuffer bufferName = ByteBufferUtil.bytes(text);
+ for (ColumnDefinition def : cfm.allColumns())
+ {
+ if (def.name.bytes.equals(bufferName))
+ return def;
+ }
+ return find(thriftColumnNameType.fromString(text), cfm);
+ }
+
+ private ColumnDefinition find(CFMetaData cfm)
+ {
+ return find(ByteBufferUtil.bytes(text), cfm);
+ }
+
+ private ColumnDefinition find(ByteBuffer id, CFMetaData cfm)
+ {
+ ColumnDefinition def = cfm.getColumnDefinition(id);
+ if (def == null)
+ throw new InvalidRequestException(String.format("Undefined column name %s", toString()));
+ return def;
+ }
+
+ public String rawText()
+ {
+ return text;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ColumnIdentifier.maybeQuote(text);
+ }
+ }
+
+ // Use internally in the rare case where we need a ColumnDefinition.Raw for type-checking but
+ // actually already have the column itself.
+ private static class ForColumn extends Raw
+ {
+ private final ColumnDefinition column;
+
+ private ForColumn(ColumnDefinition column)
+ {
+ this.column = column;
+ }
+
+ public ColumnIdentifier getIdentifier(CFMetaData cfm)
+ {
+ return column.name;
+ }
+
+ public ColumnDefinition prepare(CFMetaData cfm)
+ {
+ assert cfm.getColumnDefinition(column.name) != null; // Sanity check that we're not doing something crazy
+ return column;
+ }
+
+ public String rawText()
+ {
+ return column.name.toString();
+ }
+
+ @Override
+ public String toString()
+ {
+ return column.name.toCQLString();
+ }
+ }
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/rows/AbstractCell.java
index 54bd9e8,7e93c2e..54c8f24
--- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
@@@ -44,81 -40,6 +44,81 @@@ public abstract class AbstractCell exte
super(column);
}
+ public boolean isCounterCell()
+ {
- return !isTombstone() && column.cellValueType().isCounter();
++ return !isTombstone() && column.isCounterColumn();
+ }
+
+ public boolean isLive(int nowInSec)
+ {
+ return localDeletionTime() == NO_DELETION_TIME || (ttl() != NO_TTL && nowInSec < localDeletionTime());
+ }
+
+ public boolean isTombstone()
+ {
+ return localDeletionTime() != NO_DELETION_TIME && ttl() == NO_TTL;
+ }
+
+ public boolean isExpiring()
+ {
+ return ttl() != NO_TTL;
+ }
+
+ public Cell markCounterLocalToBeCleared()
+ {
+ if (!isCounterCell())
+ return this;
+
+ ByteBuffer value = value();
+ ByteBuffer marked = CounterContext.instance().markLocalToBeCleared(value);
+ return marked == value ? this : new BufferCell(column, timestamp(), ttl(), localDeletionTime(), marked, path());
+ }
+
+ public Cell purge(DeletionPurger purger, int nowInSec)
+ {
+ if (!isLive(nowInSec))
+ {
+ if (purger.shouldPurge(timestamp(), localDeletionTime()))
+ return null;
+
+ // We slightly hijack purging to convert expired but not purgeable columns to tombstones. The reason we do that is
+ // that once a column has expired it is equivalent to a tombstone but actually using a tombstone is more compact since
+ // we don't keep the column value. The reason we do it here is that 1) it's somewhat related to dealing with tombstones
+ // so hopefully not too surprising and 2) we want to this and purging at the same places, so it's simpler/more efficient
+ // to do both here.
+ if (isExpiring())
+ {
+ // Note that as long as the expiring column and the tombstone put together live longer than GC grace seconds,
+ // we'll fulfil our responsibility to repair. See discussion at
+ // http://cassandra-user-incubator-apache-org.3065146.n2.nabble.com/repair-compaction-and-tombstone-rows-td7583481.html
+ return BufferCell.tombstone(column, timestamp(), localDeletionTime() - ttl(), path()).purge(purger, nowInSec);
+ }
+ }
+ return this;
+ }
+
+ public Cell copy(AbstractAllocator allocator)
+ {
+ CellPath path = path();
+ return new BufferCell(column, timestamp(), ttl(), localDeletionTime(), allocator.clone(value()), path == null ? null : path.copy(allocator));
+ }
+
+ // note: while the cell returned may be different, the value is the same, so if the value is offheap it must be referenced inside a guarded context (or copied)
+ public Cell updateAllTimestamp(long newTimestamp)
+ {
+ return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(), localDeletionTime(), value(), path());
+ }
+
+ public int dataSize()
+ {
+ CellPath path = path();
+ return TypeSizes.sizeof(timestamp())
+ + TypeSizes.sizeof(ttl())
+ + TypeSizes.sizeof(localDeletionTime())
+ + value().remaining()
+ + (path == null ? 0 : path.dataSize());
+ }
+
public void digest(MessageDigest digest)
{
digest.update(value().duplicate());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/tools/JsonTransformer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/JsonTransformer.java
index 360e8ff,5c32035..c3a0a17
--- a/src/java/org/apache/cassandra/tools/JsonTransformer.java
+++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java
@@@ -46,6 -50,7 +47,7 @@@ import org.apache.cassandra.db.rows.Row
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
-import org.apache.cassandra.transport.Server;
++import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
@@@ -405,9 -411,9 +407,10 @@@ public final class JsonTransforme
objectIndenter.setCompact(true);
json.writeFieldName("name");
AbstractType<?> type = cell.column().type;
++ AbstractType<?> cellType = null;
json.writeString(cell.column().name.toCQLString());
- if (cell.path() != null && cell.path().size() > 0)
+ if (type.isCollection() && type.isMultiCell()) // non-frozen collection
{
CollectionType ct = (CollectionType) type;
json.writeFieldName("path");
@@@ -419,6 -425,6 +422,30 @@@
}
json.writeEndArray();
arrayIndenter.setCompact(false);
++
++ cellType = cell.column().cellValueType();
++ }
++ else if (type.isUDT() && type.isMultiCell()) // non-frozen udt
++ {
++ UserType ut = (UserType) type;
++ json.writeFieldName("path");
++ arrayIndenter.setCompact(true);
++ json.writeStartArray();
++ for (int i = 0; i < cell.path().size(); i++)
++ {
++ Short fieldPosition = ut.nameComparator().compose(cell.path().get(i));
++ json.writeString(ut.fieldNameAsString(fieldPosition));
++ }
++ json.writeEndArray();
++ arrayIndenter.setCompact(false);
++
++ // cellType of udt
++ Short fieldPosition = ((UserType) type).nameComparator().compose(cell.path().get(0));
++ cellType = ((UserType) type).fieldType(fieldPosition);
++ }
++ else
++ {
++ cellType = cell.column().cellValueType();
}
if (cell.isTombstone())
{
@@@ -433,7 -439,7 +460,7 @@@
else
{
json.writeFieldName("value");
- json.writeString(cell.column().cellValueType().getString(cell.value()));
- json.writeRawValue(cell.column().cellValueType().toJSONString(cell.value(), Server.CURRENT_VERSION));
++ json.writeRawValue(cellType.toJSONString(cell.value(), ProtocolVersion.CURRENT));
}
if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/test/unit/org/apache/cassandra/cql3/ViewTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/ViewTest.java
index 6f6e04d,f8f8c9f..0853562
--- a/test/unit/org/apache/cassandra/cql3/ViewTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java
@@@ -777,6 -788,6 +777,39 @@@ public class ViewTest extends CQLTeste
}
@Test
++ public void testFrozenCollectionsWithComplicatedInnerType() throws Throwable
++ {
++ createTable("CREATE TABLE %s (k int, intval int, listval frozen<list<tuple<text,text>>>, PRIMARY KEY (k))");
++
++ execute("USE " + keyspace());
++ executeNet(protocolVersion, "USE " + keyspace());
++
++ createView("mv",
++ "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND listval IS NOT NULL PRIMARY KEY (k, listval)");
++
++ updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))",
++ 0,
++ 0,
++ "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
++
++ // verify input
++ assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0),
++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++ assertRows(execute("SELECT k, listval from mv"),
++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++
++ // update listval with the same value and it will be compared in view generator
++ updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))",
++ 0,
++ "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
++ // verify result
++ assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0),
++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++ assertRows(execute("SELECT k, listval from mv"),
++ row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++ }
++
++ @Test
public void testUpdate() throws Throwable
{
createTable("CREATE TABLE %s (" +
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org