You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sl...@apache.org on 2015/06/30 12:47:32 UTC
[08/51] [partial] cassandra git commit: Storage engine refactor,
a.k.a CASSANDRA-8099
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftConversion.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java
index adb925e..148fd68 100644
--- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java
+++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java
@@ -23,31 +23,35 @@ import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.cassandra.cache.CachingOptions;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.ColumnFamilyType;
-import org.apache.cassandra.schema.LegacySchemaTables;
-import org.apache.cassandra.db.WriteType;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.composites.CellNames;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.io.compress.CompressionParameters;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.schema.LegacySchemaTables;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
/**
* Static utility methods to convert internal structure to and from thrift ones.
+ *
+ *
*/
public class ThriftConversion
{
+ private static final Logger logger = LoggerFactory.getLogger(ThriftConversion.class);
+
public static org.apache.cassandra.db.ConsistencyLevel fromThrift(ConsistencyLevel cl)
{
switch (cl)
@@ -136,21 +140,14 @@ public class ThriftConversion
return new TimedOutException();
}
- public static List<org.apache.cassandra.db.IndexExpression> indexExpressionsFromThrift(List<IndexExpression> exprs)
+ public static RowFilter rowFilterFromThrift(CFMetaData metadata, List<IndexExpression> exprs)
{
- if (exprs == null)
- return null;
-
- if (exprs.isEmpty())
- return Collections.emptyList();
+ if (exprs == null || exprs.isEmpty())
+ return RowFilter.NONE;
- List<org.apache.cassandra.db.IndexExpression> converted = new ArrayList<>(exprs.size());
+ RowFilter converted = RowFilter.forThrift(exprs.size());
for (IndexExpression expr : exprs)
- {
- converted.add(new org.apache.cassandra.db.IndexExpression(expr.column_name,
- Operator.valueOf(expr.op.name()),
- expr.value));
- }
+ converted.addThriftExpression(metadata, expr.column_name, Operator.valueOf(expr.op.name()), expr.value);
return converted;
}
@@ -184,50 +181,75 @@ public class ThriftConversion
public static CFMetaData fromThrift(CfDef cf_def)
throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
{
- return internalFromThrift(cf_def, Collections.<ColumnDefinition>emptyList());
+ // This is a creation: the table is dense if it doesn't define any column_metadata
+ boolean isDense = cf_def.column_metadata == null || cf_def.column_metadata.isEmpty();
+ return internalFromThrift(cf_def, true, Collections.<ColumnDefinition>emptyList(), isDense);
}
public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate)
throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
{
- return internalFromThrift(cf_def, toUpdate.allColumns());
+ return internalFromThrift(cf_def, false, toUpdate.allColumns(), toUpdate.isDense());
}
- // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild
- private static CFMetaData internalFromThrift(CfDef cf_def, Collection<ColumnDefinition> previousCQLMetadata)
- throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
+ private static boolean isSuper(String thriftColumnType)
+ throws org.apache.cassandra.exceptions.InvalidRequestException
{
- ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type);
- if (cfType == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + cf_def.column_type);
+ switch (thriftColumnType.toLowerCase())
+ {
+ case "standard": return false;
+ case "super": return true;
+ default: throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + thriftColumnType);
+ }
+ }
+ /**
+ * Convert a thrift CfDef.
+ * <p>,
+ * This is used both for creation and update of CF.
+ *
+ * @param cf_def the thrift CfDef to convert.
+ * @param isCreation whether that is a new table creation or not.
+ * @param previousCQLMetadata if it is not a table creation, the previous
+ * definitions of the tables (which we use to preserve the CQL metadata).
+ * If it is a table creation, this will be empty.
+ * @param isDense whether the table is dense or not.
+ *
+ * @return the converted table definition.
+ */
+ private static CFMetaData internalFromThrift(CfDef cf_def,
+ boolean isCreation,
+ Collection<ColumnDefinition> previousCQLMetadata,
+ boolean isDense)
+ throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
+ {
applyImplicitDefaults(cf_def);
try
{
+ boolean isSuper = isSuper(cf_def.column_type);
AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type);
- AbstractType<?> subComparator = cfType == ColumnFamilyType.Standard
- ? null
- : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type);
+ AbstractType<?> subComparator = isSuper
+ ? cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type)
+ : null;
- AbstractType<?> fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator);
+ AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : BytesType.instance;
+ AbstractType<?> defaultValidator = TypeParser.parse(cf_def.default_validation_class);
- AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null;
-
- // Convert the REGULAR definitions from the input CfDef
+ // Convert the definitions from the input CfDef
List<ColumnDefinition> defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata);
- // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for
+ // Add the keyAlias if there is one, since that's a CQL metadata that thrift can actually change (for
// historical reasons)
boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType);
if (hasKeyAlias)
- defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null));
+ defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, UTF8Type.instance.getString(cf_def.key_alias), keyValidator, null));
// Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one
for (ColumnDefinition def : previousCQLMetadata)
{
// isPartOfCellName basically means 'is not just a CQL metadata'
- if (def.isPartOfCellName())
+ if (def.isPartOfCellName(false, isSuper))
continue;
if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias)
@@ -236,18 +258,25 @@ public class ThriftConversion
defs.add(def);
}
- CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, CFMetaData.calculateIsDense(fullRawComparator, defs));
-
UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name);
if (cfId == null)
cfId = UUIDGen.getTimeUUID();
- CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId);
+ boolean isCompound = isSuper ? false : (rawComparator instanceof CompositeType);
+ boolean isCounter = defaultValidator instanceof CounterColumnType;
- newCFMD.addAllColumnDefinitions(defs);
+ // If it's a thrift table creation, adds the default CQL metadata for the new table
+ if (isCreation)
+ addDefaultCQLMetadata(defs,
+ cf_def.keyspace,
+ cf_def.name,
+ hasKeyAlias ? null : keyValidator,
+ rawComparator,
+ subComparator,
+ defaultValidator);
+
+ CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, defs);
- if (keyValidator != null)
- newCFMD.keyValidator(keyValidator);
if (cf_def.isSetGc_grace_seconds())
newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds);
if (cf_def.isSetMin_compaction_threshold())
@@ -280,9 +309,7 @@ public class ThriftConversion
newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers));
return newCFMD.comment(cf_def.comment)
- .defaultValidator(TypeParser.parse(cf_def.default_validation_class))
- .compressionParameters(CompressionParameters.create(cf_def.compression_options))
- .rebuild();
+ .compressionParameters(CompressionParameters.create(cf_def.compression_options));
}
catch (SyntaxException | MarshalException e)
{
@@ -290,6 +317,48 @@ public class ThriftConversion
}
}
+ private static void addDefaultCQLMetadata(Collection<ColumnDefinition> defs,
+ String ks,
+ String cf,
+ AbstractType<?> keyValidator,
+ AbstractType<?> comparator,
+ AbstractType<?> subComparator,
+ AbstractType<?> defaultValidator)
+ {
+ CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs);
+ if (keyValidator != null)
+ {
+ if (keyValidator instanceof CompositeType)
+ {
+ List<AbstractType<?>> subTypes = ((CompositeType)keyValidator).types;
+ for (int i = 0; i < subTypes.size(); i++)
+ defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), subTypes.get(i), i));
+ }
+ else
+ {
+ defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), keyValidator, null));
+ }
+ }
+
+ if (subComparator != null)
+ {
+ // SuperColumn tables: we use a special map to hold dynamic values within a given super column
+ defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), comparator, 0));
+ defs.add(ColumnDefinition.regularDef(ks, cf, CompactTables.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true), null));
+ }
+ else
+ {
+ List<AbstractType<?>> subTypes = comparator instanceof CompositeType
+ ? ((CompositeType)comparator).types
+ : Collections.<AbstractType<?>>singletonList(comparator);
+
+ for (int i = 0; i < subTypes.size(); i++)
+ defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), subTypes.get(i), i));
+
+ defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator, null));
+ }
+ }
+
/** applies implicit defaults to cf definition. useful in updates */
private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def)
{
@@ -355,30 +424,30 @@ public class ThriftConversion
public static CfDef toThrift(CFMetaData cfm)
{
CfDef def = new CfDef(cfm.ksName, cfm.cfName);
- def.setColumn_type(cfm.cfType.name());
+ def.setColumn_type(cfm.isSuper() ? "Super" : "Standard");
if (cfm.isSuper())
{
def.setComparator_type(cfm.comparator.subtype(0).toString());
- def.setSubcomparator_type(cfm.comparator.subtype(1).toString());
+ def.setSubcomparator_type(cfm.thriftColumnNameType().toString());
}
else
{
- def.setComparator_type(cfm.comparator.toString());
+ def.setComparator_type(LegacyLayout.makeLegacyComparator(cfm).toString());
}
def.setComment(Strings.nullToEmpty(cfm.getComment()));
def.setRead_repair_chance(cfm.getReadRepairChance());
def.setDclocal_read_repair_chance(cfm.getDcLocalReadRepairChance());
def.setGc_grace_seconds(cfm.getGcGraceSeconds());
- def.setDefault_validation_class(cfm.getDefaultValidator().toString());
+ def.setDefault_validation_class(cfm.makeLegacyDefaultValidator().toString());
def.setKey_validation_class(cfm.getKeyValidator().toString());
def.setMin_compaction_threshold(cfm.getMinCompactionThreshold());
def.setMax_compaction_threshold(cfm.getMaxCompactionThreshold());
// We only return the alias if only one is set since thrift don't know about multiple key aliases
if (cfm.partitionKeyColumns().size() == 1)
def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes);
- def.setColumn_metadata(columnDefinitionsToThrift(cfm.allColumns()));
+ def.setColumn_metadata(columnDefinitionsToThrift(cfm, cfm.allColumns()));
def.setCompaction_strategy(cfm.compactionStrategyClass.getName());
def.setCompaction_strategy_options(new HashMap<>(cfm.compactionStrategyOptions));
def.setCompression_options(cfm.compressionParameters.asThriftOptions());
@@ -402,8 +471,9 @@ public class ThriftConversion
ColumnDef thriftColumnDef)
throws SyntaxException, ConfigurationException
{
+ boolean isSuper = thriftSubcomparator != null;
// For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component.
- Integer componentIndex = thriftSubcomparator != null ? 1 : null;
+ Integer componentIndex = isSuper ? 1 : null;
AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator;
try
{
@@ -414,15 +484,18 @@ public class ThriftConversion
throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator));
}
+ // In our generic layout, we store thrift defined columns as static, but this doesn't work for super columns so we
+ // use a regular definition (and "dynamic" columns are handled in a map).
+ ColumnDefinition.Kind kind = isSuper ? ColumnDefinition.Kind.REGULAR : ColumnDefinition.Kind.STATIC;
return new ColumnDefinition(ksName,
cfName,
- new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator),
+ ColumnIdentifier.getInterned(ByteBufferUtil.clone(thriftColumnDef.name), comparator),
TypeParser.parse(thriftColumnDef.validation_class),
thriftColumnDef.index_type == null ? null : org.apache.cassandra.config.IndexType.valueOf(thriftColumnDef.index_type.name()),
thriftColumnDef.index_options,
thriftColumnDef.index_name,
componentIndex,
- ColumnDefinition.Kind.REGULAR);
+ kind);
}
private static List<ColumnDefinition> fromThrift(String ksName,
@@ -456,11 +529,11 @@ public class ThriftConversion
return cd;
}
- private static List<ColumnDef> columnDefinitionsToThrift(Collection<ColumnDefinition> columns)
+ private static List<ColumnDef> columnDefinitionsToThrift(CFMetaData metadata, Collection<ColumnDefinition> columns)
{
List<ColumnDef> thriftDefs = new ArrayList<>(columns.size());
for (ColumnDefinition def : columns)
- if (def.kind == ColumnDefinition.Kind.REGULAR)
+ if (def.isPartOfCellName(metadata.isCQLTable(), metadata.isSuper()))
thriftDefs.add(ThriftConversion.toThrift(def));
return thriftDefs;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
new file mode 100644
index 0000000..ccb6e74
--- /dev/null
+++ b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.thrift;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.partitions.*;
+
+/**
+ * Given an iterator on a partition of a compact table, this return an iterator that merges the
+ * static row columns with the other results.
+ *
+ * Compact tables stores thrift column_metadata as static columns (see CompactTables for
+ * details). When reading for thrift however, we want to merge those static values with other
+ * results because:
+ * 1) on thrift, all "columns" are sorted together, whether or not they are declared
+ * column_metadata.
+ * 2) it's possible that a table add a value for a "dynamic" column, and later that column
+ * is statically defined. Merging "static" and "dynamic" columns make sure we don't miss
+ * a value prior to the column declaration.
+ *
+ * For example, if a thrift table declare 2 columns "c1" and "c5" and the results from a query
+ * is:
+ * Partition: static: { c1: 3, c5: 4 }
+ * "a" : { value : 2 }
+ * "c3": { value : 8 }
+ * "c7": { value : 1 }
+ * then this class transform it into:
+ * Partition: "a" : { value : 2 }
+ * "c1": { value : 3 }
+ * "c3": { value : 8 }
+ * "c5": { value : 4 }
+ * "c7": { value : 1 }
+ */
+public class ThriftResultsMerger extends WrappingUnfilteredPartitionIterator
+{
+ private final int nowInSec;
+
+ private ThriftResultsMerger(UnfilteredPartitionIterator wrapped, int nowInSec)
+ {
+ super(wrapped);
+ this.nowInSec = nowInSec;
+ }
+
+ public static UnfilteredPartitionIterator maybeWrap(UnfilteredPartitionIterator iterator, CFMetaData metadata, int nowInSec)
+ {
+ if (!metadata.isStaticCompactTable() && !metadata.isSuper())
+ return iterator;
+
+ return new ThriftResultsMerger(iterator, nowInSec);
+ }
+
+ public static UnfilteredRowIterator maybeWrap(UnfilteredRowIterator iterator, int nowInSec)
+ {
+ if (!iterator.metadata().isStaticCompactTable() && !iterator.metadata().isSuper())
+ return iterator;
+
+ return iterator.metadata().isSuper()
+ ? new SuperColumnsPartitionMerger(iterator, nowInSec)
+ : new PartitionMerger(iterator, nowInSec);
+ }
+
+ protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter)
+ {
+ return iter.metadata().isSuper()
+ ? new SuperColumnsPartitionMerger(iter, nowInSec)
+ : new PartitionMerger(iter, nowInSec);
+ }
+
+ private static class PartitionMerger extends WrappingUnfilteredRowIterator
+ {
+ private final int nowInSec;
+
+ // We initialize lazily to avoid having this iterator fetch the wrapped iterator before it's actually asked for it.
+ private boolean isInit;
+
+ private Row staticRow;
+ private int i; // the index of the next column of static row to return
+
+ private ReusableRow nextToMerge;
+ private Unfiltered nextFromWrapped;
+
+ private PartitionMerger(UnfilteredRowIterator results, int nowInSec)
+ {
+ super(results);
+ assert results.metadata().isStaticCompactTable();
+ this.nowInSec = nowInSec;
+ }
+
+ private void init()
+ {
+ assert !isInit;
+ this.staticRow = super.staticRow();
+ assert staticRow.columns().complexColumnCount() == 0;
+
+ this.nextToMerge = createReusableRow();
+ updateNextToMerge();
+ isInit = true;
+ }
+
+ @Override
+ public Row staticRow()
+ {
+ return Rows.EMPTY_STATIC_ROW;
+ }
+
+ private ReusableRow createReusableRow()
+ {
+ return new ReusableRow(metadata().clusteringColumns().size(), metadata().partitionColumns().regulars, true, metadata().isCounter());
+ }
+
+ @Override
+ public boolean hasNext()
+ {
+ if (!isInit)
+ init();
+
+ return nextFromWrapped != null || nextToMerge != null || super.hasNext();
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ if (!isInit)
+ init();
+
+ if (nextFromWrapped == null && super.hasNext())
+ nextFromWrapped = super.next();
+
+ if (nextFromWrapped == null)
+ {
+ if (nextToMerge == null)
+ throw new NoSuchElementException();
+
+ return consumeNextToMerge();
+ }
+
+ if (nextToMerge == null)
+ return consumeNextWrapped();
+
+ int cmp = metadata().comparator.compare(nextToMerge, nextFromWrapped);
+ if (cmp < 0)
+ return consumeNextToMerge();
+ if (cmp > 0)
+ return consumeNextWrapped();
+
+ // Same row, but we know the row has only a single column so just pick the more recent
+ assert nextFromWrapped instanceof Row;
+ ReusableRow row = createReusableRow();
+ Rows.merge((Row)consumeNextWrapped(), consumeNextToMerge(), columns().regulars, row.writer(), nowInSec);
+ return row;
+ }
+
+ private Unfiltered consumeNextWrapped()
+ {
+ Unfiltered toReturn = nextFromWrapped;
+ nextFromWrapped = null;
+ return toReturn;
+ }
+
+ private Row consumeNextToMerge()
+ {
+ Row toReturn = nextToMerge;
+ updateNextToMerge();
+ return toReturn;
+ }
+
+ private void updateNextToMerge()
+ {
+ while (i < staticRow.columns().simpleColumnCount())
+ {
+ Cell cell = staticRow.getCell(staticRow.columns().getSimple(i++));
+ if (cell != null)
+ {
+ // Given a static cell, the equivalent row uses the column name as clustering and the
+ // value as unique cell value.
+ Row.Writer writer = nextToMerge.writer();
+ writer.writeClusteringValue(cell.column().name.bytes);
+ writer.writeCell(metadata().compactValueColumn(), cell.isCounterCell(), cell.value(), cell.livenessInfo(), cell.path());
+ writer.endOfRow();
+ return;
+ }
+ }
+ // Nothing more to merge.
+ nextToMerge = null;
+ }
+ }
+
+ private static class SuperColumnsPartitionMerger extends WrappingUnfilteredRowIterator
+ {
+ private final int nowInSec;
+ private final ReusableRow reusableRow;
+ private final ColumnDefinition superColumnMapColumn;
+ private final AbstractType<?> columnComparator;
+
+ private SuperColumnsPartitionMerger(UnfilteredRowIterator results, int nowInSec)
+ {
+ super(results);
+ assert results.metadata().isSuper();
+ this.nowInSec = nowInSec;
+
+ this.superColumnMapColumn = results.metadata().compactValueColumn();
+ assert superColumnMapColumn != null && superColumnMapColumn.type instanceof MapType;
+
+ this.reusableRow = new ReusableRow(results.metadata().clusteringColumns().size(),
+ Columns.of(superColumnMapColumn),
+ true,
+ results.metadata().isCounter());
+ this.columnComparator = ((MapType)superColumnMapColumn.type).nameComparator();
+ }
+
+ @Override
+ public Unfiltered next()
+ {
+ Unfiltered next = super.next();
+ if (next.kind() != Unfiltered.Kind.ROW)
+ return next;
+
+ Row row = (Row)next;
+ Row.Writer writer = reusableRow.writer();
+ row.clustering().writeTo(writer);
+
+ PeekingIterator<Cell> staticCells = Iterators.peekingIterator(makeStaticCellIterator(row));
+ if (!staticCells.hasNext())
+ return row;
+
+ Iterator<Cell> cells = row.getCells(superColumnMapColumn);
+ PeekingIterator<Cell> dynamicCells = Iterators.peekingIterator(cells.hasNext() ? cells : Collections.<Cell>emptyIterator());
+
+ while (staticCells.hasNext() && dynamicCells.hasNext())
+ {
+ Cell staticCell = staticCells.peek();
+ Cell dynamicCell = dynamicCells.peek();
+ int cmp = columnComparator.compare(staticCell.column().name.bytes, dynamicCell.path().get(0));
+ if (cmp < 0)
+ {
+ staticCell = staticCells.next();
+ writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes));
+ }
+ else if (cmp > 0)
+ {
+ dynamicCells.next().writeTo(writer);
+ }
+ else
+ {
+ staticCell = staticCells.next();
+ Cell toMerge = Cells.create(superColumnMapColumn,
+ staticCell.isCounterCell(),
+ staticCell.value(),
+ staticCell.livenessInfo(),
+ CellPath.create(staticCell.column().name.bytes));
+ Cells.reconcile(toMerge, dynamicCells.next(), nowInSec).writeTo(writer);
+ }
+ }
+
+ while (staticCells.hasNext())
+ {
+ Cell staticCell = staticCells.next();
+ writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes));
+ }
+ while (dynamicCells.hasNext())
+ {
+ dynamicCells.next().writeTo(writer);
+ }
+
+ writer.endOfRow();
+ return reusableRow;
+ }
+
+ private static Iterator<Cell> makeStaticCellIterator(final Row row)
+ {
+ return new AbstractIterator<Cell>()
+ {
+ private int i;
+
+ protected Cell computeNext()
+ {
+ while (i < row.columns().simpleColumnCount())
+ {
+ Cell cell = row.getCell(row.columns().getSimple(i++));
+ if (cell != null)
+ return cell;
+ }
+ return endOfData();
+ }
+ };
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftValidation.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
index 3768952..dd5bf98 100644
--- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java
+++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
@@ -24,17 +24,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Attributes;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.*;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.NamesQueryFilter;
-import org.apache.cassandra.db.filter.SliceQueryFilter;
-import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.ColumnToCollectionType;
-import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.serializers.MarshalException;
@@ -123,7 +116,7 @@ public class ThriftValidation
*/
public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException
{
- if (metadata.cfType == ColumnFamilyType.Standard)
+ if (!metadata.isSuper())
{
if (column_path.super_column != null)
{
@@ -151,7 +144,7 @@ public class ThriftValidation
public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
{
- if (metadata.cfType == ColumnFamilyType.Standard)
+ if (!metadata.isSuper())
{
if (column_parent.super_column != null)
{
@@ -168,19 +161,19 @@ public class ThriftValidation
// column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF
static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
{
- if (metadata.cfType == ColumnFamilyType.Standard)
+ if (metadata.isSuper())
{
- if (column_path_or_parent.super_column != null)
+ if (column_path_or_parent.super_column == null && column_path_or_parent.column != null)
{
- throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName);
+ throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF "
+ + metadata.cfName);
}
}
- if (metadata.cfType == ColumnFamilyType.Super)
+ else
{
- if (column_path_or_parent.super_column == null && column_path_or_parent.column != null)
+ if (column_path_or_parent.super_column != null)
{
- throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF "
- + metadata.cfName);
+ throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName);
}
}
if (column_path_or_parent.column != null)
@@ -193,13 +186,30 @@ public class ThriftValidation
}
}
+ private static AbstractType<?> getThriftColumnNameComparator(CFMetaData metadata, ByteBuffer superColumnName)
+ {
+ if (!metadata.isSuper())
+ return LegacyLayout.makeLegacyComparator(metadata);
+
+ if (superColumnName == null)
+ {
+ // comparator for super column name
+ return metadata.comparator.subtype(0);
+ }
+ else
+ {
+ // comparator for sub columns
+ return metadata.thriftColumnNameType();
+ }
+ }
+
/**
* Validates the column names but not the parent path or data
*/
private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names)
throws org.apache.cassandra.exceptions.InvalidRequestException
{
- int maxNameLength = Cell.MAX_NAME_LENGTH;
+ int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
if (superColumnName != null)
{
@@ -207,10 +217,10 @@ public class ThriftValidation
throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength);
if (superColumnName.remaining() == 0)
throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty");
- if (metadata.cfType == ColumnFamilyType.Standard)
+ if (!metadata.isSuper())
throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns");
}
- AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, superColumnName);
+ AbstractType<?> comparator = getThriftColumnNameComparator(metadata, superColumnName);
boolean isCQL3Table = !metadata.isThriftCompatible();
for (ByteBuffer name : column_names)
{
@@ -229,31 +239,26 @@ public class ThriftValidation
if (isCQL3Table)
{
- // CQL3 table don't support having only part of their composite column names set
- Composite composite = metadata.comparator.fromByteBuffer(name);
-
- int minComponents = metadata.comparator.clusteringPrefixSize() + 1;
- if (composite.size() < minComponents)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table",
- composite.size(), minComponents, metadata.cfName));
-
- // Furthermore, the column name must be a declared one.
- int columnIndex = metadata.comparator.clusteringPrefixSize();
- ByteBuffer CQL3ColumnName = composite.get(columnIndex);
- if (!CQL3ColumnName.hasRemaining())
- continue; // Row marker, ok
-
- ColumnIdentifier columnId = new ColumnIdentifier(CQL3ColumnName, metadata.comparator.subtype(columnIndex));
- if (metadata.getColumnDefinition(columnId) == null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid cell for CQL3 table %s. The CQL3 column component (%s) does not correspond to a defined CQL3 column",
- metadata.cfName, columnId));
-
- // On top of that, if we have a collection component, he (CQL3) column must be a collection
- if (metadata.comparator.hasCollections() && composite.size() == metadata.comparator.size())
+ try
+ {
+ LegacyLayout.LegacyCellName cname = LegacyLayout.decodeCellName(metadata, name);
+ assert cname.clustering.size() == metadata.comparator.size();
+
+ // CQL3 table don't support having only part of their composite column names set
+ for (int i = 0; i < cname.clustering.size(); i++)
+ {
+ if (cname.clustering.get(i) == null)
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table",
+ i, metadata.comparator.size() + 1, metadata.cfName));
+ }
+
+ // On top of that, if we have a collection component, the (CQL3) column must be a collection
+ if (cname.column != null && cname.collectionElement != null && !cname.column.type.isCollection())
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", cname.column.name));
+ }
+ catch (IllegalArgumentException | UnknownColumnException e)
{
- ColumnToCollectionType collectionType = metadata.comparator.collectionType();
- if (!collectionType.defined.containsKey(CQL3ColumnName))
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", UTF8Type.instance.getString(CQL3ColumnName)));
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Error validating cell name for CQL3 table %s: %s", metadata.cfName, e.getMessage()));
}
}
}
@@ -269,13 +274,13 @@ public class ThriftValidation
if (range.count < 0)
throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count");
- int maxNameLength = Cell.MAX_NAME_LENGTH;
+ int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
if (range.start.remaining() > maxNameLength)
throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength);
if (range.finish.remaining() > maxNameLength)
throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength);
- AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, column_parent.super_column);
+ AbstractType<?> comparator = getThriftColumnNameComparator(metadata, column_parent.super_column);
try
{
comparator.validate(range.start);
@@ -295,7 +300,7 @@ public class ThriftValidation
}
}
- public static void validateColumnOrSuperColumn(CFMetaData metadata, ByteBuffer key, ColumnOrSuperColumn cosc)
+ public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc)
throws org.apache.cassandra.exceptions.InvalidRequestException
{
boolean isCommutative = metadata.isCounter();
@@ -316,7 +321,7 @@ public class ThriftValidation
validateTtl(cosc.column);
validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name));
- validateColumnData(metadata, key, null, cosc.column);
+ validateColumnData(metadata, null, cosc.column);
}
if (cosc.super_column != null)
@@ -327,7 +332,7 @@ public class ThriftValidation
for (Column c : cosc.super_column.columns)
{
validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name));
- validateColumnData(metadata, key, cosc.super_column.name, c);
+ validateColumnData(metadata, cosc.super_column.name, c);
}
}
@@ -356,8 +361,8 @@ public class ThriftValidation
if (column.ttl <= 0)
throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be positive");
- if (column.ttl > ExpiringCell.MAX_TTL)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, ExpiringCell.MAX_TTL));
+ if (column.ttl > Attributes.MAX_TTL)
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, Attributes.MAX_TTL));
}
else
{
@@ -366,7 +371,7 @@ public class ThriftValidation
}
}
- public static void validateMutation(CFMetaData metadata, ByteBuffer key, Mutation mut)
+ public static void validateMutation(CFMetaData metadata, Mutation mut)
throws org.apache.cassandra.exceptions.InvalidRequestException
{
ColumnOrSuperColumn cosc = mut.column_or_supercolumn;
@@ -383,7 +388,7 @@ public class ThriftValidation
if (cosc != null)
{
- validateColumnOrSuperColumn(metadata, key, cosc);
+ validateColumnOrSuperColumn(metadata, cosc);
}
else
{
@@ -400,7 +405,7 @@ public class ThriftValidation
if (del.predicate != null)
validateSlicePredicate(metadata, del.super_column, del.predicate);
- if (metadata.cfType == ColumnFamilyType.Standard && del.super_column != null)
+ if (!metadata.isSuper() && del.super_column != null)
{
String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del);
throw new org.apache.cassandra.exceptions.InvalidRequestException(msg);
@@ -409,7 +414,7 @@ public class ThriftValidation
if (metadata.isCounter())
{
// forcing server timestamp even if a timestamp was set for coherence with other counter operation
- del.timestamp = System.currentTimeMillis();
+ del.timestamp = FBUtilities.timestampMicros();
}
else if (!del.isSetTimestamp())
{
@@ -432,7 +437,7 @@ public class ThriftValidation
/**
* Validates the data part of the column (everything in the column object but the name, which is assumed to be valid)
*/
- public static void validateColumnData(CFMetaData metadata, ByteBuffer key, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
+ public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
{
validateTtl(column);
if (!column.isSetValue())
@@ -440,14 +445,17 @@ public class ThriftValidation
if (!column.isSetTimestamp())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required");
- CellName cn = scName == null
- ? metadata.comparator.cellFromByteBuffer(column.name)
- : metadata.comparator.makeCellName(scName, column.name);
try
{
- AbstractType<?> validator = metadata.getValueValidator(cn);
- if (validator != null)
- validator.validate(column.value);
+ LegacyLayout.LegacyCellName cn = LegacyLayout.decodeCellName(metadata, scName, column.name);
+ cn.column.validateCellValue(column.value);
+
+ // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
+ Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(cn.column, column.value, null);
+ }
+ catch (UnknownColumnException e)
+ {
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
}
catch (MarshalException me)
{
@@ -458,25 +466,9 @@ public class ThriftValidation
me.getMessage(),
metadata.ksName,
metadata.cfName,
- (SuperColumns.getComparatorFor(metadata, scName != null)).getString(column.name)));
+ (getThriftColumnNameComparator(metadata, scName)).getString(column.name)));
}
- // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details
- SecondaryIndex failedIndex = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(key, asDBColumn(cn, column));
- if (failedIndex != null)
- throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s",
- column.value.remaining(),
- failedIndex.getIndexName(),
- metadata.cfName,
- metadata.ksName));
- }
-
- private static Cell asDBColumn(CellName name, Column column)
- {
- if (column.ttl <= 0)
- return new BufferCell(name, column.value, column.timestamp);
- else
- return new BufferExpiringCell(name, column.value, column.timestamp, column.ttl);
}
/**
@@ -535,8 +527,8 @@ public class ThriftValidation
else if (range.start_key != null && range.end_token != null)
{
// start_token/end_token can wrap, but key/token should not
- RowPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound();
- if (RowPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum())
+ PartitionPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound();
+ if (PartitionPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token");
}
@@ -577,7 +569,7 @@ public class ThriftValidation
return false;
SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager;
- AbstractType<?> nameValidator = SuperColumns.getComparatorFor(metadata, null);
+ AbstractType<?> nameValidator = getThriftColumnNameComparator(metadata, null);
boolean isIndexed = false;
for (IndexExpression expression : index_clause)
@@ -597,11 +589,18 @@ public class ThriftValidation
if (expression.value.remaining() > 0xFFFF)
throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K");
- CellName name = metadata.comparator.cellFromByteBuffer(expression.column_name);
- AbstractType<?> valueValidator = metadata.getValueValidator(name);
+ ColumnDefinition def = metadata.getColumnDefinition(expression.column_name);
+ if (def == null)
+ {
+ if (!metadata.isCompactTable())
+ throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Unknown column %s", nameValidator.getString(expression.column_name)));
+
+ def = metadata.compactValueColumn();
+ }
+
try
{
- valueValidator.validate(expression.value);
+ def.type.validate(expression.value);
}
catch (MarshalException me)
{
@@ -611,7 +610,7 @@ public class ThriftValidation
me.getMessage()));
}
- isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(name);
+ isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(def);
}
return isIndexed;
@@ -637,32 +636,32 @@ public class ThriftValidation
throw new org.apache.cassandra.exceptions.InvalidRequestException("system keyspace is not user-modifiable");
}
- public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn)
- {
- SliceRange sr = sp.slice_range;
- IDiskAtomFilter filter;
-
- CellNameType comparator = metadata.isSuper()
- ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1))
- : metadata.comparator;
- if (sr == null)
- {
-
- SortedSet<CellName> ss = new TreeSet<CellName>(comparator);
- for (ByteBuffer bb : sp.column_names)
- ss.add(comparator.cellFromByteBuffer(bb));
- filter = new NamesQueryFilter(ss);
- }
- else
- {
- filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start),
- comparator.fromByteBuffer(sr.finish),
- sr.reversed,
- sr.count);
- }
-
- if (metadata.isSuper())
- filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter);
- return filter;
- }
+ //public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn)
+ //{
+ // SliceRange sr = sp.slice_range;
+ // IDiskAtomFilter filter;
+
+ // CellNameType comparator = metadata.isSuper()
+ // ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1))
+ // : metadata.comparator;
+ // if (sr == null)
+ // {
+
+ // SortedSet<CellName> ss = new TreeSet<CellName>(comparator);
+ // for (ByteBuffer bb : sp.column_names)
+ // ss.add(comparator.cellFromByteBuffer(bb));
+ // filter = new NamesQueryFilter(ss);
+ // }
+ // else
+ // {
+ // filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start),
+ // comparator.fromByteBuffer(sr.finish),
+ // sr.reversed,
+ // sr.count);
+ // }
+
+ // if (metadata.isSuper())
+ // filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter);
+ // return filter;
+ //}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
deleted file mode 100644
index 9f833e7..0000000
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ /dev/null
@@ -1,480 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.tools;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.*;
-
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.commons.cli.*;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
-
-/**
- * Export SSTables to JSON format.
- */
-public class SSTableExport
-{
- private static final ObjectMapper jsonMapper = new ObjectMapper();
-
- private static final String KEY_OPTION = "k";
- private static final String EXCLUDEKEY_OPTION = "x";
- private static final String ENUMERATEKEYS_OPTION = "e";
-
- private static final Options options = new Options();
- private static CommandLine cmd;
-
- static
- {
- Option optKey = new Option(KEY_OPTION, true, "Row key");
- // Number of times -k <key> can be passed on the command line.
- optKey.setArgs(500);
- options.addOption(optKey);
-
- Option excludeKey = new Option(EXCLUDEKEY_OPTION, true, "Excluded row key");
- // Number of times -x <key> can be passed on the command line.
- excludeKey.setArgs(500);
- options.addOption(excludeKey);
-
- Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only");
- options.addOption(optEnumerate);
-
- // disabling auto close of the stream
- jsonMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
- }
-
- /**
- * Checks if PrintStream error and throw exception
- *
- * @param out The PrintStream to be check
- */
- private static void checkStream(PrintStream out) throws IOException
- {
- if (out.checkError())
- throw new IOException("Error writing output stream");
- }
-
- /**
- * JSON Hash Key serializer
- *
- * @param out The output steam to write data
- * @param value value to set as a key
- */
- private static void writeKey(PrintStream out, String value)
- {
- writeJSON(out, value);
- out.print(": ");
- }
-
- private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData)
- {
- if (atom instanceof Cell)
- {
- return serializeColumn((Cell) atom, cfMetaData);
- }
- else
- {
- assert atom instanceof RangeTombstone;
- RangeTombstone rt = (RangeTombstone) atom;
- ArrayList<Object> serializedColumn = new ArrayList<Object>();
- serializedColumn.add(cfMetaData.comparator.getString(rt.min));
- serializedColumn.add(cfMetaData.comparator.getString(rt.max));
- serializedColumn.add(rt.data.markedForDeleteAt);
- serializedColumn.add("t");
- serializedColumn.add(rt.data.localDeletionTime);
- return serializedColumn;
- }
- }
-
- /**
- * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Type is
- *
- * human_readable_name, value, timestamp, [flag, [options]]
- *
- * Value is normally the human readable value as rendered by the validator, but for deleted cells we
- * give the local deletion time instead.
- *
- * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter:
- * - No options for deleted cells
- * - If expiring, options will include the TTL and local deletion time.
- * - If counter, options will include timestamp of last delete
- *
- * @param cell cell presentation
- * @param cfMetaData Column Family metadata (to get validator)
- * @return cell as serialized list
- */
- private static List<Object> serializeColumn(Cell cell, CFMetaData cfMetaData)
- {
- CellNameType comparator = cfMetaData.comparator;
- ArrayList<Object> serializedColumn = new ArrayList<Object>();
-
- serializedColumn.add(comparator.getString(cell.name()));
-
- if (cell instanceof DeletedCell)
- {
- serializedColumn.add(cell.getLocalDeletionTime());
- }
- else
- {
- AbstractType<?> validator = cfMetaData.getValueValidator(cell.name());
- serializedColumn.add(validator.getString(cell.value()));
- }
-
- serializedColumn.add(cell.timestamp());
-
- if (cell instanceof DeletedCell)
- {
- serializedColumn.add("d");
- }
- else if (cell instanceof ExpiringCell)
- {
- serializedColumn.add("e");
- serializedColumn.add(((ExpiringCell) cell).getTimeToLive());
- serializedColumn.add(cell.getLocalDeletionTime());
- }
- else if (cell instanceof CounterCell)
- {
- serializedColumn.add("c");
- serializedColumn.add(((CounterCell) cell).timestampOfLastDelete());
- }
-
- return serializedColumn;
- }
-
- /**
- * Get portion of the columns and serialize in loop while not more columns left in the row
- *
- * @param row SSTableIdentityIterator row representation with Column Family
- * @param key Decorated Key for the required row
- * @param out output stream
- */
- private static void serializeRow(SSTableIdentityIterator row, DecoratedKey key, PrintStream out)
- {
- serializeRow(row.getColumnFamily().deletionInfo(), row, row.getColumnFamily().metadata(), key, out);
- }
-
- private static void serializeRow(DeletionInfo deletionInfo, Iterator<OnDiskAtom> atoms, CFMetaData metadata, DecoratedKey key, PrintStream out)
- {
- out.print("{");
- writeKey(out, "key");
- writeJSON(out, metadata.getKeyValidator().getString(key.getKey()));
- out.print(",\n");
-
- if (!deletionInfo.isLive())
- {
- out.print(" ");
- writeKey(out, "metadata");
- out.print("{");
- writeKey(out, "deletionInfo");
- writeJSON(out, deletionInfo.getTopLevelDeletion());
- out.print("}");
- out.print(",\n");
- }
-
- out.print(" ");
- writeKey(out, "cells");
- out.print("[");
- while (atoms.hasNext())
- {
- writeJSON(out, serializeAtom(atoms.next(), metadata));
-
- if (atoms.hasNext())
- out.print(",\n ");
- }
- out.print("]");
-
- out.print("}");
- }
-
- /**
- * Enumerate row keys from an SSTableReader and write the result to a PrintStream.
- *
- * @param desc the descriptor of the file to export the rows from
- * @param outs PrintStream to write the output to
- * @param metadata Metadata to print keys in a proper format
- * @throws IOException on failure to read/write input/output
- */
- public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata)
- throws IOException
- {
- try (KeyIterator iter = new KeyIterator(desc))
- {
- DecoratedKey lastKey = null;
- while (iter.hasNext())
- {
- DecoratedKey key = iter.next();
-
- // validate order of the keys in the sstable
- if (lastKey != null && lastKey.compareTo(key) > 0)
- throw new IOException("Key out of order! " + lastKey + " > " + key);
- lastKey = key;
-
- outs.println(metadata.getKeyValidator().getString(key.getKey()));
- checkStream(outs); // flushes
- }
- }
- }
-
- /**
- * Export specific rows from an SSTable and write the resulting JSON to a PrintStream.
- *
- * @param desc the descriptor of the sstable to read from
- * @param outs PrintStream to write the output to
- * @param toExport the keys corresponding to the rows to export
- * @param excludes keys to exclude from export
- * @param metadata Metadata to print keys in a proper format
- * @throws IOException on failure to read/write input/output
- */
- public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes, CFMetaData metadata) throws IOException
- {
- SSTableReader sstable = SSTableReader.open(desc);
-
- try (RandomAccessReader dfile = sstable.openDataReader())
- {
- IPartitioner partitioner = sstable.partitioner;
-
- if (excludes != null)
- toExport.removeAll(Arrays.asList(excludes));
-
- outs.println("[");
-
- int i = 0;
-
- // last key to compare order
- DecoratedKey lastKey = null;
-
- for (String key : toExport)
- {
- DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key));
-
- if (lastKey != null && lastKey.compareTo(decoratedKey) > 0)
- throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey);
-
- lastKey = decoratedKey;
-
- RowIndexEntry entry = sstable.getPosition(decoratedKey, SSTableReader.Operator.EQ);
- if (entry == null)
- continue;
-
- dfile.seek(entry.position);
- ByteBufferUtil.readWithShortLength(dfile); // row key
- DeletionInfo deletionInfo = new DeletionInfo(DeletionTime.serializer.deserialize(dfile));
-
- Iterator<OnDiskAtom> atomIterator = sstable.metadata.getOnDiskIterator(dfile, sstable.descriptor.version);
- checkStream(outs);
-
- if (i != 0)
- outs.println(",");
- i++;
- serializeRow(deletionInfo, atomIterator, sstable.metadata, decoratedKey, outs);
- }
-
- outs.println("\n]");
- outs.flush();
- }
- }
-
- // This is necessary to accommodate the test suite since you cannot open a Reader more
- // than once from within the same process.
- static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException
- {
- Set<String> excludeSet = new HashSet<String>();
-
- if (excludes != null)
- excludeSet = new HashSet<>(Arrays.asList(excludes));
-
- SSTableIdentityIterator row;
- ISSTableScanner scanner = reader.getScanner();
- try
- {
- outs.println("[");
-
- int i = 0;
-
- // collecting keys to export
- while (scanner.hasNext())
- {
- row = (SSTableIdentityIterator) scanner.next();
-
- String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey());
-
- if (excludeSet.contains(currentKey))
- continue;
- else if (i != 0)
- outs.println(",");
-
- serializeRow(row, row.getKey(), outs);
- checkStream(outs);
-
- i++;
- }
-
- outs.println("\n]");
- outs.flush();
- }
- finally
- {
- scanner.close();
- }
- }
-
- /**
- * Export an SSTable and write the resulting JSON to a PrintStream.
- *
- * @param desc the descriptor of the sstable to read from
- * @param outs PrintStream to write the output to
- * @param excludes keys to exclude from export
- * @throws IOException on failure to read/write input/output
- */
- public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException
- {
- export(SSTableReader.open(desc), outs, excludes);
- }
-
- /**
- * Export an SSTable and write the resulting JSON to standard out.
- *
- * @param desc the descriptor of the sstable to read from
- * @param excludes keys to exclude from export
- * @throws IOException on failure to read/write SSTable/standard out
- */
- public static void export(Descriptor desc, String[] excludes) throws IOException
- {
- export(desc, System.out, excludes);
- }
-
- /**
- * Given arguments specifying an SSTable, and optionally an output file,
- * export the contents of the SSTable to JSON.
- *
- * @param args command lines arguments
- * @throws ConfigurationException on configuration failure (wrong params given)
- */
- public static void main(String[] args) throws ConfigurationException
- {
- System.err.println("WARNING: please note that sstable2json is now deprecated and will be removed in Cassandra 3.0. "
- + "Please see https://issues.apache.org/jira/browse/CASSANDRA-9618 for details.");
-
- String usage = String.format("Usage: %s <sstable> [-k key [-k key [...]] -x key [-x key [...]]]%n", SSTableExport.class.getName());
-
- CommandLineParser parser = new PosixParser();
- try
- {
- cmd = parser.parse(options, args);
- }
- catch (ParseException e1)
- {
- System.err.println(e1.getMessage());
- System.err.println(usage);
- System.exit(1);
- }
-
-
- if (cmd.getArgs().length != 1)
- {
- System.err.println("You must supply exactly one sstable");
- System.err.println(usage);
- System.exit(1);
- }
-
-
- String[] keys = cmd.getOptionValues(KEY_OPTION);
- String[] excludes = cmd.getOptionValues(EXCLUDEKEY_OPTION);
- String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
-
- Schema.instance.loadFromDisk(false);
- Descriptor descriptor = Descriptor.fromFilename(ssTableFileName);
-
- // Start by validating keyspace name
- if (Schema.instance.getKSMetaData(descriptor.ksname) == null)
- {
- System.err.println(String.format("Filename %s references to nonexistent keyspace: %s!",
- ssTableFileName, descriptor.ksname));
- System.exit(1);
- }
- Keyspace keyspace = Keyspace.open(descriptor.ksname);
-
- // Make it works for indexes too - find parent cf if necessary
- String baseName = descriptor.cfname;
- if (descriptor.cfname.contains("."))
- {
- String[] parts = descriptor.cfname.split("\\.", 2);
- baseName = parts[0];
- }
-
- // IllegalArgumentException will be thrown here if ks/cf pair does not exist
- ColumnFamilyStore cfStore = null;
- try
- {
- cfStore = keyspace.getColumnFamilyStore(baseName);
- }
- catch (IllegalArgumentException e)
- {
- System.err.println(String.format("The provided table is not part of this cassandra keyspace: keyspace = %s, table = %s",
- descriptor.ksname, descriptor.cfname));
- System.exit(1);
- }
-
- try
- {
- if (cmd.hasOption(ENUMERATEKEYS_OPTION))
- {
- enumeratekeys(descriptor, System.out, cfStore.metadata);
- }
- else
- {
- if ((keys != null) && (keys.length > 0))
- export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata);
- else
- export(descriptor, excludes);
- }
- }
- catch (IOException e)
- {
- // throwing exception outside main with broken pipe causes windows cmd to hang
- e.printStackTrace(System.err);
- }
-
- System.exit(0);
- }
-
- private static void writeJSON(PrintStream out, Object value)
- {
- try
- {
- jsonMapper.writeValue(out, value);
- }
- catch (Exception e)
- {
- throw new RuntimeException(e.getMessage(), e);
- }
- }
-}