You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/04/24 05:00:38 UTC
[1/3] git commit: make sstable2json output more readable, and remove
support for supercolumn json (CASSANDRA-7078) patch by jbellis;
reviewed by ayeschenko for CASSANDRA-7078
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 6acbb33fa -> 647bfc6d7
refs/heads/trunk e6325fa77 -> cf74bcfc6
make sstable2json output more readable, and remove support for supercolumn json (CASSANDRA-7078)
patch by jbellis; reviewed by ayeschenko for CASSANDRA-7078
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/647bfc6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/647bfc6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/647bfc6d
Branch: refs/heads/cassandra-2.1
Commit: 647bfc6d7985a9e07734315424fa6cc89e587df3
Parents: 6acbb33
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Apr 23 22:00:13 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Apr 23 22:00:21 2014 -0500
----------------------------------------------------------------------
NEWS.txt | 3 +-
.../apache/cassandra/tools/SSTableExport.java | 93 +++++--------
.../apache/cassandra/tools/SSTableImport.java | 133 ++++---------------
test/resources/CounterCF.json | 2 +-
test/resources/SimpleCF.json | 4 +-
test/resources/SimpleCF.oldformat.json | 4 -
test/resources/SimpleCFWithDeletionInfo.json | 4 +-
test/resources/SuperCF.json | 4 -
test/resources/UnsortedCF.json | 4 +-
test/resources/UnsortedSuperCF.json | 5 -
.../cassandra/tools/SSTableExportTest.java | 40 +-----
.../cassandra/tools/SSTableImportTest.java | 51 -------
12 files changed, 76 insertions(+), 271 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index ac78a73..86c6f64 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -48,9 +48,10 @@ Upgrading
to change your data model to accomodate the new implementation.
(See https://issues.apache.org/jira/browse/CASSANDRA-6504 and the dev
blog post at http://www.datastax.com/dev/blog/<PLACEHOLDER> for details).
- - (per-table) index_interval parameter has been replaced with
+ - (per-table) index_interval parameter has been replaced with
min_index_interval and max_index_interval paratemeters. index_interval
has been deprecated.
+ - support for supercolumns has been removed from json2sstable
2.0.7
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index bbc3494..02deb65 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -20,7 +20,6 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
-import java.nio.ByteBuffer;
import java.util.*;
import org.apache.commons.cli.*;
@@ -98,53 +97,6 @@ public class SSTableExport
out.print(": ");
}
- /**
- * JSON ColumnFamily metadata serializer.</br> Serializes:
- * <ul>
- * <li>column family deletion info (if present)</li>
- * </ul>
- *
- * @param out The output steam to write data
- * @param deletionInfo
- */
- private static void writeMeta(PrintStream out, DeletionInfo deletionInfo)
- {
- if (!deletionInfo.isLive())
- {
- // begin meta
- writeKey(out, "metadata");
- writeDeletionInfo(out, deletionInfo.getTopLevelDeletion());
- out.print(",");
- }
- }
-
- private static void writeDeletionInfo(PrintStream out, DeletionTime deletionTime)
- {
- out.print("{");
- writeKey(out, "deletionInfo");
- // only store topLevelDeletion (serializeForSSTable only uses this)
- writeJSON(out, deletionTime);
- out.print("}");
- }
-
- /**
- * Serialize columns using given column iterator
- *
- * @param atoms column iterator
- * @param out output stream
- * @param cfMetaData Column Family metadata (to get validator)
- */
- private static void serializeAtoms(Iterator<OnDiskAtom> atoms, PrintStream out, CFMetaData cfMetaData)
- {
- while (atoms.hasNext())
- {
- writeJSON(out, serializeAtom(atoms.next(), cfMetaData));
-
- if (atoms.hasNext())
- out.print(", ");
- }
- }
-
private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData)
{
if (atom instanceof Cell)
@@ -166,7 +118,17 @@ public class SSTableExport
}
/**
- * Serialize a given cell to the JSON format
+ * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Format is
+ *
+ * human_readable_name, value, timestamp, [flag, [options]]
+ *
+ * Value is normally the human readable value as rendered by the validator, but for deleted cells we
+ * give the local deletion time instead.
+ *
+ * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter:
+ * - No options for deleted cells
+ * - If expiring, options will include the TTL and local deletion time.
+ * - If counter, options will include timestamp of last delete
*
* @param cell cell presentation
* @param cfMetaData Column Family metadata (to get validator)
@@ -177,18 +139,18 @@ public class SSTableExport
CellNameType comparator = cfMetaData.comparator;
ArrayList<Object> serializedColumn = new ArrayList<Object>();
- ByteBuffer value = cell.value();
-
serializedColumn.add(comparator.getString(cell.name()));
+
if (cell instanceof DeletedCell)
{
- serializedColumn.add(ByteBufferUtil.bytesToHex(value));
+ serializedColumn.add(cell.getLocalDeletionTime());
}
else
{
AbstractType<?> validator = cfMetaData.getValueValidator(cell.name());
- serializedColumn.add(validator.getString(value));
+ serializedColumn.add(validator.getString(cell.value()));
}
+
serializedColumn.add(cell.timestamp());
if (cell instanceof DeletedCell)
@@ -227,16 +189,31 @@ public class SSTableExport
out.print("{");
writeKey(out, "key");
writeJSON(out, bytesToHex(key.key));
- out.print(",");
+ out.print(",\n");
- writeMeta(out, deletionInfo);
+ if (!deletionInfo.isLive())
+ {
+ out.print(" ");
+ writeKey(out, "metadata");
+ out.print("{");
+ writeKey(out, "deletionInfo");
+ writeJSON(out, deletionInfo.getTopLevelDeletion());
+ out.print("}");
+ out.print(",\n");
+ }
- writeKey(out, "columns");
+ out.print(" ");
+ writeKey(out, "cells");
out.print("[");
+ while (atoms.hasNext())
+ {
+ writeJSON(out, serializeAtom(atoms.next(), metadata));
- serializeAtoms(atoms, out, metadata);
-
+ if (atoms.hasNext())
+ out.print(",\n ");
+ }
out.print("]");
+
out.print("}");
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 6bd2119..6b34f59 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -63,14 +63,12 @@ public class SSTableImport
private static final String COLUMN_FAMILY_OPTION = "c";
private static final String KEY_COUNT_OPTION = "n";
private static final String IS_SORTED_OPTION = "s";
- private static final String OLD_SC_FORMAT_OPTION = "S";
private static final Options options = new Options();
private static CommandLine cmd;
private Integer keyCountToImport;
private final boolean isSorted;
- private final boolean oldSCFormat;
private static final JsonFactory factory = new MappingJsonFactory().configure(
JsonParser.Feature.INTERN_FIELD_NAMES, false);
@@ -87,7 +85,6 @@ public class SSTableImport
options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional)."));
options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional)."));
- options.addOption(new Option(OLD_SC_FORMAT_OPTION, false, "Assume JSON file use legacy super column format (Optional)."));
}
private static class JsonColumn<T>
@@ -104,11 +101,11 @@ public class SSTableImport
// Counter columns
private long timestampOfLastDelete;
- public JsonColumn(T json, CFMetaData meta, boolean oldSCFormat, boolean isSubColumn)
+ public JsonColumn(T json, CFMetaData meta)
{
if (json instanceof List)
{
- CellNameType comparator = oldSCFormat ? new SimpleDenseCellNameType(SuperColumns.getComparatorFor(meta, isSubColumn)) : meta.comparator;
+ CellNameType comparator = meta.comparator;
List fields = (List<?>) json;
assert fields.size() >= 3 : "Cell definition should have at least 3";
@@ -119,42 +116,25 @@ public class SSTableImport
if (fields.size() > 3)
{
- if (fields.get(3) instanceof Boolean)
+ kind = (String) fields.get(3);
+ if (isExpiring())
{
- // old format, reading this for backward compatibility sake
- if (fields.size() == 6)
- {
- kind = "e";
- ttl = (Integer) fields.get(4);
- localExpirationTime = (Integer) fields.get(5);
- }
- else
- {
- kind = ((Boolean) fields.get(3)) ? "d" : "";
- }
+ ttl = (Integer) fields.get(4);
+ localExpirationTime = (Integer) fields.get(5);
}
- else
+ else if (isCounter())
{
- kind = (String) fields.get(3);
- if (isExpiring())
- {
- ttl = (Integer) fields.get(4);
- localExpirationTime = (Integer) fields.get(5);
- }
- else if (isCounter())
- {
- timestampOfLastDelete = (long) ((Integer) fields.get(4));
- }
- else if (isRangeTombstone())
- {
- localExpirationTime = (Integer) fields.get(4);
- }
+ timestampOfLastDelete = (long) ((Integer) fields.get(4));
+ }
+ else if (isRangeTombstone())
+ {
+ localExpirationTime = (Integer) fields.get(4);
}
}
if (isDeleted())
{
- value = ByteBufferUtil.hexToBytes((String) fields.get(1));
+ value = ByteBufferUtil.bytes((Integer) fields.get(1));
}
else if (isRangeTombstone())
{
@@ -200,61 +180,43 @@ public class SSTableImport
public SSTableImport()
{
- this(null, false, false);
+ this(null, false);
}
public SSTableImport(boolean isSorted)
{
- this(isSorted, false);
- }
-
- public SSTableImport(boolean isSorted, boolean oldSCFormat)
- {
- this(null, isSorted, oldSCFormat);
+ this(null, isSorted);
}
- public SSTableImport(Integer keyCountToImport, boolean isSorted, boolean oldSCFormat)
+ public SSTableImport(Integer keyCountToImport, boolean isSorted)
{
this.keyCountToImport = keyCountToImport;
this.isSorted = isSorted;
- this.oldSCFormat = oldSCFormat;
- }
-
- private void addToStandardCF(List<?> row, ColumnFamily cfamily)
- {
- addColumnsToCF(row, null, cfamily);
}
/**
* Add columns to a column family.
*
* @param row the columns associated with a row
- * @param superName name of the super column if any
* @param cfamily the column family to add columns to
*/
- private void addColumnsToCF(List<?> row, ByteBuffer superName, ColumnFamily cfamily)
+ private void addColumnsToCF(List<?> row, ColumnFamily cfamily)
{
CFMetaData cfm = cfamily.metadata();
assert cfm != null;
for (Object c : row)
{
- JsonColumn col = new JsonColumn<List>((List) c, cfm, oldSCFormat, (superName != null));
+ JsonColumn col = new JsonColumn<List>((List) c, cfm);
if (col.isRangeTombstone())
{
- Composite start = superName == null
- ? cfm.comparator.fromByteBuffer(col.getName())
- : cfm.comparator.make(superName, col.getName());
- Composite end = superName == null
- ? cfm.comparator.fromByteBuffer(col.getValue())
- : cfm.comparator.make(superName, col.getValue());
+ Composite start = cfm.comparator.fromByteBuffer(col.getName());
+ Composite end = cfm.comparator.fromByteBuffer(col.getValue());
cfamily.addAtom(new RangeTombstone(start, end, col.timestamp, col.localExpirationTime));
continue;
}
- CellName cname = superName == null
- ? cfm.comparator.cellFromByteBuffer(col.getName())
- : cfm.comparator.makeCellName(superName, col.getName());
+ CellName cname = cfm.comparator.cellFromByteBuffer(col.getName());
if (col.isExpiring())
{
@@ -270,9 +232,7 @@ public class SSTableImport
}
else if (col.isRangeTombstone())
{
- CellName end = superName == null
- ? cfm.comparator.cellFromByteBuffer(col.getValue())
- : cfm.comparator.makeCellName(superName, col.getValue());
+ CellName end = cfm.comparator.cellFromByteBuffer(col.getValue());
cfamily.addAtom(new RangeTombstone(cname, end, col.timestamp, col.localExpirationTime));
}
// cql3 row marker, see CASSANDRA-5852
@@ -305,35 +265,6 @@ public class SSTableImport
}
/**
- * Add super columns to a column family.
- *
- * @param row the super columns associated with a row
- * @param cfamily the column family to add columns to
- */
- private void addToSuperCF(Map<?, ?> row, ColumnFamily cfamily)
- {
- CFMetaData metaData = cfamily.metadata();
- assert metaData != null;
-
- CellNameType comparator = metaData.comparator;
-
- // Super columns
- for (Map.Entry<?, ?> entry : row.entrySet())
- {
- Map<?, ?> data = (Map<?, ?>) entry.getValue();
-
- ByteBuffer superName = stringAsType((String) entry.getKey(), comparator.subtype(0));
-
- addColumnsToCF((List<?>) data.get("subColumns"), superName, cfamily);
-
- if (data.containsKey("metadata"))
- {
- parseMeta((Map<?, ?>) data.get("metadata"), cfamily, superName);
- }
- }
- }
-
- /**
* Convert a JSON formatted file to an SSTable.
*
* @param jsonFile the file containing JSON formatted data
@@ -387,11 +318,8 @@ public class SSTableImport
parseMeta((Map<?, ?>) row.getValue().get("metadata"), columnFamily, null);
}
- Object columns = row.getValue().get("columns");
- if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat)
- addToSuperCF((Map<?, ?>) columns, columnFamily);
- else
- addToStandardCF((List<?>) columns, columnFamily);
+ Object columns = row.getValue().get("cells");
+ addColumnsToCF((List<?>) columns, columnFamily);
writer.append(row.getKey(), columnFamily);
@@ -458,10 +386,7 @@ public class SSTableImport
if (row.containsKey("metadata"))
parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null);
- if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat)
- addToSuperCF((Map<?, ?>)row.get("columns"), columnFamily);
- else
- addToStandardCF((List<?>)row.get("columns"), columnFamily);
+ addColumnsToCF((List<?>) row.get("cells"), columnFamily);
if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
{
@@ -545,7 +470,6 @@ public class SSTableImport
Integer keyCountToImport = null;
boolean isSorted = false;
- boolean oldSCFormat = false;
if (cmd.hasOption(KEY_COUNT_OPTION))
{
@@ -557,11 +481,6 @@ public class SSTableImport
isSorted = true;
}
- if (cmd.hasOption(OLD_SC_FORMAT_OPTION))
- {
- oldSCFormat = true;
- }
-
DatabaseDescriptor.loadSchemas();
if (Schema.instance.getNonSystemKeyspaces().size() < 1)
{
@@ -572,7 +491,7 @@ public class SSTableImport
try
{
- new SSTableImport(keyCountToImport, isSorted, oldSCFormat).importJson(json, keyspace, cfamily, ssTable);
+ new SSTableImport(keyCountToImport, isSorted).importJson(json, keyspace, cfamily, ssTable);
}
catch (Exception e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/CounterCF.json
----------------------------------------------------------------------
diff --git a/test/resources/CounterCF.json b/test/resources/CounterCF.json
index b6bca73..dcd087f 100644
--- a/test/resources/CounterCF.json
+++ b/test/resources/CounterCF.json
@@ -1,3 +1,3 @@
[
- {"key": "726f7741", "columns": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]}
+ {"key": "726f7741", "cells": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCF.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCF.json b/test/resources/SimpleCF.json
index 45f57eb..7ef95f6 100644
--- a/test/resources/SimpleCF.json
+++ b/test/resources/SimpleCF.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
+ {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
+ {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCF.oldformat.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCF.oldformat.json b/test/resources/SimpleCF.oldformat.json
deleted file mode 100644
index d920cfb..0000000
--- a/test/resources/SimpleCF.oldformat.json
+++ /dev/null
@@ -1,4 +0,0 @@
-[
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068, false], ["636f6c4142", "76616c4142", 1294532915069, false], ["636f6c4143", "76616c4143", 1294532915071, false, 42, 2000000000 ]]},
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070, false], ["636f6c4242", "76616c4242", 1294532915073, false]]}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCFWithDeletionInfo.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCFWithDeletionInfo.json b/test/resources/SimpleCFWithDeletionInfo.json
index ef673f2..5090699 100644
--- a/test/resources/SimpleCFWithDeletionInfo.json
+++ b/test/resources/SimpleCFWithDeletionInfo.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
- {"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
+ {"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
+ {"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SuperCF.json
----------------------------------------------------------------------
diff --git a/test/resources/SuperCF.json b/test/resources/SuperCF.json
deleted file mode 100644
index 5afec7f..0000000
--- a/test/resources/SuperCF.json
+++ /dev/null
@@ -1,4 +0,0 @@
-[
- {"key": "726f7741", "columns": {"737570657241": {"metadata": {"deletionInfo": {"markedForDeleteAt":0,"localDeletionTime":0}}, "subColumns": [["636f6c4141", "76616c75654141", 1294532915069], ["636f6c4142", "76616c75654142", 1294532915069]]}}},
- {"key": "726f7742", "columns": {"737570657242": {"subColumns": [["636f6c4241", "76616c75654241", 1294532915069], ["636f6c4242", "76616c75654242", 1294532915069]]}}}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/UnsortedCF.json
----------------------------------------------------------------------
diff --git a/test/resources/UnsortedCF.json b/test/resources/UnsortedCF.json
index 814f182..64ef24f 100644
--- a/test/resources/UnsortedCF.json
+++ b/test/resources/UnsortedCF.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]},
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}
+ {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]},
+ {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/UnsortedSuperCF.json
----------------------------------------------------------------------
diff --git a/test/resources/UnsortedSuperCF.json b/test/resources/UnsortedSuperCF.json
deleted file mode 100644
index bd07e81..0000000
--- a/test/resources/UnsortedSuperCF.json
+++ /dev/null
@@ -1,5 +0,0 @@
-[
- {"key": "303935", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366338333439636337323630", 1294656637116, false], ["4331", "366338333439636337323630", 1294656637116, false], ["4332", "366338333439636337323630", 1294656637116, false], ["4333", "366338333439636337323630", 1294656637116, false], ["4334", "366338333439636337323630", 1294656637116, false]]}}} ,
- {"key": "303630", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "643364393434363830326134", 1294656636902, false], ["4331", "643364393434363830326134", 1294656636902, false], ["4332", "643364393434363830326134", 1294656636902, false], ["4333", "643364393434363830326134", 1294656636902, false], ["4334", "643364393434363830326134", 1294656636902, false]]}}} ,
- {"key": "303638", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366634393232663435353638", 1294656636885, false], ["4331", "366634393232663435353638", 1294656636885, false], ["4332", "366634393232663435353638", 1294656636885, false], ["4333", "366634393232663435353638", 1294656636885, false], ["4334", "366634393232663435353638", 1294656636885, false]]}}}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index 454600d..9e2014d 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -56,31 +56,6 @@ public class SSTableExportTest extends SchemaLoader
return bytesToHex(ByteBufferUtil.bytes(str));
}
- public SSTableWriter getDummyWriter() throws IOException
- {
- File tempSS = tempSSTableFile("Keyspace1", "Standard1");
- ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
-
- // Add rowA
- cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
- writer.append(Util.dk("rowA"), cfamily);
- cfamily.clear();
-
- cfamily.addColumn(Util.cellname("colB"), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
- writer.append(Util.dk("rowB"), cfamily);
- cfamily.clear();
-
-
- return writer;
-
- }
-
-
- public PrintStream dummyStream = new PrintStream(new OutputStream(){
- public void write(int b) throws IOException { throw new IOException(); }
- });
-
@Test
public void testEnumeratekeys() throws IOException
{
@@ -153,7 +128,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowA.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),rowA.get("key"));
- JSONArray colsA = (JSONArray)rowA.get("columns");
+ JSONArray colsA = (JSONArray)rowA.get("cells");
JSONArray colA = (JSONArray)colsA.get(0);
assert hexToBytes((String)colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
@@ -165,7 +140,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowB.keySet().size());
assertEquals("unexpected row key",asHex("rowB"),rowB.get("key"));
- JSONArray colsB = (JSONArray)rowB.get("columns");
+ JSONArray colsB = (JSONArray)rowB.get("cells");
JSONArray colB = (JSONArray)colsB.get(0);
assert colB.size() == 3;
@@ -235,7 +210,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
- JSONArray cols = (JSONArray)row.get("columns");
+ JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("colA"));
assert ((String) colA.get(3)).equals("c");
@@ -267,7 +242,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
- JSONArray cols = (JSONArray)row.get("columns");
+ JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("data"));
assert colA.get(1).equals("{\"foo\":\"bar\"}");
@@ -276,7 +251,6 @@ public class SSTableExportTest extends SchemaLoader
@Test
public void testExportColumnsWithMetadata() throws IOException, ParseException
{
-
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
@@ -293,7 +267,6 @@ public class SSTableExportTest extends SchemaLoader
SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
- System.out.println(json.toJSONString());
assertEquals("unexpected number of rows", 1, json.size());
JSONObject row = (JSONObject)json.get(0);
@@ -320,7 +293,7 @@ public class SSTableExportTest extends SchemaLoader
serializedDeletionInfo.toJSONString());
// check the colums are what we put in
- JSONArray cols = (JSONArray) row.get("columns");
+ JSONArray cols = (JSONArray) row.get("cells");
assertNotNull("expecing columns to be present", cols);
assertEquals("expecting two columns", 2, cols.size());
@@ -331,7 +304,6 @@ public class SSTableExportTest extends SchemaLoader
JSONArray col2 = (JSONArray) cols.get(1);
assertEquals("column name did not match", ByteBufferUtil.bytes("colName1"), hexToBytes((String) col2.get(0)));
assertEquals("column value did not match", ByteBufferUtil.bytes("val1"), hexToBytes((String) col2.get(1)));
-
}
/**
@@ -357,7 +329,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals(1, json.size());
JSONObject row = (JSONObject)json.get(0);
- JSONArray cols = (JSONArray) row.get("columns");
+ JSONArray cols = (JSONArray) row.get("cells");
assertEquals(1, cols.size());
// check column name and value
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index 3576005..3befc44 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@ -75,57 +75,6 @@ public class SSTableImportTest extends SchemaLoader
}
@Test
- public void testImportSimpleCfOldFormat() throws IOException, URISyntaxException
- {
- // Import JSON to temp SSTable file
- String jsonUrl = resourcePath("SimpleCF.oldformat.json");
- File tempSS = tempSSTableFile("Keyspace1", "Standard1");
- new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "Standard1", tempSS.getPath());
-
- // Verify results
- SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
- QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis());
- OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
- ColumnFamily cf = cloneForAdditions(iter);
- while (iter.hasNext()) cf.addAtom(iter.next());
- assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141"));
- assert !(cf.getColumn(Util.cellname("colAA")) instanceof DeletedCell);
- Cell expCol = cf.getColumn(Util.cellname("colAC"));
- assert expCol.value().equals(hexToBytes("76616c4143"));
- assert expCol instanceof ExpiringCell;
- assert ((ExpiringCell)expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000;
- }
-
- @Test
- public void testImportSuperCf() throws IOException, URISyntaxException
- {
- String jsonUrl = resourcePath("SuperCF.json");
- File tempSS = tempSSTableFile("Keyspace1", "Super4");
- new SSTableImport(true, true).importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath());
-
- // Verify results
- SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
- QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Super4", System.currentTimeMillis());
- ColumnFamily cf = cloneForAdditions(qf.getSSTableColumnIterator(reader));
- qf.collateOnDiskAtom(cf, qf.getSSTableColumnIterator(reader), Integer.MIN_VALUE);
-
- DeletionTime delTime = cf.deletionInfo().deletionTimeFor(cf.getComparator().make(ByteBufferUtil.bytes("superA")));
- assertEquals("supercolumn deletion time did not match the expected time", new DeletionInfo(0, 0), new DeletionInfo(delTime));
- Cell subCell = cf.getColumn(Util.cellname("superA", "636f6c4141"));
- assert subCell.value().equals(hexToBytes("76616c75654141"));
- }
-
- @Test
- public void testImportUnsortedDataWithSortedOptionFails() throws IOException, URISyntaxException
- {
- String jsonUrl = resourcePath("UnsortedSuperCF.json");
- File tempSS = tempSSTableFile("Keyspace1", "Super4");
-
- int result = new SSTableImport(3,true, true).importJson(jsonUrl, "Keyspace1","Super4", tempSS.getPath());
- assert result == -1;
- }
-
- @Test
public void testImportUnsortedMode() throws IOException, URISyntaxException
{
String jsonUrl = resourcePath("UnsortedCF.json");
[3/3] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cf74bcfc
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cf74bcfc
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cf74bcfc
Branch: refs/heads/trunk
Commit: cf74bcfc63f75c8ae188f804f3f42ef5cfc8379a
Parents: e6325fa 647bfc6
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Apr 23 22:00:29 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Apr 23 22:00:29 2014 -0500
----------------------------------------------------------------------
NEWS.txt | 3 +-
.../apache/cassandra/tools/SSTableExport.java | 93 +++++--------
.../apache/cassandra/tools/SSTableImport.java | 133 ++++---------------
test/resources/CounterCF.json | 2 +-
test/resources/SimpleCF.json | 4 +-
test/resources/SimpleCF.oldformat.json | 4 -
test/resources/SimpleCFWithDeletionInfo.json | 4 +-
test/resources/SuperCF.json | 4 -
test/resources/UnsortedCF.json | 4 +-
test/resources/UnsortedSuperCF.json | 5 -
.../cassandra/tools/SSTableExportTest.java | 40 +-----
.../cassandra/tools/SSTableImportTest.java | 51 -------
12 files changed, 76 insertions(+), 271 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/cf74bcfc/NEWS.txt
----------------------------------------------------------------------
[2/3] git commit: make sstable2json output more readable, and remove
support for supercolumn json (CASSANDRA-7078) patch by jbellis;
reviewed by ayeschenko for CASSANDRA-7078
Posted by jb...@apache.org.
make sstable2json output more readable, and remove support for supercolumn json (CASSANDRA-7078)
patch by jbellis; reviewed by ayeschenko for CASSANDRA-7078
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/647bfc6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/647bfc6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/647bfc6d
Branch: refs/heads/trunk
Commit: 647bfc6d7985a9e07734315424fa6cc89e587df3
Parents: 6acbb33
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Apr 23 22:00:13 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Apr 23 22:00:21 2014 -0500
----------------------------------------------------------------------
NEWS.txt | 3 +-
.../apache/cassandra/tools/SSTableExport.java | 93 +++++--------
.../apache/cassandra/tools/SSTableImport.java | 133 ++++---------------
test/resources/CounterCF.json | 2 +-
test/resources/SimpleCF.json | 4 +-
test/resources/SimpleCF.oldformat.json | 4 -
test/resources/SimpleCFWithDeletionInfo.json | 4 +-
test/resources/SuperCF.json | 4 -
test/resources/UnsortedCF.json | 4 +-
test/resources/UnsortedSuperCF.json | 5 -
.../cassandra/tools/SSTableExportTest.java | 40 +-----
.../cassandra/tools/SSTableImportTest.java | 51 -------
12 files changed, 76 insertions(+), 271 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index ac78a73..86c6f64 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -48,9 +48,10 @@ Upgrading
to change your data model to accomodate the new implementation.
(See https://issues.apache.org/jira/browse/CASSANDRA-6504 and the dev
blog post at http://www.datastax.com/dev/blog/<PLACEHOLDER> for details).
- - (per-table) index_interval parameter has been replaced with
+ - (per-table) index_interval parameter has been replaced with
min_index_interval and max_index_interval paratemeters. index_interval
has been deprecated.
+ - support for supercolumns has been removed from json2sstable
2.0.7
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index bbc3494..02deb65 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -20,7 +20,6 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
-import java.nio.ByteBuffer;
import java.util.*;
import org.apache.commons.cli.*;
@@ -98,53 +97,6 @@ public class SSTableExport
out.print(": ");
}
- /**
- * JSON ColumnFamily metadata serializer.</br> Serializes:
- * <ul>
- * <li>column family deletion info (if present)</li>
- * </ul>
- *
- * @param out The output steam to write data
- * @param deletionInfo
- */
- private static void writeMeta(PrintStream out, DeletionInfo deletionInfo)
- {
- if (!deletionInfo.isLive())
- {
- // begin meta
- writeKey(out, "metadata");
- writeDeletionInfo(out, deletionInfo.getTopLevelDeletion());
- out.print(",");
- }
- }
-
- private static void writeDeletionInfo(PrintStream out, DeletionTime deletionTime)
- {
- out.print("{");
- writeKey(out, "deletionInfo");
- // only store topLevelDeletion (serializeForSSTable only uses this)
- writeJSON(out, deletionTime);
- out.print("}");
- }
-
- /**
- * Serialize columns using given column iterator
- *
- * @param atoms column iterator
- * @param out output stream
- * @param cfMetaData Column Family metadata (to get validator)
- */
- private static void serializeAtoms(Iterator<OnDiskAtom> atoms, PrintStream out, CFMetaData cfMetaData)
- {
- while (atoms.hasNext())
- {
- writeJSON(out, serializeAtom(atoms.next(), cfMetaData));
-
- if (atoms.hasNext())
- out.print(", ");
- }
- }
-
private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData)
{
if (atom instanceof Cell)
@@ -166,7 +118,17 @@ public class SSTableExport
}
/**
- * Serialize a given cell to the JSON format
+ * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Format is
+ *
+ * human_readable_name, value, timestamp, [flag, [options]]
+ *
+ * Value is normally the human readable value as rendered by the validator, but for deleted cells we
+ * give the local deletion time instead.
+ *
+ * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter:
+ * - No options for deleted cells
+ * - If expiring, options will include the TTL and local deletion time.
+ * - If counter, options will include timestamp of last delete
*
* @param cell cell presentation
* @param cfMetaData Column Family metadata (to get validator)
@@ -177,18 +139,18 @@ public class SSTableExport
CellNameType comparator = cfMetaData.comparator;
ArrayList<Object> serializedColumn = new ArrayList<Object>();
- ByteBuffer value = cell.value();
-
serializedColumn.add(comparator.getString(cell.name()));
+
if (cell instanceof DeletedCell)
{
- serializedColumn.add(ByteBufferUtil.bytesToHex(value));
+ serializedColumn.add(cell.getLocalDeletionTime());
}
else
{
AbstractType<?> validator = cfMetaData.getValueValidator(cell.name());
- serializedColumn.add(validator.getString(value));
+ serializedColumn.add(validator.getString(cell.value()));
}
+
serializedColumn.add(cell.timestamp());
if (cell instanceof DeletedCell)
@@ -227,16 +189,31 @@ public class SSTableExport
out.print("{");
writeKey(out, "key");
writeJSON(out, bytesToHex(key.key));
- out.print(",");
+ out.print(",\n");
- writeMeta(out, deletionInfo);
+ if (!deletionInfo.isLive())
+ {
+ out.print(" ");
+ writeKey(out, "metadata");
+ out.print("{");
+ writeKey(out, "deletionInfo");
+ writeJSON(out, deletionInfo.getTopLevelDeletion());
+ out.print("}");
+ out.print(",\n");
+ }
- writeKey(out, "columns");
+ out.print(" ");
+ writeKey(out, "cells");
out.print("[");
+ while (atoms.hasNext())
+ {
+ writeJSON(out, serializeAtom(atoms.next(), metadata));
- serializeAtoms(atoms, out, metadata);
-
+ if (atoms.hasNext())
+ out.print(",\n ");
+ }
out.print("]");
+
out.print("}");
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 6bd2119..6b34f59 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -63,14 +63,12 @@ public class SSTableImport
private static final String COLUMN_FAMILY_OPTION = "c";
private static final String KEY_COUNT_OPTION = "n";
private static final String IS_SORTED_OPTION = "s";
- private static final String OLD_SC_FORMAT_OPTION = "S";
private static final Options options = new Options();
private static CommandLine cmd;
private Integer keyCountToImport;
private final boolean isSorted;
- private final boolean oldSCFormat;
private static final JsonFactory factory = new MappingJsonFactory().configure(
JsonParser.Feature.INTERN_FIELD_NAMES, false);
@@ -87,7 +85,6 @@ public class SSTableImport
options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional)."));
options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional)."));
- options.addOption(new Option(OLD_SC_FORMAT_OPTION, false, "Assume JSON file use legacy super column format (Optional)."));
}
private static class JsonColumn<T>
@@ -104,11 +101,11 @@ public class SSTableImport
// Counter columns
private long timestampOfLastDelete;
- public JsonColumn(T json, CFMetaData meta, boolean oldSCFormat, boolean isSubColumn)
+ public JsonColumn(T json, CFMetaData meta)
{
if (json instanceof List)
{
- CellNameType comparator = oldSCFormat ? new SimpleDenseCellNameType(SuperColumns.getComparatorFor(meta, isSubColumn)) : meta.comparator;
+ CellNameType comparator = meta.comparator;
List fields = (List<?>) json;
assert fields.size() >= 3 : "Cell definition should have at least 3";
@@ -119,42 +116,25 @@ public class SSTableImport
if (fields.size() > 3)
{
- if (fields.get(3) instanceof Boolean)
+ kind = (String) fields.get(3);
+ if (isExpiring())
{
- // old format, reading this for backward compatibility sake
- if (fields.size() == 6)
- {
- kind = "e";
- ttl = (Integer) fields.get(4);
- localExpirationTime = (Integer) fields.get(5);
- }
- else
- {
- kind = ((Boolean) fields.get(3)) ? "d" : "";
- }
+ ttl = (Integer) fields.get(4);
+ localExpirationTime = (Integer) fields.get(5);
}
- else
+ else if (isCounter())
{
- kind = (String) fields.get(3);
- if (isExpiring())
- {
- ttl = (Integer) fields.get(4);
- localExpirationTime = (Integer) fields.get(5);
- }
- else if (isCounter())
- {
- timestampOfLastDelete = (long) ((Integer) fields.get(4));
- }
- else if (isRangeTombstone())
- {
- localExpirationTime = (Integer) fields.get(4);
- }
+ timestampOfLastDelete = (long) ((Integer) fields.get(4));
+ }
+ else if (isRangeTombstone())
+ {
+ localExpirationTime = (Integer) fields.get(4);
}
}
if (isDeleted())
{
- value = ByteBufferUtil.hexToBytes((String) fields.get(1));
+ value = ByteBufferUtil.bytes((Integer) fields.get(1));
}
else if (isRangeTombstone())
{
@@ -200,61 +180,43 @@ public class SSTableImport
public SSTableImport()
{
- this(null, false, false);
+ this(null, false);
}
public SSTableImport(boolean isSorted)
{
- this(isSorted, false);
- }
-
- public SSTableImport(boolean isSorted, boolean oldSCFormat)
- {
- this(null, isSorted, oldSCFormat);
+ this(null, isSorted);
}
- public SSTableImport(Integer keyCountToImport, boolean isSorted, boolean oldSCFormat)
+ public SSTableImport(Integer keyCountToImport, boolean isSorted)
{
this.keyCountToImport = keyCountToImport;
this.isSorted = isSorted;
- this.oldSCFormat = oldSCFormat;
- }
-
- private void addToStandardCF(List<?> row, ColumnFamily cfamily)
- {
- addColumnsToCF(row, null, cfamily);
}
/**
* Add columns to a column family.
*
* @param row the columns associated with a row
- * @param superName name of the super column if any
* @param cfamily the column family to add columns to
*/
- private void addColumnsToCF(List<?> row, ByteBuffer superName, ColumnFamily cfamily)
+ private void addColumnsToCF(List<?> row, ColumnFamily cfamily)
{
CFMetaData cfm = cfamily.metadata();
assert cfm != null;
for (Object c : row)
{
- JsonColumn col = new JsonColumn<List>((List) c, cfm, oldSCFormat, (superName != null));
+ JsonColumn col = new JsonColumn<List>((List) c, cfm);
if (col.isRangeTombstone())
{
- Composite start = superName == null
- ? cfm.comparator.fromByteBuffer(col.getName())
- : cfm.comparator.make(superName, col.getName());
- Composite end = superName == null
- ? cfm.comparator.fromByteBuffer(col.getValue())
- : cfm.comparator.make(superName, col.getValue());
+ Composite start = cfm.comparator.fromByteBuffer(col.getName());
+ Composite end = cfm.comparator.fromByteBuffer(col.getValue());
cfamily.addAtom(new RangeTombstone(start, end, col.timestamp, col.localExpirationTime));
continue;
}
- CellName cname = superName == null
- ? cfm.comparator.cellFromByteBuffer(col.getName())
- : cfm.comparator.makeCellName(superName, col.getName());
+ CellName cname = cfm.comparator.cellFromByteBuffer(col.getName());
if (col.isExpiring())
{
@@ -270,9 +232,7 @@ public class SSTableImport
}
else if (col.isRangeTombstone())
{
- CellName end = superName == null
- ? cfm.comparator.cellFromByteBuffer(col.getValue())
- : cfm.comparator.makeCellName(superName, col.getValue());
+ CellName end = cfm.comparator.cellFromByteBuffer(col.getValue());
cfamily.addAtom(new RangeTombstone(cname, end, col.timestamp, col.localExpirationTime));
}
// cql3 row marker, see CASSANDRA-5852
@@ -305,35 +265,6 @@ public class SSTableImport
}
/**
- * Add super columns to a column family.
- *
- * @param row the super columns associated with a row
- * @param cfamily the column family to add columns to
- */
- private void addToSuperCF(Map<?, ?> row, ColumnFamily cfamily)
- {
- CFMetaData metaData = cfamily.metadata();
- assert metaData != null;
-
- CellNameType comparator = metaData.comparator;
-
- // Super columns
- for (Map.Entry<?, ?> entry : row.entrySet())
- {
- Map<?, ?> data = (Map<?, ?>) entry.getValue();
-
- ByteBuffer superName = stringAsType((String) entry.getKey(), comparator.subtype(0));
-
- addColumnsToCF((List<?>) data.get("subColumns"), superName, cfamily);
-
- if (data.containsKey("metadata"))
- {
- parseMeta((Map<?, ?>) data.get("metadata"), cfamily, superName);
- }
- }
- }
-
- /**
* Convert a JSON formatted file to an SSTable.
*
* @param jsonFile the file containing JSON formatted data
@@ -387,11 +318,8 @@ public class SSTableImport
parseMeta((Map<?, ?>) row.getValue().get("metadata"), columnFamily, null);
}
- Object columns = row.getValue().get("columns");
- if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat)
- addToSuperCF((Map<?, ?>) columns, columnFamily);
- else
- addToStandardCF((List<?>) columns, columnFamily);
+ Object columns = row.getValue().get("cells");
+ addColumnsToCF((List<?>) columns, columnFamily);
writer.append(row.getKey(), columnFamily);
@@ -458,10 +386,7 @@ public class SSTableImport
if (row.containsKey("metadata"))
parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null);
- if (columnFamily.getType() == ColumnFamilyType.Super && oldSCFormat)
- addToSuperCF((Map<?, ?>)row.get("columns"), columnFamily);
- else
- addToStandardCF((List<?>)row.get("columns"), columnFamily);
+ addColumnsToCF((List<?>) row.get("cells"), columnFamily);
if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
{
@@ -545,7 +470,6 @@ public class SSTableImport
Integer keyCountToImport = null;
boolean isSorted = false;
- boolean oldSCFormat = false;
if (cmd.hasOption(KEY_COUNT_OPTION))
{
@@ -557,11 +481,6 @@ public class SSTableImport
isSorted = true;
}
- if (cmd.hasOption(OLD_SC_FORMAT_OPTION))
- {
- oldSCFormat = true;
- }
-
DatabaseDescriptor.loadSchemas();
if (Schema.instance.getNonSystemKeyspaces().size() < 1)
{
@@ -572,7 +491,7 @@ public class SSTableImport
try
{
- new SSTableImport(keyCountToImport, isSorted, oldSCFormat).importJson(json, keyspace, cfamily, ssTable);
+ new SSTableImport(keyCountToImport, isSorted).importJson(json, keyspace, cfamily, ssTable);
}
catch (Exception e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/CounterCF.json
----------------------------------------------------------------------
diff --git a/test/resources/CounterCF.json b/test/resources/CounterCF.json
index b6bca73..dcd087f 100644
--- a/test/resources/CounterCF.json
+++ b/test/resources/CounterCF.json
@@ -1,3 +1,3 @@
[
- {"key": "726f7741", "columns": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]}
+ {"key": "726f7741", "cells": [["636f6c4141", "000100008c619170467411e00000fe8ebeead9ee0000000000000001000000000000002a", 1294532915068, "c", 0]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCF.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCF.json b/test/resources/SimpleCF.json
index 45f57eb..7ef95f6 100644
--- a/test/resources/SimpleCF.json
+++ b/test/resources/SimpleCF.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
+ {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
+ {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCF.oldformat.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCF.oldformat.json b/test/resources/SimpleCF.oldformat.json
deleted file mode 100644
index d920cfb..0000000
--- a/test/resources/SimpleCF.oldformat.json
+++ /dev/null
@@ -1,4 +0,0 @@
-[
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068, false], ["636f6c4142", "76616c4142", 1294532915069, false], ["636f6c4143", "76616c4143", 1294532915071, false, 42, 2000000000 ]]},
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070, false], ["636f6c4242", "76616c4242", 1294532915073, false]]}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SimpleCFWithDeletionInfo.json
----------------------------------------------------------------------
diff --git a/test/resources/SimpleCFWithDeletionInfo.json b/test/resources/SimpleCFWithDeletionInfo.json
index ef673f2..5090699 100644
--- a/test/resources/SimpleCFWithDeletionInfo.json
+++ b/test/resources/SimpleCFWithDeletionInfo.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
- {"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
+ {"key": "726f7741","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]},
+ {"key": "726f7742","metadata":{"deletionInfo":{"markedForDeleteAt":0,"localDeletionTime":0}}, "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/SuperCF.json
----------------------------------------------------------------------
diff --git a/test/resources/SuperCF.json b/test/resources/SuperCF.json
deleted file mode 100644
index 5afec7f..0000000
--- a/test/resources/SuperCF.json
+++ /dev/null
@@ -1,4 +0,0 @@
-[
- {"key": "726f7741", "columns": {"737570657241": {"metadata": {"deletionInfo": {"markedForDeleteAt":0,"localDeletionTime":0}}, "subColumns": [["636f6c4141", "76616c75654141", 1294532915069], ["636f6c4142", "76616c75654142", 1294532915069]]}}},
- {"key": "726f7742", "columns": {"737570657242": {"subColumns": [["636f6c4241", "76616c75654241", 1294532915069], ["636f6c4242", "76616c75654242", 1294532915069]]}}}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/UnsortedCF.json
----------------------------------------------------------------------
diff --git a/test/resources/UnsortedCF.json b/test/resources/UnsortedCF.json
index 814f182..64ef24f 100644
--- a/test/resources/UnsortedCF.json
+++ b/test/resources/UnsortedCF.json
@@ -1,4 +1,4 @@
[
- {"key": "726f7742", "columns": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]},
- {"key": "726f7741", "columns": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}
+ {"key": "726f7742", "cells": [["636f6c4241", "76616c4241", 1294532915070], ["636f6c4242", "76616c4242", 1294532915073]]},
+ {"key": "726f7741", "cells": [["636f6c4141", "76616c4141", 1294532915068], ["636f6c4142", "76616c4142", 1294532915069], ["636f6c4143", "76616c4143", 1294532915071, "e", 42, 2000000000]]}
]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/resources/UnsortedSuperCF.json
----------------------------------------------------------------------
diff --git a/test/resources/UnsortedSuperCF.json b/test/resources/UnsortedSuperCF.json
deleted file mode 100644
index bd07e81..0000000
--- a/test/resources/UnsortedSuperCF.json
+++ /dev/null
@@ -1,5 +0,0 @@
-[
- {"key": "303935", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366338333439636337323630", 1294656637116, false], ["4331", "366338333439636337323630", 1294656637116, false], ["4332", "366338333439636337323630", 1294656637116, false], ["4333", "366338333439636337323630", 1294656637116, false], ["4334", "366338333439636337323630", 1294656637116, false]]}}} ,
- {"key": "303630", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "643364393434363830326134", 1294656636902, false], ["4331", "643364393434363830326134", 1294656636902, false], ["4332", "643364393434363830326134", 1294656636902, false], ["4333", "643364393434363830326134", 1294656636902, false], ["4334", "643364393434363830326134", 1294656636902, false]]}}} ,
- {"key": "303638", "columns": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366634393232663435353638", 1294656636885, false], ["4331", "366634393232663435353638", 1294656636885, false], ["4332", "366634393232663435353638", 1294656636885, false], ["4333", "366634393232663435353638", 1294656636885, false], ["4334", "366634393232663435353638", 1294656636885, false]]}}}
-]
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index 454600d..9e2014d 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -56,31 +56,6 @@ public class SSTableExportTest extends SchemaLoader
return bytesToHex(ByteBufferUtil.bytes(str));
}
- public SSTableWriter getDummyWriter() throws IOException
- {
- File tempSS = tempSSTableFile("Keyspace1", "Standard1");
- ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
-
- // Add rowA
- cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
- writer.append(Util.dk("rowA"), cfamily);
- cfamily.clear();
-
- cfamily.addColumn(Util.cellname("colB"), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
- writer.append(Util.dk("rowB"), cfamily);
- cfamily.clear();
-
-
- return writer;
-
- }
-
-
- public PrintStream dummyStream = new PrintStream(new OutputStream(){
- public void write(int b) throws IOException { throw new IOException(); }
- });
-
@Test
public void testEnumeratekeys() throws IOException
{
@@ -153,7 +128,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowA.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),rowA.get("key"));
- JSONArray colsA = (JSONArray)rowA.get("columns");
+ JSONArray colsA = (JSONArray)rowA.get("cells");
JSONArray colA = (JSONArray)colsA.get(0);
assert hexToBytes((String)colA.get(1)).equals(ByteBufferUtil.bytes("valA"));
@@ -165,7 +140,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, rowB.keySet().size());
assertEquals("unexpected row key",asHex("rowB"),rowB.get("key"));
- JSONArray colsB = (JSONArray)rowB.get("columns");
+ JSONArray colsB = (JSONArray)rowB.get("cells");
JSONArray colB = (JSONArray)colsB.get(0);
assert colB.size() == 3;
@@ -235,7 +210,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
- JSONArray cols = (JSONArray)row.get("columns");
+ JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("colA"));
assert ((String) colA.get(3)).equals("c");
@@ -267,7 +242,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals("unexpected number of keys", 2, row.keySet().size());
assertEquals("unexpected row key",asHex("rowA"),row.get("key"));
- JSONArray cols = (JSONArray)row.get("columns");
+ JSONArray cols = (JSONArray)row.get("cells");
JSONArray colA = (JSONArray)cols.get(0);
assert hexToBytes((String)colA.get(0)).equals(ByteBufferUtil.bytes("data"));
assert colA.get(1).equals("{\"foo\":\"bar\"}");
@@ -276,7 +251,6 @@ public class SSTableExportTest extends SchemaLoader
@Test
public void testExportColumnsWithMetadata() throws IOException, ParseException
{
-
File tempSS = tempSSTableFile("Keyspace1", "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "Standard1");
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
@@ -293,7 +267,6 @@ public class SSTableExportTest extends SchemaLoader
SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
- System.out.println(json.toJSONString());
assertEquals("unexpected number of rows", 1, json.size());
JSONObject row = (JSONObject)json.get(0);
@@ -320,7 +293,7 @@ public class SSTableExportTest extends SchemaLoader
serializedDeletionInfo.toJSONString());
// check the colums are what we put in
- JSONArray cols = (JSONArray) row.get("columns");
+ JSONArray cols = (JSONArray) row.get("cells");
assertNotNull("expecing columns to be present", cols);
assertEquals("expecting two columns", 2, cols.size());
@@ -331,7 +304,6 @@ public class SSTableExportTest extends SchemaLoader
JSONArray col2 = (JSONArray) cols.get(1);
assertEquals("column name did not match", ByteBufferUtil.bytes("colName1"), hexToBytes((String) col2.get(0)));
assertEquals("column value did not match", ByteBufferUtil.bytes("val1"), hexToBytes((String) col2.get(1)));
-
}
/**
@@ -357,7 +329,7 @@ public class SSTableExportTest extends SchemaLoader
assertEquals(1, json.size());
JSONObject row = (JSONObject)json.get(0);
- JSONArray cols = (JSONArray) row.get("columns");
+ JSONArray cols = (JSONArray) row.get("cells");
assertEquals(1, cols.size());
// check column name and value
http://git-wip-us.apache.org/repos/asf/cassandra/blob/647bfc6d/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index 3576005..3befc44 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@ -75,57 +75,6 @@ public class SSTableImportTest extends SchemaLoader
}
@Test
- public void testImportSimpleCfOldFormat() throws IOException, URISyntaxException
- {
- // Import JSON to temp SSTable file
- String jsonUrl = resourcePath("SimpleCF.oldformat.json");
- File tempSS = tempSSTableFile("Keyspace1", "Standard1");
- new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "Standard1", tempSS.getPath());
-
- // Verify results
- SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
- QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Standard1", System.currentTimeMillis());
- OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
- ColumnFamily cf = cloneForAdditions(iter);
- while (iter.hasNext()) cf.addAtom(iter.next());
- assert cf.getColumn(Util.cellname("colAA")).value().equals(hexToBytes("76616c4141"));
- assert !(cf.getColumn(Util.cellname("colAA")) instanceof DeletedCell);
- Cell expCol = cf.getColumn(Util.cellname("colAC"));
- assert expCol.value().equals(hexToBytes("76616c4143"));
- assert expCol instanceof ExpiringCell;
- assert ((ExpiringCell)expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000;
- }
-
- @Test
- public void testImportSuperCf() throws IOException, URISyntaxException
- {
- String jsonUrl = resourcePath("SuperCF.json");
- File tempSS = tempSSTableFile("Keyspace1", "Super4");
- new SSTableImport(true, true).importJson(jsonUrl, "Keyspace1", "Super4", tempSS.getPath());
-
- // Verify results
- SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
- QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "Super4", System.currentTimeMillis());
- ColumnFamily cf = cloneForAdditions(qf.getSSTableColumnIterator(reader));
- qf.collateOnDiskAtom(cf, qf.getSSTableColumnIterator(reader), Integer.MIN_VALUE);
-
- DeletionTime delTime = cf.deletionInfo().deletionTimeFor(cf.getComparator().make(ByteBufferUtil.bytes("superA")));
- assertEquals("supercolumn deletion time did not match the expected time", new DeletionInfo(0, 0), new DeletionInfo(delTime));
- Cell subCell = cf.getColumn(Util.cellname("superA", "636f6c4141"));
- assert subCell.value().equals(hexToBytes("76616c75654141"));
- }
-
- @Test
- public void testImportUnsortedDataWithSortedOptionFails() throws IOException, URISyntaxException
- {
- String jsonUrl = resourcePath("UnsortedSuperCF.json");
- File tempSS = tempSSTableFile("Keyspace1", "Super4");
-
- int result = new SSTableImport(3,true, true).importJson(jsonUrl, "Keyspace1","Super4", tempSS.getPath());
- assert result == -1;
- }
-
- @Test
public void testImportUnsortedMode() throws IOException, URISyntaxException
{
String jsonUrl = resourcePath("UnsortedCF.json");