You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/08/05 23:48:25 UTC
[1/3] git commit: SSTableExport uses correct validator to create
string representation of partition keys patch by Alexey Zotov;
reviewed by Benjamin Lerer for CASSANDRA-7498
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 bd191f18e -> 91d61b327
refs/heads/trunk 25041771f -> 50191828f
SSTableExport uses correct validator to create string representation of partition keys
patch by Alexey Zotov; reviewed by Benjamin Lerer for CASSANDRA-7498
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/91d61b32
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/91d61b32
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/91d61b32
Branch: refs/heads/cassandra-2.1
Commit: 91d61b3276ec5ffda85a99009a779f1ebcb4358a
Parents: bd191f1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Tue Aug 5 16:41:01 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Aug 5 16:41:09 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/tools/SSTableExport.java | 40 ++++++++-------
.../apache/cassandra/tools/SSTableImport.java | 23 +++++++--
.../unit/org/apache/cassandra/SchemaLoader.java | 3 +-
test/unit/org/apache/cassandra/Util.java | 6 +++
.../cassandra/tools/SSTableExportTest.java | 52 +++++++++++++++++---
.../cassandra/tools/SSTableImportTest.java | 40 ++++++++++++++-
7 files changed, 132 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d3e2f5a..79c84b2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.1.1
+ * SSTableExport uses correct validator to create string representation of partition
+ keys (CASSANDRA-7498)
* Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689)
* Add support for custom 2i validation (CASSANDRA-7575)
* Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index 41e9fdc..c4d693c 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -38,9 +38,6 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex;
-import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
-
/**
* Export SSTables to JSON format.
*/
@@ -188,7 +185,7 @@ public class SSTableExport
{
out.print("{");
writeKey(out, "key");
- writeJSON(out, bytesToHex(key.getKey()));
+ writeJSON(out, metadata.getKeyValidator().getString(key.getKey()));
out.print(",\n");
if (!deletionInfo.isLive())
@@ -222,9 +219,10 @@ public class SSTableExport
*
* @param desc the descriptor of the file to export the rows from
* @param outs PrintStream to write the output to
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void enumeratekeys(Descriptor desc, PrintStream outs)
+ public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata)
throws IOException
{
KeyIterator iter = new KeyIterator(desc);
@@ -238,7 +236,7 @@ public class SSTableExport
throw new IOException("Key out of order! " + lastKey + " > " + key);
lastKey = key;
- outs.println(bytesToHex(key.getKey()));
+ outs.println(metadata.getKeyValidator().getString(key.getKey()));
checkStream(outs); // flushes
}
iter.close();
@@ -251,9 +249,10 @@ public class SSTableExport
* @param outs PrintStream to write the output to
* @param toExport the keys corresponding to the rows to export
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes) throws IOException
+ public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes, CFMetaData metadata) throws IOException
{
SSTableReader sstable = SSTableReader.open(desc);
RandomAccessReader dfile = sstable.openDataReader();
@@ -272,7 +271,7 @@ public class SSTableExport
for (String key : toExport)
{
- DecoratedKey decoratedKey = partitioner.decorateKey(hexToBytes(key));
+ DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key));
if (lastKey != null && lastKey.compareTo(decoratedKey) > 0)
throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey);
@@ -302,7 +301,7 @@ public class SSTableExport
// This is necessary to accommodate the test suite since you cannot open a Reader more
// than once from within the same process.
- static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException
+ static void export(SSTableReader reader, PrintStream outs, String[] excludes, CFMetaData metadata) throws IOException
{
Set<String> excludeSet = new HashSet<String>();
@@ -322,7 +321,7 @@ public class SSTableExport
{
row = (SSTableIdentityIterator) scanner.next();
- String currentKey = bytesToHex(row.getKey().getKey());
+ String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey());
if (excludeSet.contains(currentKey))
continue;
@@ -347,11 +346,12 @@ public class SSTableExport
* @param desc the descriptor of the sstable to read from
* @param outs PrintStream to write the output to
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException
+ public static void export(Descriptor desc, PrintStream outs, String[] excludes, CFMetaData metadata) throws IOException
{
- export(SSTableReader.open(desc), outs, excludes);
+ export(SSTableReader.open(desc), outs, excludes, metadata);
}
/**
@@ -359,11 +359,12 @@ public class SSTableExport
*
* @param desc the descriptor of the sstable to read from
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write SSTable/standard out
*/
- public static void export(Descriptor desc, String[] excludes) throws IOException
+ public static void export(Descriptor desc, String[] excludes, CFMetaData metadata) throws IOException
{
- export(desc, System.out, excludes);
+ export(desc, System.out, excludes, metadata);
}
/**
@@ -415,7 +416,7 @@ public class SSTableExport
}
Keyspace keyspace = Keyspace.open(descriptor.ksname);
- // Make it work for indexes too - find parent cf if necessary
+ // Make it works for indexes too - find parent cf if necessary
String baseName = descriptor.cfname;
if (descriptor.cfname.contains("."))
{
@@ -424,9 +425,10 @@ public class SSTableExport
}
// IllegalArgumentException will be thrown here if ks/cf pair does not exist
+ ColumnFamilyStore cfStore = null;
try
{
- keyspace.getColumnFamilyStore(baseName);
+ cfStore = keyspace.getColumnFamilyStore(baseName);
}
catch (IllegalArgumentException e)
{
@@ -439,14 +441,14 @@ public class SSTableExport
{
if (cmd.hasOption(ENUMERATEKEYS_OPTION))
{
- enumeratekeys(descriptor, System.out);
+ enumeratekeys(descriptor, System.out, cfStore.metadata);
}
else
{
if ((keys != null) && (keys.length > 0))
- export(descriptor, System.out, Arrays.asList(keys), excludes);
+ export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata);
else
- export(descriptor, excludes);
+ export(descriptor, excludes, cfStore.metadata);
}
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 4e7bf06..5352eb4 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.tools;
-import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
-
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -308,7 +306,7 @@ public class SSTableImport
for (Object row : data)
{
Map<?,?> rowAsMap = (Map<?, ?>)row;
- decoratedKeys.put(partitioner.decorateKey(hexToBytes((String)rowAsMap.get("key"))), rowAsMap);
+ decoratedKeys.put(partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) rowAsMap.get("key"))), rowAsMap);
}
for (Map.Entry<DecoratedKey, Map<?, ?>> row : decoratedKeys.entrySet())
@@ -381,7 +379,7 @@ public class SSTableImport
{
String key = parser.getCurrentName();
Map<?, ?> row = parser.readValueAs(new TypeReference<Map<?, ?>>(){});
- DecoratedKey currentKey = partitioner.decorateKey(hexToBytes((String) row.get("key")));
+ DecoratedKey currentKey = partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) row.get("key")));
if (row.containsKey("metadata"))
parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null);
@@ -423,6 +421,21 @@ public class SSTableImport
}
/**
+ * Get key validator for column family
+ * @param columnFamily column family instance
+ * @return key validator for given column family
+ */
+ private AbstractType<?> getKeyValidator(ColumnFamily columnFamily) {
+ // this is a fix to support backward compatibility
+ // which allows to skip the current key validator
+ // please, take a look onto CASSANDRA-7498 for more details
+ if ("true".equals(System.getProperty("skip.key.validator", "false"))) {
+ return BytesType.instance;
+ }
+ return columnFamily.metadata().getKeyValidator();
+ }
+
+ /**
* Get JsonParser object for file
* @param fileName name of the file
* @return json parser instance for given file
@@ -526,7 +539,7 @@ public class SSTableImport
{
try
{
- return (type == BytesType.instance) ? hexToBytes(content) : type.fromString(content);
+ return type.fromString(content);
}
catch (MarshalException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index a9d69fd..0334117 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -180,7 +180,8 @@ public class SchemaLoader
standardCFMD(ks1, "UUIDKeys").keyValidator(UUIDType.instance),
CFMetaData.denseCFMetaData(ks1, "MixedTypes", LongType.instance).keyValidator(UUIDType.instance).defaultValidator(BooleanType.instance),
- CFMetaData.denseCFMetaData(ks1, "MixedTypesComposite", composite).keyValidator(composite).defaultValidator(BooleanType.instance)
+ CFMetaData.denseCFMetaData(ks1, "MixedTypesComposite", composite).keyValidator(composite).defaultValidator(BooleanType.instance),
+ standardCFMD(ks1, "AsciiKeys").keyValidator(AsciiType.instance)
));
// Keyspace 2
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/Util.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index 88031a4..1015be6 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.db.filter.IDiskAtomFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.SliceQueryFilter;
import org.apache.cassandra.db.filter.NamesQueryFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
@@ -65,6 +66,11 @@ public class Util
return StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(key));
}
+ public static DecoratedKey dk(String key, AbstractType type)
+ {
+ return StorageService.getPartitioner().decorateKey(type.fromString(key));
+ }
+
public static DecoratedKey dk(ByteBuffer key)
{
return StorageService.getPartitioner().decorateKey(key);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index d9133f6..c3f3419 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -28,7 +28,6 @@ import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.OutputStream;
import java.io.PrintStream;
import org.apache.cassandra.SchemaLoader;
@@ -36,6 +35,8 @@ import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
@@ -77,7 +78,9 @@ public class SSTableExportTest extends SchemaLoader
// Enumerate and verify
File temp = File.createTempFile("Standard1", ".txt");
- SSTableExport.enumeratekeys(Descriptor.fromFilename(writer.getFilename()), new PrintStream(temp.getPath()));
+ final Descriptor descriptor = Descriptor.fromFilename(writer.getFilename());
+ SSTableExport.enumeratekeys(descriptor, new PrintStream(temp.getPath()),
+ CFMetaData.sparseCFMetaData(descriptor.ksname, descriptor.cfname, BytesType.instance));
try (FileReader file = new FileReader(temp))
@@ -119,7 +122,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")},
+ CFMetaData.sparseCFMetaData("Keyspace1", "Standard1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 2, json.size());
@@ -168,7 +172,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")},
+ CFMetaData.sparseCFMetaData("Keyspace1", "Standard1", BytesType.instance));
// Import JSON to another SSTable file
File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
@@ -202,7 +207,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Counter1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "Counter1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -233,7 +239,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("ValuesWithQuotes", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "ValuesWithQuotes", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -264,7 +271,8 @@ public class SSTableExportTest extends SchemaLoader
SSTableReader reader = writer.closeAndOpenReader();
// Export to JSON and verify
File tempJson = File.createTempFile("CFWithDeletionInfo", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "Counter1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -323,7 +331,8 @@ public class SSTableExportTest extends SchemaLoader
SSTableReader reader = writer.closeAndOpenReader();
// Export to JSON and verify
File tempJson = File.createTempFile("CFWithColumnNameEqualToDefaultKeyAlias", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "UUIDKeys", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals(1, json.size());
@@ -337,4 +346,31 @@ public class SSTableExportTest extends SchemaLoader
assertEquals(CFMetaData.DEFAULT_KEY_ALIAS, ByteBufferUtil.string(hexToBytes((String) col.get(0))));
assertEquals("not a uuid", ByteBufferUtil.string(hexToBytes((String) col.get(1))));
}
+
+ @Test
+ public void testAsciiKeyValidator() throws IOException, ParseException
+ {
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "AsciiKeys");
+ SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
+
+ // Add a row
+ cfamily.addColumn(column("column", "value", 1L));
+ writer.append(Util.dk("key", AsciiType.instance), cfamily);
+
+ SSTableReader reader = writer.closeAndOpenReader();
+ // Export to JSON and verify
+ File tempJson = File.createTempFile("CFWithAsciiKeys", ".json");
+ SSTableExport.export(reader,
+ new PrintStream(tempJson.getPath()),
+ new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "AsciiKeys", BytesType.instance));
+
+ JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
+ assertEquals(1, json.size());
+
+ JSONObject row = (JSONObject)json.get(0);
+ // check row key
+ assertEquals("key", row.get("key"));
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index 2fdeaf4..edaf0fd 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@ -27,6 +27,8 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@ -36,7 +38,6 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
public class SSTableImportTest extends SchemaLoader
{
@@ -138,4 +139,41 @@ public class SSTableImportTest extends SchemaLoader
assert c instanceof CounterCell : c;
assert ((CounterCell) c).total() == 42;
}
+
+ @Test
+ public void testImportWithAsciiKeyValidator() throws IOException, URISyntaxException
+ {
+ // Import JSON to temp SSTable file
+ String jsonUrl = resourcePath("SimpleCF.json");
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "AsciiKeys", tempSS.getPath());
+
+ // Verify results
+ SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
+ // check that keys are treated as ascii
+ QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("726f7741", AsciiType.instance), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
+ assert iter.hasNext(); // "ascii" key exists
+ QueryFilter qf2 = QueryFilter.getIdentityFilter(Util.dk("726f7741", BytesType.instance), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter2 = qf2.getSSTableColumnIterator(reader);
+ assert !iter2.hasNext(); // "bytes" key does not exist
+ }
+
+ @Test
+ public void testBackwardCompatibilityOfImportWithAsciiKeyValidator() throws IOException, URISyntaxException
+ {
+ // Import JSON to temp SSTable file
+ String jsonUrl = resourcePath("SimpleCF.json");
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ // To ignore current key validator
+ System.setProperty("skip.key.validator", "true");
+ new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "AsciiKeys", tempSS.getPath());
+
+ // Verify results
+ SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
+ // check that keys are treated as bytes
+ QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
+ assert iter.hasNext(); // "bytes" key exists
+ }
}
[2/3] git commit: SSTableExport uses correct validator to create
string representation of partition keys patch by Alexey Zotov;
reviewed by Benjamin Lerer for CASSANDRA-7498
Posted by jb...@apache.org.
SSTableExport uses correct validator to create string representation of partition keys
patch by Alexey Zotov; reviewed by Benjamin Lerer for CASSANDRA-7498
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/91d61b32
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/91d61b32
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/91d61b32
Branch: refs/heads/trunk
Commit: 91d61b3276ec5ffda85a99009a779f1ebcb4358a
Parents: bd191f1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Tue Aug 5 16:41:01 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Aug 5 16:41:09 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/tools/SSTableExport.java | 40 ++++++++-------
.../apache/cassandra/tools/SSTableImport.java | 23 +++++++--
.../unit/org/apache/cassandra/SchemaLoader.java | 3 +-
test/unit/org/apache/cassandra/Util.java | 6 +++
.../cassandra/tools/SSTableExportTest.java | 52 +++++++++++++++++---
.../cassandra/tools/SSTableImportTest.java | 40 ++++++++++++++-
7 files changed, 132 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d3e2f5a..79c84b2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
2.1.1
+ * SSTableExport uses correct validator to create string representation of partition
+ keys (CASSANDRA-7498)
* Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689)
* Add support for custom 2i validation (CASSANDRA-7575)
* Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index 41e9fdc..c4d693c 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -38,9 +38,6 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
-import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex;
-import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
-
/**
* Export SSTables to JSON format.
*/
@@ -188,7 +185,7 @@ public class SSTableExport
{
out.print("{");
writeKey(out, "key");
- writeJSON(out, bytesToHex(key.getKey()));
+ writeJSON(out, metadata.getKeyValidator().getString(key.getKey()));
out.print(",\n");
if (!deletionInfo.isLive())
@@ -222,9 +219,10 @@ public class SSTableExport
*
* @param desc the descriptor of the file to export the rows from
* @param outs PrintStream to write the output to
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void enumeratekeys(Descriptor desc, PrintStream outs)
+ public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata)
throws IOException
{
KeyIterator iter = new KeyIterator(desc);
@@ -238,7 +236,7 @@ public class SSTableExport
throw new IOException("Key out of order! " + lastKey + " > " + key);
lastKey = key;
- outs.println(bytesToHex(key.getKey()));
+ outs.println(metadata.getKeyValidator().getString(key.getKey()));
checkStream(outs); // flushes
}
iter.close();
@@ -251,9 +249,10 @@ public class SSTableExport
* @param outs PrintStream to write the output to
* @param toExport the keys corresponding to the rows to export
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes) throws IOException
+ public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes, CFMetaData metadata) throws IOException
{
SSTableReader sstable = SSTableReader.open(desc);
RandomAccessReader dfile = sstable.openDataReader();
@@ -272,7 +271,7 @@ public class SSTableExport
for (String key : toExport)
{
- DecoratedKey decoratedKey = partitioner.decorateKey(hexToBytes(key));
+ DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key));
if (lastKey != null && lastKey.compareTo(decoratedKey) > 0)
throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey);
@@ -302,7 +301,7 @@ public class SSTableExport
// This is necessary to accommodate the test suite since you cannot open a Reader more
// than once from within the same process.
- static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException
+ static void export(SSTableReader reader, PrintStream outs, String[] excludes, CFMetaData metadata) throws IOException
{
Set<String> excludeSet = new HashSet<String>();
@@ -322,7 +321,7 @@ public class SSTableExport
{
row = (SSTableIdentityIterator) scanner.next();
- String currentKey = bytesToHex(row.getKey().getKey());
+ String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey());
if (excludeSet.contains(currentKey))
continue;
@@ -347,11 +346,12 @@ public class SSTableExport
* @param desc the descriptor of the sstable to read from
* @param outs PrintStream to write the output to
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write input/output
*/
- public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException
+ public static void export(Descriptor desc, PrintStream outs, String[] excludes, CFMetaData metadata) throws IOException
{
- export(SSTableReader.open(desc), outs, excludes);
+ export(SSTableReader.open(desc), outs, excludes, metadata);
}
/**
@@ -359,11 +359,12 @@ public class SSTableExport
*
* @param desc the descriptor of the sstable to read from
* @param excludes keys to exclude from export
+ * @param metadata Metadata to print keys in a proper format
* @throws IOException on failure to read/write SSTable/standard out
*/
- public static void export(Descriptor desc, String[] excludes) throws IOException
+ public static void export(Descriptor desc, String[] excludes, CFMetaData metadata) throws IOException
{
- export(desc, System.out, excludes);
+ export(desc, System.out, excludes, metadata);
}
/**
@@ -415,7 +416,7 @@ public class SSTableExport
}
Keyspace keyspace = Keyspace.open(descriptor.ksname);
- // Make it work for indexes too - find parent cf if necessary
+ // Make it works for indexes too - find parent cf if necessary
String baseName = descriptor.cfname;
if (descriptor.cfname.contains("."))
{
@@ -424,9 +425,10 @@ public class SSTableExport
}
// IllegalArgumentException will be thrown here if ks/cf pair does not exist
+ ColumnFamilyStore cfStore = null;
try
{
- keyspace.getColumnFamilyStore(baseName);
+ cfStore = keyspace.getColumnFamilyStore(baseName);
}
catch (IllegalArgumentException e)
{
@@ -439,14 +441,14 @@ public class SSTableExport
{
if (cmd.hasOption(ENUMERATEKEYS_OPTION))
{
- enumeratekeys(descriptor, System.out);
+ enumeratekeys(descriptor, System.out, cfStore.metadata);
}
else
{
if ((keys != null) && (keys.length > 0))
- export(descriptor, System.out, Arrays.asList(keys), excludes);
+ export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata);
else
- export(descriptor, excludes);
+ export(descriptor, excludes, cfStore.metadata);
}
}
catch (IOException e)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 4e7bf06..5352eb4 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -17,8 +17,6 @@
*/
package org.apache.cassandra.tools;
-import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
-
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -308,7 +306,7 @@ public class SSTableImport
for (Object row : data)
{
Map<?,?> rowAsMap = (Map<?, ?>)row;
- decoratedKeys.put(partitioner.decorateKey(hexToBytes((String)rowAsMap.get("key"))), rowAsMap);
+ decoratedKeys.put(partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) rowAsMap.get("key"))), rowAsMap);
}
for (Map.Entry<DecoratedKey, Map<?, ?>> row : decoratedKeys.entrySet())
@@ -381,7 +379,7 @@ public class SSTableImport
{
String key = parser.getCurrentName();
Map<?, ?> row = parser.readValueAs(new TypeReference<Map<?, ?>>(){});
- DecoratedKey currentKey = partitioner.decorateKey(hexToBytes((String) row.get("key")));
+ DecoratedKey currentKey = partitioner.decorateKey(getKeyValidator(columnFamily).fromString((String) row.get("key")));
if (row.containsKey("metadata"))
parseMeta((Map<?, ?>) row.get("metadata"), columnFamily, null);
@@ -423,6 +421,21 @@ public class SSTableImport
}
/**
+ * Get key validator for column family
+ * @param columnFamily column family instance
+ * @return key validator for given column family
+ */
+ private AbstractType<?> getKeyValidator(ColumnFamily columnFamily) {
+ // this is a fix to support backward compatibility
+ // which allows to skip the current key validator
+ // please, take a look onto CASSANDRA-7498 for more details
+ if ("true".equals(System.getProperty("skip.key.validator", "false"))) {
+ return BytesType.instance;
+ }
+ return columnFamily.metadata().getKeyValidator();
+ }
+
+ /**
* Get JsonParser object for file
* @param fileName name of the file
* @return json parser instance for given file
@@ -526,7 +539,7 @@ public class SSTableImport
{
try
{
- return (type == BytesType.instance) ? hexToBytes(content) : type.fromString(content);
+ return type.fromString(content);
}
catch (MarshalException e)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index a9d69fd..0334117 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -180,7 +180,8 @@ public class SchemaLoader
standardCFMD(ks1, "UUIDKeys").keyValidator(UUIDType.instance),
CFMetaData.denseCFMetaData(ks1, "MixedTypes", LongType.instance).keyValidator(UUIDType.instance).defaultValidator(BooleanType.instance),
- CFMetaData.denseCFMetaData(ks1, "MixedTypesComposite", composite).keyValidator(composite).defaultValidator(BooleanType.instance)
+ CFMetaData.denseCFMetaData(ks1, "MixedTypesComposite", composite).keyValidator(composite).defaultValidator(BooleanType.instance),
+ standardCFMD(ks1, "AsciiKeys").keyValidator(AsciiType.instance)
));
// Keyspace 2
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/Util.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index 88031a4..1015be6 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.db.filter.IDiskAtomFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.SliceQueryFilter;
import org.apache.cassandra.db.filter.NamesQueryFilter;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
@@ -65,6 +66,11 @@ public class Util
return StorageService.getPartitioner().decorateKey(ByteBufferUtil.bytes(key));
}
+ public static DecoratedKey dk(String key, AbstractType type)
+ {
+ return StorageService.getPartitioner().decorateKey(type.fromString(key));
+ }
+
public static DecoratedKey dk(ByteBuffer key)
{
return StorageService.getPartitioner().decorateKey(key);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index d9133f6..c3f3419 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -28,7 +28,6 @@ import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.OutputStream;
import java.io.PrintStream;
import org.apache.cassandra.SchemaLoader;
@@ -36,6 +35,8 @@ import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
@@ -77,7 +78,9 @@ public class SSTableExportTest extends SchemaLoader
// Enumerate and verify
File temp = File.createTempFile("Standard1", ".txt");
- SSTableExport.enumeratekeys(Descriptor.fromFilename(writer.getFilename()), new PrintStream(temp.getPath()));
+ final Descriptor descriptor = Descriptor.fromFilename(writer.getFilename());
+ SSTableExport.enumeratekeys(descriptor, new PrintStream(temp.getPath()),
+ CFMetaData.sparseCFMetaData(descriptor.ksname, descriptor.cfname, BytesType.instance));
try (FileReader file = new FileReader(temp))
@@ -119,7 +122,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")},
+ CFMetaData.sparseCFMetaData("Keyspace1", "Standard1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 2, json.size());
@@ -168,7 +172,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")},
+ CFMetaData.sparseCFMetaData("Keyspace1", "Standard1", BytesType.instance));
// Import JSON to another SSTable file
File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
@@ -202,7 +207,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("Counter1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "Counter1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -233,7 +239,8 @@ public class SSTableExportTest extends SchemaLoader
// Export to JSON and verify
File tempJson = File.createTempFile("ValuesWithQuotes", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "ValuesWithQuotes", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -264,7 +271,8 @@ public class SSTableExportTest extends SchemaLoader
SSTableReader reader = writer.closeAndOpenReader();
// Export to JSON and verify
File tempJson = File.createTempFile("CFWithDeletionInfo", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "Counter1", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals("unexpected number of rows", 1, json.size());
@@ -323,7 +331,8 @@ public class SSTableExportTest extends SchemaLoader
SSTableReader reader = writer.closeAndOpenReader();
// Export to JSON and verify
File tempJson = File.createTempFile("CFWithColumnNameEqualToDefaultKeyAlias", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0]);
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "UUIDKeys", BytesType.instance));
JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
assertEquals(1, json.size());
@@ -337,4 +346,31 @@ public class SSTableExportTest extends SchemaLoader
assertEquals(CFMetaData.DEFAULT_KEY_ALIAS, ByteBufferUtil.string(hexToBytes((String) col.get(0))));
assertEquals("not a uuid", ByteBufferUtil.string(hexToBytes((String) col.get(1))));
}
+
+ @Test
+ public void testAsciiKeyValidator() throws IOException, ParseException
+ {
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create("Keyspace1", "AsciiKeys");
+ SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
+
+ // Add a row
+ cfamily.addColumn(column("column", "value", 1L));
+ writer.append(Util.dk("key", AsciiType.instance), cfamily);
+
+ SSTableReader reader = writer.closeAndOpenReader();
+ // Export to JSON and verify
+ File tempJson = File.createTempFile("CFWithAsciiKeys", ".json");
+ SSTableExport.export(reader,
+ new PrintStream(tempJson.getPath()),
+ new String[0],
+ CFMetaData.sparseCFMetaData("Keyspace1", "AsciiKeys", BytesType.instance));
+
+ JSONArray json = (JSONArray)JSONValue.parseWithException(new FileReader(tempJson));
+ assertEquals(1, json.size());
+
+ JSONObject row = (JSONObject)json.get(0);
+ // check row key
+ assertEquals("key", row.get("key"));
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/91d61b32/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index 2fdeaf4..edaf0fd 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@ -27,6 +27,8 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@ -36,7 +38,6 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
public class SSTableImportTest extends SchemaLoader
{
@@ -138,4 +139,41 @@ public class SSTableImportTest extends SchemaLoader
assert c instanceof CounterCell : c;
assert ((CounterCell) c).total() == 42;
}
+
+ @Test
+ public void testImportWithAsciiKeyValidator() throws IOException, URISyntaxException
+ {
+ // Import JSON to temp SSTable file
+ String jsonUrl = resourcePath("SimpleCF.json");
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "AsciiKeys", tempSS.getPath());
+
+ // Verify results
+ SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
+ // check that keys are treated as ascii
+ QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("726f7741", AsciiType.instance), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
+ assert iter.hasNext(); // "ascii" key exists
+ QueryFilter qf2 = QueryFilter.getIdentityFilter(Util.dk("726f7741", BytesType.instance), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter2 = qf2.getSSTableColumnIterator(reader);
+ assert !iter2.hasNext(); // "bytes" key does not exist
+ }
+
+ @Test
+ public void testBackwardCompatibilityOfImportWithAsciiKeyValidator() throws IOException, URISyntaxException
+ {
+ // Import JSON to temp SSTable file
+ String jsonUrl = resourcePath("SimpleCF.json");
+ File tempSS = tempSSTableFile("Keyspace1", "AsciiKeys");
+ // To ignore current key validator
+ System.setProperty("skip.key.validator", "true");
+ new SSTableImport(true).importJson(jsonUrl, "Keyspace1", "AsciiKeys", tempSS.getPath());
+
+ // Verify results
+ SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
+ // check that keys are treated as bytes
+ QueryFilter qf = QueryFilter.getIdentityFilter(Util.dk("rowA"), "AsciiKeys", System.currentTimeMillis());
+ OnDiskAtomIterator iter = qf.getSSTableColumnIterator(reader);
+ assert iter.hasNext(); // "bytes" key exists
+ }
}
[3/3] git commit: merge from 2.1
Posted by jb...@apache.org.
merge from 2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/50191828
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/50191828
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/50191828
Branch: refs/heads/trunk
Commit: 50191828f4d421ae8acfa85019bec5eaecdfa6bb
Parents: 2504177 91d61b3
Author: Jonathan Ellis <jb...@apache.org>
Authored: Tue Aug 5 16:48:13 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Tue Aug 5 16:48:13 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/tools/SSTableExport.java | 40 +++++-----
.../apache/cassandra/tools/SSTableImport.java | 23 ++++--
.../unit/org/apache/cassandra/SchemaLoader.java | 3 +-
test/unit/org/apache/cassandra/Util.java | 6 ++
.../cassandra/tools/SSTableExportTest.java | 79 +++++++++++++++-----
.../cassandra/tools/SSTableImportTest.java | 56 ++++++++++++--
7 files changed, 160 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d60ed3d,79c84b2..991331f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,20 -1,6 +1,22 @@@
+3.0
+ * Permit configurable timestamps with cassandra-stress (CASSANDRA-7416)
+ * Move sstable RandomAccessReader to nio2, which allows using the
+ FILE_SHARE_DELETE flag on Windows (CASSANDRA-4050)
+ * Remove CQL2 (CASSANDRA-5918)
+ * Add Thrift get_multi_slice call (CASSANDRA-6757)
+ * Optimize fetching multiple cells by name (CASSANDRA-6933)
+ * Allow compilation in java 8 (CASSANDRA-7028)
+ * Make incremental repair default (CASSANDRA-7250)
+ * Enable code coverage thru JaCoCo (CASSANDRA-7226)
+ * Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
+ * Shorten SSTable path (CASSANDRA-6962)
+ * Use unsafe mutations for most unit tests (CASSANDRA-6969)
+ * Fix race condition during calculation of pending ranges (CASSANDRA-7390)
+
+
2.1.1
+ * SSTableExport uses correct validator to create string representation of partition
+ keys (CASSANDRA-7498)
* Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689)
* Add support for custom 2i validation (CASSANDRA-7575)
* Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/test/unit/org/apache/cassandra/Util.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index dbc6703,c3f3419..f1b7564
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@@ -18,30 -18,26 +18,33 @@@
*/
package org.apache.cassandra.tools;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
- import java.io.OutputStream;
import java.io.PrintStream;
++import org.junit.BeforeClass;
++import org.junit.Test;
++
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.config.KSMetaData;
- import org.apache.cassandra.db.*;
++import org.apache.cassandra.db.ArrayBackedSortedColumns;
++import org.apache.cassandra.db.BufferCell;
++import org.apache.cassandra.db.BufferCounterCell;
++import org.apache.cassandra.db.BufferExpiringCell;
++import org.apache.cassandra.db.ColumnFamily;
++import org.apache.cassandra.db.ColumnFamilyStore;
++import org.apache.cassandra.db.DeletionInfo;
++import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.filter.QueryFilter;
+ import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableWriter;
@@@ -54,30 -48,10 +57,35 @@@ import org.json.simple.JSONArray
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.json.simple.parser.ParseException;
- import org.junit.BeforeClass;
--import org.junit.Test;
+
-public class SSTableExportTest extends SchemaLoader
++import static org.apache.cassandra.Util.column;
++import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
++import static org.apache.cassandra.utils.ByteBufferUtil.bytesToHex;
++import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
++import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertNotNull;
+
+public class SSTableExportTest
{
+ public static final String KEYSPACE1 = "SSTableExportTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_COUNTER = "Counter1";
+ public static final String CF_UUID = "UUIDKeys";
+ public static final String CF_VALSWITHQUOTES = "ValuesWithQuotes";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException, IOException, TException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
+ CFMetaData.denseCFMetaData(KEYSPACE1, CF_COUNTER, BytesType.instance).defaultValidator(CounterColumnType.instance),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_UUID).keyValidator(UUIDType.instance),
+ CFMetaData.denseCFMetaData(KEYSPACE1, CF_VALSWITHQUOTES, BytesType.instance).defaultValidator(UTF8Type.instance));
+ }
+
public String asHex(String str)
{
return bytesToHex(ByteBufferUtil.bytes(str));
@@@ -195,11 -172,12 +206,12 @@@
// Export to JSON and verify
File tempJson = File.createTempFile("Standard1", ".json");
- SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")});
+ SSTableExport.export(reader, new PrintStream(tempJson.getPath()), new String[]{asHex("rowExclude")},
+ CFMetaData.sparseCFMetaData("Keyspace1", "Standard1", BytesType.instance));
// Import JSON to another SSTable file
- File tempSS2 = tempSSTableFile("Keyspace1", "Standard1");
- new SSTableImport().importJson(tempJson.getPath(), "Keyspace1", "Standard1", tempSS2.getPath());
+ File tempSS2 = tempSSTableFile(KEYSPACE1, "Standard1");
+ new SSTableImport().importJson(tempJson.getPath(), KEYSPACE1, "Standard1", tempSS2.getPath());
reader = SSTableReader.open(Descriptor.fromFilename(tempSS2.getPath()));
QueryFilter qf = Util.namesQueryFilter(cfs, Util.dk("rowA"), "name");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/50191828/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index dba8408,edaf0fd..8ea0ab8
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@@ -18,10 -18,10 +18,6 @@@
*/
package org.apache.cassandra.tools;
--import static org.junit.Assert.assertEquals;
--import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
--import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
--
import java.io.File;
import java.io.IOException;
import java.net.URI;
@@@ -32,36 -33,14 +28,47 @@@ import org.junit.Test
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
-import org.apache.cassandra.db.*;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.KSMetaData;
- import org.apache.cassandra.db.*;
++import org.apache.cassandra.db.ArrayBackedSortedColumns;
++import org.apache.cassandra.db.BufferDeletedCell;
++import org.apache.cassandra.db.Cell;
++import org.apache.cassandra.db.ColumnFamily;
++import org.apache.cassandra.db.CounterCell;
++import org.apache.cassandra.db.DeletionInfo;
++import org.apache.cassandra.db.ExpiringCell;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.filter.QueryFilter;
++import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.CounterColumnType;
+import org.apache.cassandra.exceptions.ConfigurationException;
- import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
++import org.apache.cassandra.locator.SimpleStrategy;
+import org.apache.thrift.TException;
-public class SSTableImportTest extends SchemaLoader
++import static org.apache.cassandra.io.sstable.SSTableUtils.tempSSTableFile;
++import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
++import static org.junit.Assert.assertEquals;
++
+public class SSTableImportTest
{
+ public static final String KEYSPACE1 = "SSTableImportTest";
+ public static final String CF_STANDARD = "Standard1";
+ public static final String CF_COUNTER = "Counter1";
+
+ @BeforeClass
+ public static void defineSchema() throws ConfigurationException, IOException, TException
+ {
+ SchemaLoader.prepareServer();
+ SchemaLoader.createKeyspace(KEYSPACE1,
+ SimpleStrategy.class,
+ KSMetaData.optsWithRF(1),
+ SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
+ CFMetaData.denseCFMetaData(KEYSPACE1, CF_COUNTER, BytesType.instance).defaultValidator(CounterColumnType.instance));
+ }
+
@Test
public void testImportSimpleCf() throws IOException, URISyntaxException
{