You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by br...@apache.org on 2011/01/14 00:09:10 UTC
svn commit: r1058788 - in /cassandra/branches/cassandra-0.7:
src/java/org/apache/cassandra/tools/ test/resources/
test/unit/org/apache/cassandra/tools/
Author: brandonwilliams
Date: Thu Jan 13 23:09:10 2011
New Revision: 1058788
URL: http://svn.apache.org/viewvc?rev=1058788&view=rev
Log:
Streaming support for json2sstable.
Patch by Pavel Yaskevich, reviewed by brandonwilliams for CASSANDRA-1898
Added:
cassandra/branches/cassandra-0.7/test/resources/UnsortedSuperCF.json
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableExport.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableImport.java
cassandra/branches/cassandra-0.7/test/resources/SimpleCF.json
cassandra/branches/cassandra-0.7/test/resources/SuperCF.json
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableExport.java Thu Jan 13 23:09:10 2011
@@ -251,19 +251,25 @@ public class SSTableExport
outs.println("{");
+ SSTableIdentityIterator row;
+
+ boolean elementWritten = false;
while (scanner.hasNext())
{
- SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
+ row = (SSTableIdentityIterator) scanner.next();
+
if (excludeSet.contains(bytesToHex(row.getKey().key)))
continue;
+ else if (elementWritten)
+ outs.println(",");
+
try
{
serializeRow(outs, row);
- outs.print(" ");
- if (scanner.hasNext())
- outs.println(",");
- else
- outs.println();
+
+ // used to decide should we put ',' after previous row or not
+ if (!elementWritten)
+ elementWritten = true;
}
catch (IOException ioexcep)
{
@@ -277,7 +283,7 @@ public class SSTableExport
}
}
- outs.println("}");
+ outs.printf("%n}%n");
outs.flush();
}
Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableImport.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableImport.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/tools/SSTableImport.java Thu Jan 13 23:09:10 2011
@@ -18,13 +18,10 @@
package org.apache.cassandra.tools;
-import java.io.FileReader;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
+import java.util.*;
import org.apache.commons.cli.*;
@@ -35,10 +32,12 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.SSTableWriter;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.JSONValue;
-import org.json.simple.parser.ParseException;
+import org.codehaus.jackson.type.TypeReference;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.map.MappingJsonFactory;
+
+import org.codehaus.jackson.JsonParser;
import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
@@ -48,71 +47,108 @@ import static org.apache.cassandra.utils
public class SSTableImport
{
private static final String KEYSPACE_OPTION = "K";
- private static final String COLFAM_OPTION = "c";
+ private static final String COLUMN_FAMILY_OPTION = "c";
+ private static final String KEY_COUNT_OPTION = "n";
+ private static final String IS_SORTED_OPTION = "s";
+
private static Options options;
private static CommandLine cmd;
+ private static Integer keyCountToImport = null;
+ private static boolean isSorted = false;
+
+ private static JsonFactory factory = new MappingJsonFactory();
+
static
{
options = new Options();
- Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace name");
+
+ Option optKeyspace = new Option(KEYSPACE_OPTION, true, "Keyspace name.");
optKeyspace.setRequired(true);
options.addOption(optKeyspace);
- Option optColfamily = new Option(COLFAM_OPTION, true, "Column family");
+
+ Option optColfamily = new Option(COLUMN_FAMILY_OPTION, true, "Column Family name.");
optColfamily.setRequired(true);
options.addOption(optColfamily);
+
+ options.addOption(new Option(KEY_COUNT_OPTION, true, "Number of keys to import (Optional)."));
+ options.addOption(new Option(IS_SORTED_OPTION, false, "Assume JSON file as already sorted (e.g. created by sstable2json tool) (Optional)."));
}
- private static class JsonColumn
+ private static class JsonColumn<T>
{
- private String name;
- private String value;
+ private ByteBuffer name;
+ private ByteBuffer value;
private long timestamp;
private boolean isDeleted;
private int ttl;
private int localExpirationTime;
-
- private JsonColumn(Object obj) throws ClassCastException
+
+ public JsonColumn(T json)
{
- JSONArray colSpec = (JSONArray)obj;
- assert colSpec.size() == 4 || colSpec.size() == 6;
- name = (String)colSpec.get(0);
- value = (String)colSpec.get(1);
- timestamp = (Long)colSpec.get(2);
- isDeleted = (Boolean)colSpec.get(3);
- if (colSpec.size() == 6)
+ if (json instanceof List)
{
- ttl = (int)(long)((Long)colSpec.get(4));
- localExpirationTime = (int)(long)((Long)colSpec.get(5));
+ List fields = (List<?>) json;
+
+ assert fields.size() == 4 || fields.size() == 6 : "Column definition should have 4 or 6 fields.";
+
+ name = ByteBuffer.wrap(hexToBytes((String) fields.get(0)));
+ value = ByteBuffer.wrap(hexToBytes((String) fields.get(1)));
+ timestamp = (Long) fields.get(2);
+ isDeleted = (Boolean) fields.get(3);
+
+ if (fields.size() == 6)
+ {
+ ttl = (Integer) fields.get(4);
+ localExpirationTime = (int) (long) ((Long) fields.get(5));
+ }
}
}
+
+ public ByteBuffer getName()
+ {
+ return name.duplicate();
+ }
+
+ public ByteBuffer getValue()
+ {
+ return value.duplicate();
+ }
+ }
+
+ private static void addToStandardCF(List<?> row, ColumnFamily cfamily)
+ {
+ addColumnsToCF(row, null, cfamily);
}
/**
* Add columns to a column family.
*
* @param row the columns associated with a row
+ * @param superName name of the super column if any
* @param cfamily the column family to add columns to
*/
- private static void addToStandardCF(JSONArray row, ColumnFamily cfamily)
+ private static void addColumnsToCF(List<?> row, ByteBuffer superName, ColumnFamily cfamily)
{
CFMetaData cfm = cfamily.metadata();
assert cfm != null;
+
for (Object c : row)
{
- JsonColumn col = new JsonColumn(c);
- QueryPath path = new QueryPath(cfm.cfName, null, ByteBuffer.wrap(hexToBytes(col.name)));
+ JsonColumn col = new JsonColumn<List>((List) c);
+ QueryPath path = new QueryPath(cfm.cfName, superName, col.getName());
+
if (col.ttl > 0)
{
- cfamily.addColumn(null, new ExpiringColumn(ByteBuffer.wrap(hexToBytes(col.name)), ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp, col.ttl, col.localExpirationTime));
+ cfamily.addColumn(null, new ExpiringColumn(col.getName(), col.getValue(), col.timestamp, col.ttl, col.localExpirationTime));
}
else if (col.isDeleted)
{
- cfamily.addTombstone(path, ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp);
+ cfamily.addTombstone(path, col.getValue(), col.timestamp);
}
else
{
- cfamily.addColumn(path, ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp);
+ cfamily.addColumn(path, col.getValue(), col.timestamp);
}
}
}
@@ -123,38 +159,23 @@ public class SSTableImport
* @param row the super columns associated with a row
* @param cfamily the column family to add columns to
*/
- private static void addToSuperCF(JSONObject row, ColumnFamily cfamily)
+ private static void addToSuperCF(Map<?, ?> row, ColumnFamily cfamily)
{
- CFMetaData cfm = cfamily.metadata();
- assert cfm != null;
+ CFMetaData metaData = cfamily.metadata();
+ assert metaData != null;
+
// Super columns
- for (Map.Entry<String, JSONObject> entry : (Set<Map.Entry<String, JSONObject>>)row.entrySet())
+ for (Map.Entry<?, ?> entry : row.entrySet())
{
- ByteBuffer superName = ByteBuffer.wrap(hexToBytes(entry.getKey()));
- long deletedAt = (Long)entry.getValue().get("deletedAt");
- JSONArray subColumns = (JSONArray)entry.getValue().get("subColumns");
-
- // Add sub-columns
- for (Object c : subColumns)
- {
- JsonColumn col = new JsonColumn(c);
- QueryPath path = new QueryPath(cfm.cfName, superName, ByteBuffer.wrap(hexToBytes(col.name)));
- if (col.ttl > 0)
- {
- cfamily.addColumn(superName, new ExpiringColumn(ByteBuffer.wrap(hexToBytes(col.name)), ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp, col.ttl, col.localExpirationTime));
- }
- else if (col.isDeleted)
- {
- cfamily.addTombstone(path, ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp);
- }
- else
- {
- cfamily.addColumn(path, ByteBuffer.wrap(hexToBytes(col.value)), col.timestamp);
- }
- }
-
- SuperColumn superColumn = (SuperColumn)cfamily.getColumn(superName);
- superColumn.markForDeleteAt((int)(System.currentTimeMillis()/1000), deletedAt);
+ ByteBuffer superName = ByteBuffer.wrap(hexToBytes((String) entry.getKey()));
+ Map<?, ?> data = (Map<?, ?>) entry.getValue();
+
+ addColumnsToCF((List<?>) data.get("subColumns"), superName, cfamily);
+
+ // *WARNING* markForDeleteAt has been DEPRECATED at Cassandra side
+ //BigInteger deletedAt = (BigInteger) data.get("deletedAt");
+ //SuperColumn superColumn = (SuperColumn) cfamily.getColumn(superName);
+ //superColumn.markForDeleteAt((int) (System.currentTimeMillis()/1000), deletedAt);
}
}
@@ -165,44 +186,181 @@ public class SSTableImport
* @param keyspace keyspace the data belongs to
* @param cf column family the data belongs to
* @param ssTablePath file to write the SSTable to
+ *
* @throws IOException for errors reading/writing input/output
- * @throws ParseException for errors encountered parsing JSON input
*/
- public static void importJson(String jsonFile, String keyspace, String cf, String ssTablePath)
- throws IOException, ParseException
+ public static void importJson(String jsonFile, String keyspace, String cf, String ssTablePath) throws IOException
{
- ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
- ColumnFamilyType cfType = cfamily.getColumnFamilyType(); // Super or Standard
+ ColumnFamily columnFamily = ColumnFamily.create(keyspace, cf);
IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
- try
+ int importedKeys = (isSorted) ? importSorted(jsonFile, columnFamily, ssTablePath, partitioner)
+ : importUnsorted(getParser(jsonFile), columnFamily, ssTablePath, partitioner);
+
+ if (importedKeys != -1)
+ System.out.printf("%d keys imported successfully.%n", importedKeys);
+ }
+
+ private static int importUnsorted(JsonParser parser, ColumnFamily columnFamily, String ssTablePath, IPartitioner<?> partitioner) throws IOException
+ {
+ int importedKeys = 0;
+ long start = System.currentTimeMillis();
+ Map<?, ?> data = parser.readValueAs(new TypeReference<Map<?, ?>>() {});
+
+ keyCountToImport = (keyCountToImport == null) ? data.size() : keyCountToImport;
+ SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);
+
+ System.out.printf("Importing %s keys...%n", keyCountToImport);
+
+ // sort by dk representation, but hold onto the hex version
+ SortedMap<DecoratedKey,String> decoratedKeys = new TreeMap<DecoratedKey,String>();
+
+ for (Object keyObject : data.keySet())
{
- JSONObject json = (JSONObject)JSONValue.parseWithException(new FileReader(jsonFile));
-
- SSTableWriter writer = new SSTableWriter(ssTablePath, json.size());
- SortedMap<DecoratedKey,String> decoratedKeys = new TreeMap<DecoratedKey,String>();
-
- // sort by dk representation, but hold onto the hex version
- for (String key : (Set<String>)json.keySet())
- decoratedKeys.put(partitioner.decorateKey(ByteBuffer.wrap(hexToBytes(key))), key);
+ String key = (String) keyObject;
+ decoratedKeys.put(partitioner.decorateKey(ByteBuffer.wrap(hexToBytes(key))), key);
+ }
- for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet())
+ for (Map.Entry<DecoratedKey, String> rowKey : decoratedKeys.entrySet())
+ {
+ if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Super)
{
- if (cfType == ColumnFamilyType.Super)
- addToSuperCF((JSONObject)json.get(rowKey.getValue()), cfamily);
- else
- addToStandardCF((JSONArray)json.get(rowKey.getValue()), cfamily);
-
- writer.append(rowKey.getKey(), cfamily);
- cfamily.clear();
+ addToSuperCF((Map<?, ?>) data.get(rowKey.getValue()), columnFamily);
}
-
- writer.closeAndOpenReader();
+ else
+ {
+ addToStandardCF((List<?>) data.get(rowKey.getValue()), columnFamily);
+ }
+
+ writer.append(rowKey.getKey(), columnFamily);
+ columnFamily.clear();
+
+ importedKeys++;
+
+ long current = System.currentTimeMillis();
+
+ if (current - start >= 5000) // 5 secs.
+ {
+ System.out.printf("Currently imported %d keys.%n", importedKeys);
+ start = current;
+ }
+
+ if (keyCountToImport == importedKeys)
+ break;
}
- catch (ClassCastException cce)
+
+ writer.closeAndOpenReader();
+
+ return importedKeys;
+ }
+
+ public static int importSorted(String jsonFile, ColumnFamily columnFamily, String ssTablePath, IPartitioner<?> partitioner) throws IOException
+ {
+ int importedKeys = 0; // already imported keys count
+ long start = System.currentTimeMillis();
+
+ JsonParser parser = getParser(jsonFile);
+
+ if (keyCountToImport == null)
{
- throw new RuntimeException("Invalid JSON input, or incorrect column family.", cce);
+ keyCountToImport = 0;
+ System.out.println("Counting keys to import, please wait... (NOTE: to skip this use -n <num_keys>)");
+
+ parser.nextToken(); // START_OBJECT
+ while (parser.nextToken() != null)
+ {
+ parser.nextToken();
+ parser.skipChildren();
+ if (parser.getCurrentName() == null) continue;
+
+ keyCountToImport++;
+ }
+ }
+
+ System.out.printf("Importing %s keys...%n", keyCountToImport);
+
+ parser = getParser(jsonFile); // renewing parser
+ SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);
+
+ int lineNumber = 1;
+ DecoratedKey prevStoredKey = null;
+
+ while (parser.nextToken() != null)
+ {
+ String key = parser.getCurrentName();
+
+ if (key != null)
+ {
+ String tokenName = parser.nextToken().name();
+
+ if (tokenName.equals("START_ARRAY"))
+ {
+ if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Super)
+ {
+ throw new RuntimeException("Can't write Standard columns to the Super Column Family.");
+ }
+
+ List<?> columns = parser.readValueAs(new TypeReference<List<?>>() {});
+ addToStandardCF(columns, columnFamily);
+ }
+ else if (tokenName.equals("START_OBJECT"))
+ {
+ if (columnFamily.getColumnFamilyType() == ColumnFamilyType.Standard)
+ {
+ throw new RuntimeException("Can't write Super columns to the Standard Column Family.");
+ }
+
+ Map<?, ?> columns = parser.readValueAs(new TypeReference<Map<?, ?>>() {});
+ addToSuperCF(columns, columnFamily);
+ }
+ else
+ {
+ throw new UnsupportedOperationException("Only Array or Hash allowed as row content.");
+ }
+
+ DecoratedKey currentKey = partitioner.decorateKey(ByteBuffer.wrap(hexToBytes(key)));
+
+ if (prevStoredKey != null && prevStoredKey.compareTo(currentKey) != -1)
+ {
+ System.err.printf("Line %d: Key %s is greater than previous, collection is not sorted properly. Aborting import. You might need to delete SSTables manually.%n", lineNumber, key);
+ return -1;
+ }
+
+ // saving decorated key
+ writer.append(currentKey, columnFamily);
+ columnFamily.clear();
+
+ prevStoredKey = currentKey;
+ importedKeys++;
+ lineNumber++;
+
+ long current = System.currentTimeMillis();
+
+ if (current - start >= 5000) // 5 secs.
+ {
+ System.out.printf("Currently imported %d keys.%n", importedKeys);
+ start = current;
+ }
+
+ if (keyCountToImport == importedKeys)
+ break;
+ }
}
+
+ writer.closeAndOpenReader();
+
+ return importedKeys;
+ }
+
+ /**
+ * Get JsonParser object for file
+ * @param fileName name of the file
+ * @return json parser instance for given file
+ * @throws IOException if any I/O error.
+ */
+ private static JsonParser getParser(String fileName) throws IOException
+ {
+ return factory.createJsonParser(new File(fileName));
}
/**
@@ -212,33 +370,43 @@ public class SSTableImport
* @param args command line arguments
* @throws IOException on failure to open/read/write files or output streams
* @throws ParseException on failure to parse JSON input
+ * @throws ConfigurationException on configuration error.
*/
public static void main(String[] args) throws IOException, ParseException, ConfigurationException
{
- String usage = String.format("Usage: %s -K keyspace -c column_family <json> <sstable>%n",
- SSTableImport.class.getName());
-
CommandLineParser parser = new PosixParser();
+
try
{
cmd = parser.parse(options, args);
- } catch (org.apache.commons.cli.ParseException e1)
+ }
+ catch (org.apache.commons.cli.ParseException e)
{
- System.err.println(e1.getMessage());
- System.err.println(usage);
+ System.err.println(e.getMessage());
+ printProgramUsage();
System.exit(1);
}
if (cmd.getArgs().length != 2)
{
- System.err.println(usage);
+ printProgramUsage();
System.exit(1);
}
- String json = cmd.getArgs()[0];
- String ssTable = cmd.getArgs()[1];
+ String json = cmd.getArgs()[0];
+ String ssTable = cmd.getArgs()[1];
String keyspace = cmd.getOptionValue(KEYSPACE_OPTION);
- String cfamily = cmd.getOptionValue(COLFAM_OPTION);
+ String cfamily = cmd.getOptionValue(COLUMN_FAMILY_OPTION);
+
+ if (cmd.hasOption(KEY_COUNT_OPTION))
+ {
+ keyCountToImport = Integer.valueOf(cmd.getOptionValue(KEY_COUNT_OPTION));
+ }
+
+ if (cmd.hasOption(IS_SORTED_OPTION))
+ {
+ isSorted = true;
+ }
DatabaseDescriptor.loadSchemas();
if (DatabaseDescriptor.getNonSystemTables().size() < 1)
@@ -248,9 +416,40 @@ public class SSTableImport
throw new ConfigurationException(msg);
}
- importJson(json, keyspace, cfamily, ssTable);
-
+ try
+ {
+ importJson(json, keyspace, cfamily, ssTable);
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ System.err.println("ERROR: " + e.getMessage());
+ System.exit(-1);
+ }
+
System.exit(0);
}
+ private static void printProgramUsage()
+ {
+ System.out.printf("Usage: %s -s -K <keyspace> -c <column_family> -n <num_keys> <json> <sstable>%n%n",
+ SSTableImport.class.getName());
+
+ System.out.println("Options:");
+ for (Object o : options.getOptions())
+ {
+ Option opt = (Option) o;
+ System.out.println(" -" +opt.getOpt() + " - " + opt.getDescription());
+ }
+ }
+
+ /**
+ * Used by test framework to set key count
+ * @param keyCount numbers of keys to import
+ */
+ public static void setKeyCountToImport(Integer keyCount)
+ {
+ keyCountToImport = keyCount;
+ }
+
}
Modified: cassandra/branches/cassandra-0.7/test/resources/SimpleCF.json
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/resources/SimpleCF.json?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/resources/SimpleCF.json (original)
+++ cassandra/branches/cassandra-0.7/test/resources/SimpleCF.json Thu Jan 13 23:09:10 2011
@@ -1,4 +1,4 @@
{
- "726f7741": [["636f6c4141", "76616c4141", 1, false], ["636f6c4142", "76616c4142", 1, false], ["636f6c4143", "76616c4143", 1, false, 42, 2000000000 ]],
- "726f7742": [["636f6c4241", "76616c4241", 1, false], ["636f6c4242", "76616c4242", 1, false]]
+ "726f7741": [["636f6c4141", "76616c4141", 1294532915068, false], ["636f6c4142", "76616c4142", 1294532915069, false], ["636f6c4143", "76616c4143", 1294532915071, false, 42, 2000000000 ]],
+ "726f7742": [["636f6c4241", "76616c4241", 1294532915070, false], ["636f6c4242", "76616c4242", 1294532915073, false]]
}
Modified: cassandra/branches/cassandra-0.7/test/resources/SuperCF.json
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/resources/SuperCF.json?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/resources/SuperCF.json (original)
+++ cassandra/branches/cassandra-0.7/test/resources/SuperCF.json Thu Jan 13 23:09:10 2011
@@ -1,4 +1,4 @@
{
- "726f7741": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141", "76616c75654141", 1, false], ["636f6c4142", "76616c75654142", 1, false]]}},
- "726f7742": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241", "76616c75654241", 1, false], ["636f6c4242", "76616c75654242", 1, false]]}}
+ "726f7741": {"737570657241": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4141", "76616c75654141", 1294532915069, false], ["636f6c4142", "76616c75654142", 1294532915069, false]]}},
+ "726f7742": {"737570657242": {"deletedAt": -9223372036854775808, "subColumns": [["636f6c4241", "76616c75654241", 1294532915069, false], ["636f6c4242", "76616c75654242", 1294532915069, false]]}}
}
Added: cassandra/branches/cassandra-0.7/test/resources/UnsortedSuperCF.json
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/resources/UnsortedSuperCF.json?rev=1058788&view=auto
==============================================================================
--- cassandra/branches/cassandra-0.7/test/resources/UnsortedSuperCF.json (added)
+++ cassandra/branches/cassandra-0.7/test/resources/UnsortedSuperCF.json Thu Jan 13 23:09:10 2011
@@ -0,0 +1,5 @@
+{
+ "303935": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366338333439636337323630", 1294656637116, false], ["4331", "366338333439636337323630", 1294656637116, false], ["4332", "366338333439636337323630", 1294656637116, false], ["4333", "366338333439636337323630", 1294656637116, false], ["4334", "366338333439636337323630", 1294656637116, false]]}} ,
+ "303630": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "643364393434363830326134", 1294656636902, false], ["4331", "643364393434363830326134", 1294656636902, false], ["4332", "643364393434363830326134", 1294656636902, false], ["4333", "643364393434363830326134", 1294656636902, false], ["4334", "643364393434363830326134", 1294656636902, false]]}} ,
+ "303638": { "5330": {"deletedAt": -9223372036854775808, "subColumns": [["4330", "366634393232663435353638", 1294656636885, false], ["4331", "366634393232663435353638", 1294656636885, false], ["4332", "366634393232663435353638", 1294656636885, false], ["4333", "366634393232663435353638", 1294656636885, false], ["4334", "366634393232663435353638", 1294656636885, false]]}}
+}
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Thu Jan 13 23:09:10 2011
@@ -65,12 +65,12 @@ public class SSTableExportTest extends S
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
// Add rowA
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
@@ -99,18 +99,18 @@ public class SSTableExportTest extends S
int nowInSec = (int)(System.currentTimeMillis() / 1000) + 42; //live for 42 seconds
// Add rowA
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), 1);
- cfamily.addColumn(null, new ExpiringColumn(ByteBufferUtil.bytes("colExp"), ByteBufferUtil.bytes("valExp"), 1, 42, nowInSec));
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
+ cfamily.addColumn(null, new ExpiringColumn(ByteBufferUtil.bytes("colExp"), ByteBufferUtil.bytes("valExp"), System.currentTimeMillis(), 42, nowInSec));
writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
// Add rowExclude
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
@@ -146,17 +146,17 @@ public class SSTableExportTest extends S
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
// Add rowA
- cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superA"), ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), 1);
+ cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superA"), ByteBufferUtil.bytes("colA")), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowB
- cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superB"), ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), 1);
+ cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superB"), ByteBufferUtil.bytes("colB")), ByteBufferUtil.bytes("valB"), System.currentTimeMillis());
writer.append(Util.dk("rowB"), cfamily);
cfamily.clear();
// Add rowExclude
- cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superX"), ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), 1);
+ cfamily.addColumn(new QueryPath("Super4", ByteBufferUtil.bytes("superX"), ByteBufferUtil.bytes("colX")), ByteBufferUtil.bytes("valX"), System.currentTimeMillis());
writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
@@ -186,12 +186,12 @@ public class SSTableExportTest extends S
SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
// Add rowA
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
writer.append(Util.dk("rowA"), cfamily);
cfamily.clear();
// Add rowExclude
- cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), 1);
+ cfamily.addColumn(new QueryPath("Standard1", null, ByteBufferUtil.bytes("name")), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
writer.append(Util.dk("rowExclude"), cfamily);
cfamily.clear();
Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java?rev=1058788&r1=1058787&r2=1058788&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/tools/SSTableImportTest.java Thu Jan 13 23:09:10 2011
@@ -21,7 +21,6 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Arrays;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -32,6 +31,7 @@ import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.columniterator.IColumnIterator;
+import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableReader;
import static org.apache.cassandra.utils.FBUtilities.hexToBytes;
@@ -44,13 +44,10 @@ import org.json.simple.parser.ParseExcep
import org.junit.Test;
import org.apache.cassandra.utils.ByteBufferUtil;
-
-import org.apache.cassandra.utils.FBUtilities;
-
public class SSTableImportTest extends SchemaLoader
{
@Test
- public void testImportSimpleCf() throws IOException, ParseException
+ public void testImportSimpleCf() throws IOException
{
// Import JSON to temp SSTable file
String jsonUrl = getClass().getClassLoader().getResource("SimpleCF.json").getPath();
@@ -68,7 +65,7 @@ public class SSTableImportTest extends S
IColumn expCol = cf.getColumn(ByteBufferUtil.bytes("colAC"));
assert expCol.value().equals(ByteBuffer.wrap(hexToBytes("76616c4143")));
assert expCol instanceof ExpiringColumn;
- assert ((ExpiringColumn)expCol).getTimeToLive() == 42 && ((ExpiringColumn)expCol).getLocalDeletionTime() == 2000000000;
+ assert ((ExpiringColumn)expCol).getTimeToLive() == 42 && expCol.getLocalDeletionTime() == 2000000000;
}
@Test
@@ -88,4 +85,18 @@ public class SSTableImportTest extends S
IColumn subColumn = superCol.getSubColumn(ByteBufferUtil.bytes("colAA"));
assert subColumn.value().equals(ByteBuffer.wrap(hexToBytes("76616c75654141")));
}
+
+ @Test
+ public void testImportUnsortedMode() throws IOException
+ {
+ String jsonUrl = getClass().getClassLoader().getResource("UnsortedSuperCF.json").getPath();
+ File tempSS = tempSSTableFile("Keyspace1", "Super4");
+
+ ColumnFamily columnFamily = ColumnFamily.create("Keyspace1", "Super4");
+ IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
+
+ SSTableImport.setKeyCountToImport(3);
+ int result = SSTableImport.importSorted(jsonUrl, columnFamily, tempSS.getPath(), partitioner);
+ assert result == -1;
+ }
}