You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ee...@apache.org on 2011/02/09 22:38:57 UTC
svn commit: r1069125 -
/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
Author: eevans
Date: Wed Feb 9 21:38:57 2011
New Revision: 1069125
URL: http://svn.apache.org/viewvc?rev=1069125&view=rev
Log:
reverting the insanity of my last merge
Modified:
cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=1069125&r1=1069124&r2=1069125&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableExport.java Wed Feb 9 21:38:57 2011
@@ -29,6 +29,11 @@ import org.apache.commons.cli.*;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ExpiringColumn;
+import org.apache.cassandra.db.IColumn;
+import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.*;
@@ -40,8 +45,7 @@ import static org.apache.cassandra.utils
*/
public class SSTableExport
{
- // size of the columns page
- private static final int PAGE_SIZE = 1000;
+ private static int INPUT_FILE_BUFFER_SIZE = 8 * 1024 * 1024;
private static final String KEY_OPTION = "k";
private static final String EXCLUDEKEY_OPTION = "x";
@@ -66,22 +70,12 @@ public class SSTableExport
Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only");
options.addOption(optEnumerate);
}
-
- /**
- * Wraps given string into quotes
- * @param val string to quote
- * @return quoted string
- */
+
private static String quote(String val)
{
return String.format("\"%s\"", val);
}
-
- /**
- * JSON Hash Key serializer
- * @param val value to set as a key
- * @return JSON Hash key
- */
+
private static String asKey(String val)
{
return String.format("%s: ", quote(val));
@@ -89,9 +83,7 @@ public class SSTableExport
private static void serializeColumns(PrintStream outs, Collection<IColumn> columns, AbstractType comparator, CFMetaData cfMetaData)
{
- while (columns.hasNext())
- {
- serializeColumn(columns.next(), out);
+ outs.print("[");
Iterator<IColumn> iter = columns.iterator();
@@ -126,18 +118,11 @@ public class SSTableExport
outs.print(", ");
}
}
-
- out.print("]");
+
+ outs.print("]");
}
-
- /**
- * Get portion of the columns and serialize in loop while not more columns left in the row
- * @param reader SSTableReader for given SSTable
- * @param row SSTableIdentityIterator row representation with Column Family
- * @param key Decorated Key for the required row
- * @param out output stream
- */
- private static void serializeRow(SSTableReader reader, SSTableIdentityIterator row, DecoratedKey key, PrintStream out)
+
+ private static void serializeRow(PrintStream outs, SSTableIdentityIterator row) throws IOException
{
ColumnFamily columnFamily = row.getColumnFamilyWithColumns();
CFMetaData cfMetaData = columnFamily.metadata();
@@ -149,12 +134,7 @@ public class SSTableExport
if (columnFamily.isSuper())
{
- QueryFilter filter = QueryFilter.getSliceFilter(key,
- new QueryPath(columnFamily.metadata().tableName),
- startColumn,
- ByteBufferUtil.EMPTY_BYTE_BUFFER,
- false,
- PAGE_SIZE);
+ outs.print("{ ");
Iterator<IColumn> iter = columnFamily.getSortedColumns().iterator();
while (iter.hasNext())
@@ -179,55 +159,8 @@ public class SSTableExport
outs.print(", ");
}
}
-
- try
- {
- columns = filter.getSSTableColumnIterator(reader); // iterator reset
- serializeRow(columns, isSuperCF, out);
- }
- catch (IOException e)
- {
- System.err.println("WARNING: Corrupt row " + key + " (skipping).");
- }
-
- if (columnCount < PAGE_SIZE)
- break;
- }
-
- out.print(isSuperCF ? "}" : "]");
- }
-
- /**
- * Serialize a row with already given column iterator
- *
- * @param columns columns of the row
- * @param isSuper true if wrapping Column Family is Super
- * @param out output stream
- *
- * @throws IOException on any I/O error.
- */
- private static void serializeRow(IColumnIterator columns, boolean isSuper, PrintStream out) throws IOException
- {
- if (isSuper)
- {
- while (columns.hasNext())
- {
- IColumn column = columns.next();
-
- out.print(asKey(bytesToHex(column.name())));
- out.print("{");
- out.print(asKey("deletedAt"));
- out.print(column.getMarkedForDeleteAt());
- out.print(", ");
- out.print(asKey("subColumns"));
- out.print("[");
- serializeColumns(column.getSubColumns(), out);
- out.print("]");
- out.print("}");
-
- if (columns.hasNext())
- out.print(", ");
- }
+
+ outs.print("}");
}
else
{
@@ -266,10 +199,9 @@ public class SSTableExport
/**
* Export specific rows from an SSTable and write the resulting JSON to a PrintStream.
*
- * @param ssTableFile the SSTableScanner to export the rows from
+ * @param ssTableFile the SSTable to export the rows from
* @param outs PrintStream to write the output to
- * @param toExport the keys corresponding to the rows to export
- * @param excludes keys to exclude from export
+ * @param keys the keys corresponding to the rows to export
* @param excludes the keys to exclude from export
*
* @throws IOException on failure to read/write input/output
@@ -277,23 +209,20 @@ public class SSTableExport
public static void export(String ssTableFile, PrintStream outs, String[] keys, String[] excludes) throws IOException
{
SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(ssTableFile));
- SSTableScanner scanner = reader.getDirectScanner(BufferedRandomAccessFile.DEFAULT_BUFFER_SIZE);
-
- IPartitioner<?> partitioner = StorageService.getPartitioner();
-
- for (String toExclude : excludes)
- {
- toExport.remove(toExclude); // excluding key from export
- }
+ SSTableScanner scanner = reader.getDirectScanner(INPUT_FILE_BUFFER_SIZE);
+ IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
+ Set<String> excludeSet = new HashSet<String>();
+ int i = 0;
+ if (excludes != null)
+ excludeSet = new HashSet<String>(Arrays.asList(excludes));
+
outs.println("{");
- int i = 0;
-
- // last key to compare order
+ // last key to compare order
DecoratedKey lastKey = null;
-
- for (String key : toExport)
+
+ for (String key : keys)
{
if (excludeSet.contains(key))
continue;
@@ -306,65 +235,84 @@ public class SSTableExport
lastKey = dk;
- lastKey = decoratedKey;
-
- scanner.seekTo(decoratedKey);
-
- if (!scanner.hasNext())
- continue;
+ scanner.seekTo(dk);
+
+ i++;
- serializeRow(reader, (SSTableIdentityIterator) scanner.next(), decoratedKey, outs);
+ if (scanner.hasNext())
+ {
+ SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
- if (i != 0)
- outs.println(",");
+ try
+ {
+ serializeRow(outs, row);
+ }
+ catch (IOException ioexc)
+ {
+ System.err.println("WARNING: Corrupt row " + key + " (skipping).");
+ continue;
+ }
+ catch (OutOfMemoryError oom)
+ {
+ System.err.println("ERROR: Out of memory deserializing row " + key);
+ continue;
+ }
- i++;
+ if (i != 1)
+ outs.println(",");
+ }
}
-
+
outs.println("\n}");
outs.flush();
-
- scanner.close();
}
// This is necessary to accommodate the test suite since you cannot open a Reader more
// than once from within the same process.
static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException
{
+ SSTableScanner scanner = reader.getDirectScanner(INPUT_FILE_BUFFER_SIZE);
Set<String> excludeSet = new HashSet<String>();
if (excludes != null)
excludeSet = new HashSet<String>(Arrays.asList(excludes));
-
- SSTableIdentityIterator row;
- SSTableScanner scanner = reader.getDirectScanner(BufferedRandomAccessFile.DEFAULT_BUFFER_SIZE);
-
outs.println("{");
- int i = 0;
+ SSTableIdentityIterator row;
- // collecting keys to export
+ boolean elementWritten = false;
while (scanner.hasNext())
{
row = (SSTableIdentityIterator) scanner.next();
- String currentKey = bytesToHex(row.getKey().key);
-
- if (excludeSet.contains(currentKey))
+ if (excludeSet.contains(bytesToHex(row.getKey().key)))
continue;
- else if (i != 0)
+ else if (elementWritten)
outs.println(",");
- serializeRow(reader, row, row.getKey(), outs);
+ try
+ {
+ serializeRow(outs, row);
- i++;
+ // used to decide should we put ',' after previous row or not
+ if (!elementWritten)
+ elementWritten = true;
+ }
+ catch (IOException ioexcep)
+ {
+ System.err.println("WARNING: Corrupt row " + bytesToHex(row.getKey().key) + " (skipping).");
+ elementWritten = false;
+ }
+ catch (OutOfMemoryError oom)
+ {
+ System.err.println("ERROR: Out of memory deserializing row " + bytesToHex(row.getKey().key));
+ elementWritten = false;
+ }
}
-
- outs.println("\n}");
+
+ outs.printf("%n}%n");
outs.flush();
-
- scanner.close();
}
/**
@@ -378,7 +326,8 @@ public class SSTableExport
*/
public static void export(String ssTableFile, PrintStream outs, String[] excludes) throws IOException
{
- export(SSTableReader.open(Descriptor.fromFilename(ssTableFile)), outs, excludes);
+ SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(ssTableFile));
+ export(reader, outs, excludes);
}
/**
@@ -399,7 +348,6 @@ public class SSTableExport
* export the contents of the SSTable to JSON.
*
* @param args command lines arguments
- *
* @throws IOException on failure to open/read/write files or output streams
* @throws ConfigurationException if configuration is invalid
*/
@@ -447,7 +395,7 @@ public class SSTableExport
else
{
if ((keys != null) && (keys.length > 0))
- export(ssTableFileName, System.out, Arrays.asList(keys), excludes);
+ export(ssTableFileName, System.out, keys, excludes);
else
export(ssTableFileName, excludes);
}