You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by br...@apache.org on 2010/09/04 05:31:38 UTC
svn commit: r992524 - in /cassandra/trunk:
src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/
src/java/org/apache/cassandra/io/sstable/
src/java/org/apache/cassandra/utils/ test/unit/org/apache/cassandra/db/
Author: brandonwilliams
Date: Sat Sep 4 03:31:36 2010
New Revision: 992524
URL: http://svn.apache.org/viewvc?rev=992524&view=rev
Log:
Move persistent sstable stats from the system table to an sstable component. Patch by brandonwilliams, reviewed by jbellis for CASSANDRA-1430
Removed:
cassandra/trunk/src/java/org/apache/cassandra/db/StatisticsTable.java
Modified:
cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Sat Sep 4 03:31:36 2010
@@ -39,7 +39,6 @@ import org.apache.cassandra.db.clock.Abs
import org.apache.cassandra.db.clock.TimestampReconciler;
import org.apache.cassandra.db.HintedHandOffManager;
import org.apache.cassandra.db.SystemTable;
-import org.apache.cassandra.db.StatisticsTable;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
@@ -65,7 +64,6 @@ public final class CFMetaData
public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance);
public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null);
public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null);
- public static final CFMetaData StatisticsCf = newSystemTable(StatisticsTable.STATISTICS_CF, 4, "persistent CF statistics for the local node", UTF8Type.instance, BytesType.instance);
private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator)
{
Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Sat Sep 4 03:31:36 2010
@@ -355,13 +355,11 @@ public class DatabaseDescriptor
CFMetaData.HintsCf,
CFMetaData.MigrationsCf,
CFMetaData.SchemaCf,
- CFMetaData.StatisticsCf
});
CFMetaData.map(CFMetaData.StatusCf);
CFMetaData.map(CFMetaData.HintsCf);
CFMetaData.map(CFMetaData.MigrationsCf);
CFMetaData.map(CFMetaData.SchemaCf);
- CFMetaData.map(CFMetaData.StatisticsCf);
tables.put(Table.SYSTEM_TABLE, systemMeta);
/* Load the seeds for node contact points */
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat Sep 4 03:31:36 2010
@@ -382,37 +382,6 @@ public class ColumnFamilyStore implement
}
}
- // returns runnables that need to update the system table.
- static Collection<Runnable> deleteCompactedFiles(String table, String columnFamily)
- {
- Collection<Runnable> runnables = new ArrayList<Runnable>();
- for (File file : files(table, columnFamily))
- {
- if (file.getName().contains("-Data.db"))
- {
- final String delPath = file.getAbsolutePath();
- if (SSTable.deleteIfCompacted(delPath))
- {
- runnables.add(new Runnable()
- {
- public void run()
- {
- try
- {
- StatisticsTable.deleteSSTableStatistics(delPath);
- }
- catch (IOException ex)
- {
- throw new RuntimeException(ex);
- }
- }
- });
- }
- }
- }
- return runnables;
- }
-
private static Set<File> files(String table, String columnFamily)
{
assert table != null;
Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Sat Sep 4 03:31:36 2010
@@ -97,17 +97,12 @@ public class Table
if (tableInstance == null)
{
// do some housekeeping on the column families.
- Collection<Runnable> systemTableUpdates = new ArrayList<Runnable>();
for (CFMetaData cfm : DatabaseDescriptor.getTableDefinition(table).cfMetaData().values())
{
ColumnFamilyStore.scrubDataDirectories(table, cfm.cfName);
- systemTableUpdates.addAll(ColumnFamilyStore.deleteCompactedFiles(table, cfm.cfName));
}
tableInstance = new Table(table);
instances.put(table, tableInstance);
-
- for (Runnable r : systemTableUpdates)
- r.run();
}
}
}
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Sat Sep 4 03:31:36 2010
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.db.StatisticsTable;
import org.apache.cassandra.utils.EstimatedHistogram;
/**
@@ -54,6 +53,7 @@ public abstract class SSTable
public static final String COMPONENT_DATA = "Data.db";
public static final String COMPONENT_INDEX = "Index.db";
public static final String COMPONENT_FILTER = "Filter.db";
+ public static final String COMPONENT_STATS = "Statistics.db";
public static final String COMPONENT_COMPACTED = "Compacted";
@@ -63,7 +63,7 @@ public abstract class SSTable
public static final String TEMPFILE_MARKER = "tmp";
- public static List<String> components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER, COMPONENT_INDEX, COMPONENT_DATA));
+ public static List<String> components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER, COMPONENT_INDEX, COMPONENT_DATA, COMPONENT_STATS));
protected EstimatedHistogram estimatedRowSize = new EstimatedHistogram(150);
protected EstimatedHistogram estimatedColumnCount = new EstimatedHistogram(114);
@@ -132,6 +132,7 @@ public abstract class SSTable
FileUtils.deleteWithConfirm(new File(dataFilename));
FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(dataFilename)));
FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(dataFilename)));
+ FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(dataFilename)));
FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(dataFilename)));
}
catch (IOException e)
@@ -154,11 +155,21 @@ public abstract class SSTable
return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_FILTER);
}
+ protected static String statisticsFilename(String dataFile)
+ {
+ return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_STATS);
+ }
+
public String filterFilename()
{
return desc.filenameFor(COMPONENT_FILTER);
}
+ public String statisticsFilename()
+ {
+ return desc.filenameFor(COMPONENT_STATS);
+ }
+
public String getFilename()
{
return desc.filenameFor(COMPONENT_DATA);
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java Sat Sep 4 03:31:36 2010
@@ -30,7 +30,6 @@ import java.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.StatisticsTable;
import org.apache.cassandra.io.DeletionService;
import org.apache.cassandra.io.util.FileUtils;
@@ -95,8 +94,8 @@ public class SSTableDeletingReference ex
{
FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(path)));
FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(path)));
+ FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(path)));
FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(path)));
- StatisticsTable.deleteSSTableStatistics(datafile.getAbsolutePath());
}
catch (IOException e)
{
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Sat Sep 4 03:31:36 2010
@@ -143,18 +143,14 @@ public class SSTableReader extends SSTab
private void loadStatistics(Descriptor desc) throws IOException
{
- // skip loading stats for the system table, or we will infinitely recurse
- if (desc.ksname.equals(Table.SYSTEM_TABLE))
+ if (!new File(desc.filenameFor(SSTable.COMPONENT_STATS)).exists())
return;
if (logger.isDebugEnabled())
logger.debug("Load statistics for " + desc);
- long[] rowsizes = StatisticsTable.getSSTableRowSizeStatistics(desc.filenameFor(SSTable.COMPONENT_DATA));
- long[] colcounts = StatisticsTable.getSSTableColumnCountStatistics(desc.filenameFor(SSTable.COMPONENT_DATA));
- if (rowsizes.length > 0)
- {
- estimatedRowSize = new EstimatedHistogram(rowsizes);
- estimatedColumnCount = new EstimatedHistogram(colcounts);
- }
+ DataInputStream dis = new DataInputStream(new FileInputStream(desc.filenameFor(SSTable.COMPONENT_STATS)));
+ estimatedRowSize = EstimatedHistogram.serializer.deserialize(dis);
+ estimatedColumnCount = EstimatedHistogram.serializer.deserialize(dis);
+ dis.close();
}
public static SSTableReader open(Descriptor desc) throws IOException
Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Sat Sep 4 03:31:36 2010
@@ -37,7 +37,7 @@ import org.apache.cassandra.io.util.Segm
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.BloomFilter;
import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.utils.EstimatedHistogram;
public class SSTableWriter extends SSTable
{
@@ -140,17 +140,12 @@ public class SSTableWriter extends SSTab
// main data
dataFile.close(); // calls force
+ // write sstable statistics
+ writeStatistics(desc);
+
// remove the 'tmp' marker from all components
final Descriptor newdesc = rename(desc);
- Runnable runnable = new WrappedRunnable()
- {
- protected void runMayThrow() throws IOException
- {
- StatisticsTable.persistSSTableStatistics(newdesc, estimatedRowSize, estimatedColumnCount);
- }
- };
- ColumnFamilyStore.submitPostFlush(runnable);
// finalize in-memory state for the reader
SegmentedFile ifile = iwriter.builder.complete(newdesc.filenameFor(SSTable.COMPONENT_INDEX));
@@ -161,6 +156,14 @@ public class SSTableWriter extends SSTab
return sstable;
}
+ private void writeStatistics(Descriptor desc) throws IOException
+ {
+ DataOutputStream dos = new DataOutputStream(new FileOutputStream(desc.filenameFor(SSTable.COMPONENT_STATS)));
+ EstimatedHistogram.serializer.serialize(estimatedRowSize, dos);
+ EstimatedHistogram.serializer.serialize(estimatedColumnCount, dos);
+ dos.close();
+ }
+
static Descriptor rename(Descriptor tmpdesc)
{
Descriptor newdesc = tmpdesc.asTemporary(false);
Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java Sat Sep 4 03:31:36 2010
@@ -18,8 +18,13 @@
*/
package org.apache.cassandra.utils;
+import java.io.IOException;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.Arrays;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+
+import org.apache.cassandra.io.ICompactSerializer;
public class EstimatedHistogram
{
@@ -38,6 +43,8 @@ public class EstimatedHistogram
final AtomicLongArray buckets;
+ public static EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer();
+
public EstimatedHistogram()
{
this(90);
@@ -55,6 +62,14 @@ public class EstimatedHistogram
buckets = new AtomicLongArray(bucketData);
}
+ public EstimatedHistogram(long[] offsets, long[] bucketData)
+ {
+ assert bucketData.length == offsets.length +1;
+ bucketOffsets = offsets;
+ buckets = new AtomicLongArray(bucketData);
+ numBuckets = bucketData.length;
+ }
+
private void makeOffsets(int size)
{
bucketOffsets = new long[size];
@@ -144,4 +159,32 @@ public class EstimatedHistogram
}
return median;
}
+
+ public static class EstimatedHistogramSerializer implements ICompactSerializer<EstimatedHistogram>
+ {
+ public void serialize(EstimatedHistogram eh, DataOutputStream dos) throws IOException
+ {
+ long[] offsets = eh.getBucketOffsets();
+ long[] buckets = eh.get(false);
+ dos.writeInt(buckets.length);
+ for (int i = 0; i < buckets.length; i++)
+ {
+ dos.writeLong(offsets[i == 0 ? 0 : i - 1]);
+ dos.writeLong(buckets[i]);
+ }
+ }
+
+ public EstimatedHistogram deserialize(DataInputStream dis) throws IOException
+ {
+ int size = dis.readInt();
+ long[] offsets = new long[size - 1];
+ long[] buckets = new long[size];
+
+ for (int i = 0; i < size; i++) {
+ offsets[i == 0 ? 0 : i - 1] = dis.readLong();
+ buckets[i] = dis.readLong();
+ }
+ return new EstimatedHistogram(offsets, buckets);
+ }
+ }
}
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java Sat Sep 4 03:31:36 2010
@@ -51,7 +51,6 @@ public class DefsTest extends CleanupHel
assert CFMetaData.HintsCf.cfId == 1;
assert CFMetaData.MigrationsCf.cfId == 2;
assert CFMetaData.SchemaCf.cfId == 3;
- assert CFMetaData.StatisticsCf.cfId == 4;
}
@Test
Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java Sat Sep 4 03:31:36 2010
@@ -48,11 +48,6 @@ public class RecoveryManager2Test extend
ColumnFamilyStore cfs = table1.getColumnFamilyStore("Standard1");
cfs.forceBlockingFlush();
- // forceBlockingFlush above adds persistent stats to the current commit log segment
- // it ends up in the same segment as key99 meaning that segment still has unwritten data
- // thus the commit log replays it when recover is called below
- Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(StatisticsTable.STATISTICS_CF).forceBlockingFlush();
-
// remove all SSTable/MemTables
cfs.clearUnsafe();