You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2014/10/23 17:11:48 UTC
[1/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Repository: cassandra
Updated Branches:
refs/heads/trunk 7e53db0b9 -> 0368e97ee
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 5be732e..d6fb0ee 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -30,7 +30,6 @@ import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.filter.QueryFilter;
@@ -40,14 +39,11 @@ import org.apache.cassandra.dht.BytesToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
-import org.apache.cassandra.io.sstable.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.locator.SimpleStrategy;
-import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@@ -193,7 +189,7 @@ public class CompactionsTest
// check that the shadowed column is gone
SSTableReader sstable = cfs.getSSTables().iterator().next();
Range keyRange = new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
- SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
+ ICompactionScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
OnDiskAtomIterator iter = scanner.next();
assertEquals(key, iter.getKey());
assertTrue(iter.next() instanceof RangeTombstone);
@@ -406,12 +402,7 @@ public class CompactionsTest
cf.addColumn(Util.column("a", "a", 3));
cf.deletionInfo().add(new RangeTombstone(Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis()/1000)),cfmeta.comparator);
- SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
- 0,
- 0,
- cfs.metadata,
- StorageService.getPartitioner(),
- new MetadataCollector(cfs.metadata.comparator));
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0);
writer.append(Util.dk("0"), cf);
@@ -419,12 +410,7 @@ public class CompactionsTest
writer.append(Util.dk("3"), cf);
cfs.addSSTable(writer.closeAndOpenReader());
- writer = new SSTableWriter(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
- 0,
- 0,
- cfs.metadata,
- StorageService.getPartitioner(),
- new MetadataCollector(cfs.metadata.comparator));
+ writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables())), 0, 0, 0);
writer.append(Util.dk("0"), cf);
writer.append(Util.dk("1"), cf);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index eeb98b7..309f43f 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -34,9 +34,8 @@ import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
-import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.getBuckets;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 7d79c76..a999025 100644
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -43,7 +44,6 @@ import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.Validator;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
index 56fa7a1..962957c 100644
--- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction;
import java.nio.ByteBuffer;
import java.util.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -28,7 +29,6 @@ import org.apache.cassandra.Util;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.utils.Pair;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index f3a693c..636370b 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction;
*
*/
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,8 +32,6 @@ import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.junit.Assert.assertEquals;
@@ -148,7 +147,7 @@ public class TTLExpiryTest
cfs.enableAutoCompaction(true);
assertEquals(1, cfs.getSSTables().size());
SSTableReader sstable = cfs.getSSTables().iterator().next();
- SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
+ ICompactionScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
assertTrue(scanner.hasNext());
while(scanner.hasNext())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
index 71145f7..70ab8ba 100644
--- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
@@ -21,7 +21,9 @@ import java.io.File;
import java.io.IOException;
import java.util.UUID;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
import org.junit.Test;
import org.apache.cassandra.db.Directories;
@@ -87,11 +89,11 @@ public class DescriptorTest
checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 5, Descriptor.Type.TEMP), false);
// legacy version
- checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 1, Descriptor.Type.FINAL), false);
+ checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 1, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY), false);
// legacy tmp
- checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 2, Descriptor.Type.TEMP), false);
+ checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 2, Descriptor.Type.TEMP, SSTableFormat.Type.LEGACY), false);
// legacy secondary index
- checkFromFilename(new Descriptor("ja", dir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 3, Descriptor.Type.FINAL), false);
+ checkFromFilename(new Descriptor("ja", dir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 3, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY), false);
}
private void checkFromFilename(Descriptor original, boolean skipComponent)
@@ -117,4 +119,48 @@ public class DescriptorTest
assertEquals(Component.DATA.name(), pair.right);
}
}
+
+ @Test
+ public void validateNames()
+ {
+
+ String names[] = {
+ /*"system-schema_keyspaces-ka-1-CompressionInfo.db", "system-schema_keyspaces-ka-1-Summary.db",
+ "system-schema_keyspaces-ka-1-Data.db", "system-schema_keyspaces-ka-1-TOC.txt",
+ "system-schema_keyspaces-ka-1-Digest.sha1", "system-schema_keyspaces-ka-2-CompressionInfo.db",
+ "system-schema_keyspaces-ka-1-Filter.db", "system-schema_keyspaces-ka-2-Data.db",
+ "system-schema_keyspaces-ka-1-Index.db", "system-schema_keyspaces-ka-2-Digest.sha1",
+ "system-schema_keyspaces-ka-1-Statistics.db",
+ "system-schema_keyspacest-tmp-ka-1-Data.db",*/
+ "system-schema_keyspace-ka-1-"+ SSTableFormat.Type.BIG.name+"-Data.db"
+ };
+
+ for (String name : names)
+ {
+ Descriptor d = Descriptor.fromFilename(name);
+ }
+ }
+
+ @Test
+ public void badNames()
+ {
+ String names[] = {
+ "system-schema_keyspaces-k234a-1-CompressionInfo.db", "system-schema_keyspaces-ka-aa-Summary.db",
+ "system-schema_keyspaces-XXX-ka-1-Data.db", "system-schema_keyspaces-k",
+ "system-schema_keyspace-ka-1-AAA-Data.db", "system-schema-keyspace-ka-1-AAA-Data.db"
+ };
+
+ for (String name : names)
+ {
+ try
+ {
+ Descriptor d = Descriptor.fromFilename(name);
+ Assert.fail(name);
+ } catch (Throwable e) {
+ //good
+ }
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
index 2d0c40b..2b88135 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index f0767db..2383095 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -26,6 +26,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,7 +41,6 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionInfo;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.columniterator.SSTableNamesIterator;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
@@ -92,7 +95,7 @@ public class LegacySSTableTest
protected Descriptor getDescriptor(String ver)
{
File directory = new File(LEGACY_SSTABLE_ROOT + File.separator + ver + File.separator + KSNAME);
- return new Descriptor(ver, directory, KSNAME, CFNAME, 0, Descriptor.Type.FINAL);
+ return new Descriptor(ver, directory, KSNAME, CFNAME, 0, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
}
/**
@@ -119,7 +122,7 @@ public class LegacySSTableTest
StorageService.instance.initServer();
for (File version : LEGACY_SSTABLE_ROOT.listFiles())
- if (Descriptor.Version.validate(version.getName()) && new Descriptor.Version(version.getName()).isCompatible())
+ if (Version.validate(version.getName()) && SSTableFormat.Type.LEGACY.info.getVersion(version.getName()).isCompatible())
testStreaming(version.getName());
}
@@ -144,7 +147,7 @@ public class LegacySSTableTest
for (String keystring : TEST_DATA)
{
ByteBuffer key = ByteBufferUtil.bytes(keystring);
- SSTableNamesIterator iter = new SSTableNamesIterator(sstable, Util.dk(key), FBUtilities.singleton(Util.cellname(key), type));
+ OnDiskAtomIterator iter = sstable.iterator(Util.dk(key), FBUtilities.singleton(Util.cellname(key), type));
ColumnFamily cf = iter.getColumnFamily();
// check not deleted (CASSANDRA-6527)
@@ -160,7 +163,7 @@ public class LegacySSTableTest
for (File version : LEGACY_SSTABLE_ROOT.listFiles())
{
- if (Descriptor.Version.validate(version.getName()) && new Descriptor.Version(version.getName()).isCompatible())
+ if (Version.validate(version.getName()) && SSTableFormat.Type.LEGACY.info.getVersion(version.getName()).isCompatible())
{
notSkipped = true;
testVersion(version.getName());
@@ -181,7 +184,7 @@ public class LegacySSTableTest
ByteBuffer key = ByteBufferUtil.bytes(keystring);
// confirm that the bloom filter does not reject any keys/names
DecoratedKey dk = reader.partitioner.decorateKey(key);
- SSTableNamesIterator iter = new SSTableNamesIterator(reader, dk, FBUtilities.singleton(Util.cellname(key), type));
+ OnDiskAtomIterator iter = reader.iterator(dk, FBUtilities.singleton(Util.cellname(key), type));
assert iter.next().name().toByteBuffer().equals(key);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
index 7af40fd..67d5e0a 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 71eaad8..46d5801 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -36,6 +36,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import com.google.common.collect.Sets;
import org.apache.cassandra.cache.CachingOptions;
import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.junit.Assert;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
index d044947..0d7d3c9 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.io.sstable;
import java.util.ArrayList;
import java.util.Collection;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,7 +92,7 @@ public class SSTableScannerTest
private static void assertScanMatches(SSTableReader sstable, int scanStart, int scanEnd, int expectedStart, int expectedEnd)
{
- SSTableScanner scanner = sstable.getScanner(new DataRange(boundsFor(scanStart, scanEnd), new IdentityQueryFilter()));
+ ICompactionScanner scanner = sstable.getScanner(new DataRange(boundsFor(scanStart, scanEnd), new IdentityQueryFilter()));
for (int i = expectedStart; i <= expectedEnd; i++)
assertEquals(toKey(i), new String(scanner.next().getKey().getKey().array()));
assertFalse(scanner.hasNext());
@@ -99,7 +100,7 @@ public class SSTableScannerTest
private static void assertScanEmpty(SSTableReader sstable, int scanStart, int scanEnd)
{
- SSTableScanner scanner = sstable.getScanner(new DataRange(boundsFor(scanStart, scanEnd), new IdentityQueryFilter()));
+ ICompactionScanner scanner = sstable.getScanner(new DataRange(boundsFor(scanStart, scanEnd), new IdentityQueryFilter()));
assertFalse(String.format("scan of (%03d, %03d] should be empty", scanStart, scanEnd), scanner.hasNext());
}
@@ -121,7 +122,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getSSTables().iterator().next();
// full range scan
- SSTableScanner scanner = sstable.getScanner();
+ ICompactionScanner scanner = sstable.getScanner();
for (int i = 2; i < 10; i++)
assertEquals(toKey(i), new String(scanner.next().getKey().getKey().array()));
@@ -185,7 +186,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getSSTables().iterator().next();
// full range scan
- SSTableScanner fullScanner = sstable.getScanner();
+ ICompactionScanner fullScanner = sstable.getScanner();
assertScanContainsRanges(fullScanner,
2, 9,
102, 109,
@@ -315,7 +316,7 @@ public class SSTableScannerTest
SSTableReader sstable = store.getSSTables().iterator().next();
// full range scan
- SSTableScanner fullScanner = sstable.getScanner();
+ ICompactionScanner fullScanner = sstable.getScanner();
assertScanContainsRanges(fullScanner, 205, 205);
// scan three ranges separately
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
index 57c9477..4f3d22a 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
@@ -25,6 +25,9 @@ import java.util.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.compaction.ICompactionScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -81,8 +84,8 @@ public class SSTableUtils
public static void assertContentEquals(SSTableReader lhs, SSTableReader rhs)
{
- SSTableScanner slhs = lhs.getScanner();
- SSTableScanner srhs = rhs.getScanner();
+ ICompactionScanner slhs = lhs.getScanner();
+ ICompactionScanner srhs = rhs.getScanner();
while (slhs.hasNext())
{
OnDiskAtomIterator ilhs = slhs.next();
@@ -211,7 +214,7 @@ public class SSTableUtils
public SSTableReader write(int expectedSize, Appender appender) throws IOException
{
File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA));
- SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), expectedSize);
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
while (appender.append(writer)) { /* pass */ }
SSTableReader reader = writer.closeAndOpenReader();
// mark all components for removal
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 7751a51..9cc2d23 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -75,7 +75,7 @@ public class MetadataSerializerTest
serializer.serialize(originalMetadata, out);
}
- Descriptor desc = new Descriptor(Descriptor.Version.CURRENT, statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
+ Descriptor desc = new Descriptor( statsFile.getParentFile(), "", "", 0, Descriptor.Type.FINAL);
try (RandomAccessReader in = RandomAccessReader.open(statsFile))
{
Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/repair/ValidatorTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
index ea46204..74d1ad3 100644
--- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java
+++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java
@@ -22,6 +22,7 @@ import java.net.InetAddress;
import java.security.MessageDigest;
import java.util.UUID;
+import org.apache.cassandra.io.util.SequentialWriter;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -143,7 +144,7 @@ public class ValidatorTest
super(key);
}
- public RowIndexEntry write(long currentPosition, DataOutputPlus out) throws IOException
+ public RowIndexEntry write(long currentPosition, SequentialWriter out) throws IOException
{
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
index 5e4e1c5..d3685d9 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java
@@ -25,6 +25,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,7 +37,6 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.FBUtilities;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
index b193d9d..6bc6827 100644
--- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
+++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -49,7 +50,6 @@ import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableUtils;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.service.StorageService;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
index f93e168..8e3f5ca 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
@@ -46,8 +46,8 @@ import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -97,7 +97,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
// Add rowA
cfamily.addColumn(Util.cellname("colA"), ByteBufferUtil.bytes("valA"), System.currentTimeMillis());
@@ -134,7 +134,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
int nowInSec = (int)(System.currentTimeMillis() / 1000) + 42; //live for 42 seconds
// Add rowA
@@ -191,7 +191,7 @@ public class SSTableExportTest
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
File tempSS = tempSSTableFile(KEYSPACE1, "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
// Add rowA
cfamily.addColumn(Util.cellname("name"), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
@@ -231,7 +231,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "Counter1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Counter1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
// Add rowA
cfamily.addColumn(BufferCounterCell.createLocal(Util.cellname("colA"), 42, System.currentTimeMillis(), Long.MIN_VALUE));
@@ -263,7 +263,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "ValuesWithQuotes");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "ValuesWithQuotes");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
// Add rowA
cfamily.addColumn(new BufferCell(Util.cellname("data"), UTF8Type.instance.fromString("{\"foo\":\"bar\"}")));
@@ -295,7 +295,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "Standard1");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "Standard1");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
// Add rowA
cfamily.addColumn(Util.cellname("colName"), ByteBufferUtil.bytes("val"), System.currentTimeMillis());
@@ -357,7 +357,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "UUIDKeys");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "UUIDKeys");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE);
// Add a row
cfamily.addColumn(column(CFMetaData.DEFAULT_KEY_ALIAS, "not a uuid", 1L));
@@ -387,7 +387,7 @@ public class SSTableExportTest
{
File tempSS = tempSSTableFile(KEYSPACE1, "AsciiKeys");
ColumnFamily cfamily = ArrayBackedSortedColumns.factory.create(KEYSPACE1, "AsciiKeys");
- SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
+ SSTableWriter writer = SSTableWriter.create(tempSS.getPath(), 2, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
// Add a row
cfamily.addColumn(column("column", "value", 1L));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
index 01becfe..e2cecb2 100644
--- a/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
+++ b/test/unit/org/apache/cassandra/tools/SSTableImportTest.java
@@ -31,6 +31,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
@@ -59,7 +60,6 @@ import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.CounterColumnType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.thrift.TException;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/tools/cqlstress-example.yaml
----------------------------------------------------------------------
diff --git a/tools/cqlstress-example.yaml b/tools/cqlstress-example.yaml
index 4dd5e4a..3c60c32 100644
--- a/tools/cqlstress-example.yaml
+++ b/tools/cqlstress-example.yaml
@@ -42,9 +42,10 @@ table_definition: |
uid timeuuid,
value blob,
PRIMARY KEY((name,choice), date, address, dbl, lval, ival, uid)
- ) WITH COMPACT STORAGE
- AND compaction = { 'class':'LeveledCompactionStrategy' }
- AND comment='A table of many types to test wide rows'
+ )
+ WITH compaction = { 'class':'LeveledCompactionStrategy' }
+# AND compression = { 'sstable_compression' : '' }
+# AND comment='A table of many types to test wide rows'
#
# Optional meta information on the generated columns in the above table
@@ -67,7 +68,7 @@ table_definition: |
columnspec:
- name: name
size: uniform(1..10)
- population: uniform(1..1M) # the range of unique values to select for the field (default is 100Billion)
+ population: uniform(1..10) # the range of unique values to select for the field (default is 100Billion)
- name: date
cluster: uniform(20..40)
- name: lval
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/tools/cqlstress-insanity-example.yaml
----------------------------------------------------------------------
diff --git a/tools/cqlstress-insanity-example.yaml b/tools/cqlstress-insanity-example.yaml
index ea4f97f..a286625 100644
--- a/tools/cqlstress-insanity-example.yaml
+++ b/tools/cqlstress-insanity-example.yaml
@@ -41,8 +41,6 @@ table_definition: |
fval float,
ival int,
uid timeuuid,
- dates list<timestamp>,
- inets set<inet>,
value blob,
PRIMARY KEY((name, choice), date)
) WITH compaction = { 'class':'LeveledCompactionStrategy' }
[5/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableScanner.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/SSTableScanner.java
deleted file mode 100644
index 62ac175..0000000
--- a/src/java/org/apache/cassandra/io/sstable/SSTableScanner.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-
-import com.google.common.collect.AbstractIterator;
-import com.google.common.util.concurrent.RateLimiter;
-
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.RowPosition;
-import org.apache.cassandra.db.columniterator.IColumnIteratorFactory;
-import org.apache.cassandra.db.columniterator.LazyColumnIterator;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
-import org.apache.cassandra.db.compaction.ICompactionScanner;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.Bounds;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public class SSTableScanner implements ICompactionScanner
-{
- protected final RandomAccessReader dfile;
- protected final RandomAccessReader ifile;
- public final SSTableReader sstable;
-
- private final Iterator<AbstractBounds<RowPosition>> rangeIterator;
- private AbstractBounds<RowPosition> currentRange;
-
- private final DataRange dataRange;
-
- protected Iterator<OnDiskAtomIterator> iterator;
-
- /**
- * @param sstable SSTable to scan; must not be null
- * @param dataRange a single range to scan; must not be null
- * @param limiter background i/o RateLimiter; may be null
- */
- SSTableScanner(SSTableReader sstable, DataRange dataRange, RateLimiter limiter)
- {
- assert sstable != null;
-
- this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter);
- this.ifile = sstable.openIndexReader();
- this.sstable = sstable;
- this.dataRange = dataRange;
-
- List<AbstractBounds<RowPosition>> boundsList = new ArrayList<>(2);
- if (dataRange.isWrapAround() && !dataRange.stopKey().isMinimum(sstable.partitioner))
- {
- // split the wrapping range into two parts: 1) the part that starts at the beginning of the sstable, and
- // 2) the part that comes before the wrap-around
- boundsList.add(new Bounds<>(sstable.partitioner.getMinimumToken().minKeyBound(), dataRange.stopKey(), sstable.partitioner));
- boundsList.add(new Bounds<>(dataRange.startKey(), sstable.partitioner.getMinimumToken().maxKeyBound(), sstable.partitioner));
- }
- else
- {
- boundsList.add(new Bounds<>(dataRange.startKey(), dataRange.stopKey(), sstable.partitioner));
- }
- this.rangeIterator = boundsList.iterator();
- }
-
- /**
- * @param sstable SSTable to scan; must not be null
- * @param tokenRanges A set of token ranges to scan
- * @param limiter background i/o RateLimiter; may be null
- */
- SSTableScanner(SSTableReader sstable, Collection<Range<Token>> tokenRanges, RateLimiter limiter)
- {
- assert sstable != null;
-
- this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter);
- this.ifile = sstable.openIndexReader();
- this.sstable = sstable;
- this.dataRange = null;
-
- List<Range<Token>> normalized = Range.normalize(tokenRanges);
- List<AbstractBounds<RowPosition>> boundsList = new ArrayList<>(normalized.size());
- for (Range<Token> range : normalized)
- boundsList.add(new Range<RowPosition>(range.left.maxKeyBound(sstable.partitioner),
- range.right.maxKeyBound(sstable.partitioner),
- sstable.partitioner));
-
- this.rangeIterator = boundsList.iterator();
- }
-
- private void seekToCurrentRangeStart()
- {
- if (currentRange.left.isMinimum(sstable.partitioner))
- return;
-
- long indexPosition = sstable.getIndexScanPosition(currentRange.left);
- // -1 means the key is before everything in the sstable. So just start from the beginning.
- if (indexPosition == -1)
- {
- // Note: this method shouldn't assume we're at the start of the sstable already (see #6638) and
- // the seeks are no-op anyway if we are.
- ifile.seek(0);
- dfile.seek(0);
- return;
- }
-
- ifile.seek(indexPosition);
- try
- {
-
- while (!ifile.isEOF())
- {
- indexPosition = ifile.getFilePointer();
- DecoratedKey indexDecoratedKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- int comparison = indexDecoratedKey.compareTo(currentRange.left);
- // because our range start may be inclusive or exclusive, we need to also contains()
- // instead of just checking (comparison >= 0)
- if (comparison > 0 || currentRange.contains(indexDecoratedKey))
- {
- // Found, just read the dataPosition and seek into index and data files
- long dataPosition = ifile.readLong();
- ifile.seek(indexPosition);
- dfile.seek(dataPosition);
- break;
- }
- else
- {
- RowIndexEntry.Serializer.skip(ifile);
- }
- }
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- }
-
- public void close() throws IOException
- {
- FileUtils.close(dfile, ifile);
- }
-
- public long getLengthInBytes()
- {
- return dfile.length();
- }
-
- public long getCurrentPosition()
- {
- return dfile.getFilePointer();
- }
-
- public String getBackingFiles()
- {
- return sstable.toString();
- }
-
- public boolean hasNext()
- {
- if (iterator == null)
- iterator = createIterator();
- return iterator.hasNext();
- }
-
- public OnDiskAtomIterator next()
- {
- if (iterator == null)
- iterator = createIterator();
- return iterator.next();
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
-
- private Iterator<OnDiskAtomIterator> createIterator()
- {
- return new KeyScanningIterator();
- }
-
- protected class KeyScanningIterator extends AbstractIterator<OnDiskAtomIterator>
- {
- private DecoratedKey nextKey;
- private RowIndexEntry nextEntry;
- private DecoratedKey currentKey;
- private RowIndexEntry currentEntry;
-
- protected OnDiskAtomIterator computeNext()
- {
- try
- {
- if (nextEntry == null)
- {
- do
- {
- // we're starting the first range or we just passed the end of the previous range
- if (!rangeIterator.hasNext())
- return endOfData();
-
- currentRange = rangeIterator.next();
- seekToCurrentRangeStart();
-
- if (ifile.isEOF())
- return endOfData();
-
- currentKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- currentEntry = sstable.metadata.comparator.rowIndexEntrySerializer().deserialize(ifile, sstable.descriptor.version);
- } while (!currentRange.contains(currentKey));
- }
- else
- {
- // we're in the middle of a range
- currentKey = nextKey;
- currentEntry = nextEntry;
- }
-
- long readEnd;
- if (ifile.isEOF())
- {
- nextEntry = null;
- nextKey = null;
- readEnd = dfile.length();
- }
- else
- {
- // we need the position of the start of the next key, regardless of whether it falls in the current range
- nextKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
- nextEntry = sstable.metadata.comparator.rowIndexEntrySerializer().deserialize(ifile, sstable.descriptor.version);
- readEnd = nextEntry.position;
-
- if (!currentRange.contains(nextKey))
- {
- nextKey = null;
- nextEntry = null;
- }
- }
-
- if (dataRange == null || dataRange.selectsFullRowFor(currentKey.getKey()))
- {
- dfile.seek(currentEntry.position);
- ByteBufferUtil.readWithShortLength(dfile); // key
- long dataSize = readEnd - dfile.getFilePointer();
- return new SSTableIdentityIterator(sstable, dfile, currentKey, dataSize);
- }
-
- return new LazyColumnIterator(currentKey, new IColumnIteratorFactory()
- {
- public OnDiskAtomIterator create()
- {
- return dataRange.columnFilter(currentKey.getKey()).getSSTableColumnIterator(sstable, dfile, currentKey, currentEntry);
- }
- });
-
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- }
- }
-
- @Override
- public String toString()
- {
- return getClass().getSimpleName() + "(" +
- "dfile=" + dfile +
- " ifile=" + ifile +
- " sstable=" + sstable +
- ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index 3cfdc7b..bc9f2ca 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -27,6 +27,7 @@ import java.util.concurrent.SynchronousQueue;
import com.google.common.base.Throwables;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ArrayBackedSortedColumns;
import org.apache.cassandra.db.Cell;
import org.apache.cassandra.db.ColumnFamily;
@@ -35,6 +36,8 @@ import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.compress.CompressionParameters;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -96,8 +99,8 @@ public class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
public SSTableSimpleUnsortedWriter(File directory, CFMetaData metadata, IPartitioner partitioner, long bufferSizeInMB)
{
super(directory, metadata, partitioner);
- this.bufferSize = bufferSizeInMB * 1024L * 1024L;
- this.diskWriter.start();
+ bufferSize = bufferSizeInMB * 1024L * 1024L;
+ diskWriter.start();
}
protected void writeRow(DecoratedKey key, ColumnFamily columnFamily) throws IOException
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
index 87c8e33..3417d68 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
@@ -24,6 +24,7 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.FSError;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
/**
* A SSTable writer that assumes rows are in (partitioner) sorted order.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
deleted file mode 100644
index 4da967e..0000000
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ /dev/null
@@ -1,641 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.Closeable;
-import java.io.DataInput;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.collect.Sets;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.ArrayBackedSortedColumns;
-import org.apache.cassandra.db.ColumnFamily;
-import org.apache.cassandra.db.ColumnIndex;
-import org.apache.cassandra.db.ColumnSerializer;
-import org.apache.cassandra.db.CounterCell;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.DeletionTime;
-import org.apache.cassandra.db.OnDiskAtom;
-import org.apache.cassandra.db.RangeTombstone;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.compress.CompressedSequentialWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
-import org.apache.cassandra.io.util.FileMark;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.SegmentedFile;
-import org.apache.cassandra.io.util.SequentialWriter;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FilterFactory;
-import org.apache.cassandra.utils.IFilter;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.StreamingHistogram;
-
-public class SSTableWriter extends SSTable
-{
- private static final Logger logger = LoggerFactory.getLogger(SSTableWriter.class);
-
- // not very random, but the only value that can't be mistaken for a legal column-name length
- public static final int END_OF_ROW = 0x0000;
-
- private IndexWriter iwriter;
- private SegmentedFile.Builder dbuilder;
- private final SequentialWriter dataFile;
- private DecoratedKey lastWrittenKey;
- private FileMark dataMark;
- private final MetadataCollector sstableMetadataCollector;
- private final long repairedAt;
-
- public SSTableWriter(String filename, long keyCount, long repairedAt, int sstableLevel)
- {
- this(filename,
- keyCount,
- repairedAt,
- Schema.instance.getCFMetaData(Descriptor.fromFilename(filename)),
- StorageService.getPartitioner(),
- new MetadataCollector(Schema.instance.getCFMetaData(Descriptor.fromFilename(filename)).comparator).sstableLevel(sstableLevel));
- }
-
- public SSTableWriter(String filename, long keyCount)
- {
- this(filename, keyCount, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
- }
-
- private static Set<Component> components(CFMetaData metadata)
- {
- Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA,
- Component.PRIMARY_INDEX,
- Component.STATS,
- Component.SUMMARY,
- Component.TOC,
- Component.DIGEST));
-
- if (metadata.getBloomFilterFpChance() < 1.0)
- components.add(Component.FILTER);
-
- if (metadata.compressionParameters().sstableCompressor != null)
- {
- components.add(Component.COMPRESSION_INFO);
- }
- else
- {
- // it would feel safer to actually add this component later in maybeWriteDigest(),
- // but the components are unmodifiable after construction
- components.add(Component.CRC);
- }
- return components;
- }
-
- public SSTableWriter(String filename,
- long keyCount,
- long repairedAt,
- CFMetaData metadata,
- IPartitioner<?> partitioner,
- MetadataCollector sstableMetadataCollector)
- {
- super(Descriptor.fromFilename(filename),
- components(metadata),
- metadata,
- partitioner);
- this.repairedAt = repairedAt;
- iwriter = new IndexWriter(keyCount);
-
- if (compression)
- {
- dataFile = SequentialWriter.open(getFilename(),
- descriptor.filenameFor(Component.COMPRESSION_INFO),
- metadata.compressionParameters(),
- sstableMetadataCollector);
- dbuilder = SegmentedFile.getCompressedBuilder((CompressedSequentialWriter) dataFile);
- }
- else
- {
- dataFile = SequentialWriter.open(new File(getFilename()), new File(descriptor.filenameFor(Component.CRC)));
- dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
- }
-
- this.sstableMetadataCollector = sstableMetadataCollector;
- }
-
- public void mark()
- {
- dataMark = dataFile.mark();
- iwriter.mark();
- }
-
- public void resetAndTruncate()
- {
- dataFile.resetAndTruncate(dataMark);
- iwriter.resetAndTruncate();
- }
-
- /**
- * Perform sanity checks on @param decoratedKey and @return the position in the data file before any data is written
- */
- private long beforeAppend(DecoratedKey decoratedKey)
- {
- assert decoratedKey != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values
- if (lastWrittenKey != null && lastWrittenKey.compareTo(decoratedKey) >= 0)
- throw new RuntimeException("Last written key " + lastWrittenKey + " >= current key " + decoratedKey + " writing into " + getFilename());
- return (lastWrittenKey == null) ? 0 : dataFile.getFilePointer();
- }
-
- private void afterAppend(DecoratedKey decoratedKey, long dataPosition, RowIndexEntry index)
- {
- sstableMetadataCollector.addKey(decoratedKey.getKey());
- lastWrittenKey = decoratedKey;
- last = lastWrittenKey;
- if (first == null)
- first = lastWrittenKey;
-
- if (logger.isTraceEnabled())
- logger.trace("wrote {} at {}", decoratedKey, dataPosition);
- iwriter.append(decoratedKey, index);
- dbuilder.addPotentialBoundary(dataPosition);
- }
-
- /**
- * @param row
- * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row
- */
- public RowIndexEntry append(AbstractCompactedRow row)
- {
- long currentPosition = beforeAppend(row.key);
- RowIndexEntry entry;
- try
- {
- entry = row.write(currentPosition, dataFile.stream);
- if (entry == null)
- return null;
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, dataFile.getPath());
- }
- sstableMetadataCollector.update(dataFile.getFilePointer() - currentPosition, row.columnStats());
- afterAppend(row.key, currentPosition, entry);
- return entry;
- }
-
- public void append(DecoratedKey decoratedKey, ColumnFamily cf)
- {
- long startPosition = beforeAppend(decoratedKey);
- try
- {
- RowIndexEntry entry = rawAppend(cf, startPosition, decoratedKey, dataFile.stream);
- afterAppend(decoratedKey, startPosition, entry);
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, dataFile.getPath());
- }
- sstableMetadataCollector.update(dataFile.getFilePointer() - startPosition, cf.getColumnStats());
- }
-
- public static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException
- {
- assert cf.hasColumns() || cf.isMarkedForDelete();
-
- ColumnIndex.Builder builder = new ColumnIndex.Builder(cf, key.getKey(), out);
- ColumnIndex index = builder.build(cf);
-
- out.writeShort(END_OF_ROW);
- return RowIndexEntry.create(startPosition, cf.deletionInfo().getTopLevelDeletion(), index);
- }
-
- /**
- * @throws IOException if a read from the DataInput fails
- * @throws FSWriteError if a write to the dataFile fails
- */
- public long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Descriptor.Version version) throws IOException
- {
- long currentPosition = beforeAppend(key);
-
- ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE);
- ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE);
- ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE);
- List<ByteBuffer> minColumnNames = Collections.emptyList();
- List<ByteBuffer> maxColumnNames = Collections.emptyList();
- StreamingHistogram tombstones = new StreamingHistogram(TOMBSTONE_HISTOGRAM_BIN_SIZE);
- boolean hasLegacyCounterShards = false;
-
- ColumnFamily cf = ArrayBackedSortedColumns.factory.create(metadata);
- cf.delete(DeletionTime.serializer.deserialize(in));
-
- ColumnIndex.Builder columnIndexer = new ColumnIndex.Builder(cf, key.getKey(), dataFile.stream);
-
- if (cf.deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE)
- {
- tombstones.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
- maxDeletionTimeTracker.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
- minTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
- maxTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
- }
-
- Iterator<RangeTombstone> rangeTombstoneIterator = cf.deletionInfo().rangeIterator();
- while (rangeTombstoneIterator.hasNext())
- {
- RangeTombstone rangeTombstone = rangeTombstoneIterator.next();
- tombstones.update(rangeTombstone.getLocalDeletionTime());
- minTimestampTracker.update(rangeTombstone.timestamp());
- maxTimestampTracker.update(rangeTombstone.timestamp());
- maxDeletionTimeTracker.update(rangeTombstone.getLocalDeletionTime());
- minColumnNames = ColumnNameHelper.minComponents(minColumnNames, rangeTombstone.min, metadata.comparator);
- maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, rangeTombstone.max, metadata.comparator);
- }
-
- Iterator<OnDiskAtom> iter = metadata.getOnDiskIterator(in, ColumnSerializer.Flag.PRESERVE_SIZE, Integer.MIN_VALUE, version);
- try
- {
- while (iter.hasNext())
- {
- OnDiskAtom atom = iter.next();
- if (atom == null)
- break;
-
- if (atom instanceof CounterCell)
- {
- atom = ((CounterCell) atom).markLocalToBeCleared();
- hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) atom).hasLegacyShards();
- }
-
- int deletionTime = atom.getLocalDeletionTime();
- if (deletionTime < Integer.MAX_VALUE)
- tombstones.update(deletionTime);
- minTimestampTracker.update(atom.timestamp());
- maxTimestampTracker.update(atom.timestamp());
- minColumnNames = ColumnNameHelper.minComponents(minColumnNames, atom.name(), metadata.comparator);
- maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, atom.name(), metadata.comparator);
- maxDeletionTimeTracker.update(atom.getLocalDeletionTime());
-
- columnIndexer.add(atom); // This write the atom on disk too
- }
-
- columnIndexer.maybeWriteEmptyRowHeader();
- dataFile.stream.writeShort(END_OF_ROW);
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, dataFile.getPath());
- }
-
- sstableMetadataCollector.updateMinTimestamp(minTimestampTracker.get())
- .updateMaxTimestamp(maxTimestampTracker.get())
- .updateMaxLocalDeletionTime(maxDeletionTimeTracker.get())
- .addRowSize(dataFile.getFilePointer() - currentPosition)
- .addColumnCount(columnIndexer.writtenAtomCount())
- .mergeTombstoneHistogram(tombstones)
- .updateMinColumnNames(minColumnNames)
- .updateMaxColumnNames(maxColumnNames)
- .updateHasLegacyCounterShards(hasLegacyCounterShards);
- afterAppend(key, currentPosition, RowIndexEntry.create(currentPosition, cf.deletionInfo().getTopLevelDeletion(), columnIndexer.build()));
- return currentPosition;
- }
-
- /**
- * After failure, attempt to close the index writer and data file before deleting all temp components for the sstable
- */
- public void abort()
- {
- abort(true);
- }
- public void abort(boolean closeBf)
- {
- assert descriptor.type.isTemporary;
- if (iwriter == null && dataFile == null)
- return;
- if (iwriter != null)
- {
- FileUtils.closeQuietly(iwriter.indexFile);
- if (closeBf)
- {
- iwriter.bf.close();
- }
- }
- if (dataFile!= null)
- FileUtils.closeQuietly(dataFile);
-
- Set<Component> components = SSTable.componentsFor(descriptor);
- try
- {
- if (!components.isEmpty())
- SSTable.delete(descriptor, components);
- }
- catch (FSWriteError e)
- {
- logger.error(String.format("Failed deleting temp components for %s", descriptor), e);
- throw e;
- }
- }
-
- // we use this method to ensure any managed data we may have retained references to during the write are no
- // longer referenced, so that we do not need to enclose the expensive call to closeAndOpenReader() in a transaction
- public void isolateReferences()
- {
- // currently we only maintain references to first/last/lastWrittenKey from the data provided; all other
- // data retention is done through copying
- first = getMinimalKey(first);
- last = lastWrittenKey = getMinimalKey(last);
- }
-
- public SSTableReader openEarly(long maxDataAge)
- {
- StatsMetadata sstableMetadata = (StatsMetadata) sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName(),
- metadata.getBloomFilterFpChance(),
- repairedAt).get(MetadataType.STATS);
-
- // find the max (exclusive) readable key
- DecoratedKey exclusiveUpperBoundOfReadableIndex = iwriter.getMaxReadableKey(0);
- if (exclusiveUpperBoundOfReadableIndex == null)
- return null;
-
- // create temp links if they don't already exist
- Descriptor link = descriptor.asType(Descriptor.Type.TEMPLINK);
- if (!new File(link.filenameFor(Component.PRIMARY_INDEX)).exists())
- {
- FileUtils.createHardLink(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), new File(link.filenameFor(Component.PRIMARY_INDEX)));
- FileUtils.createHardLink(new File(descriptor.filenameFor(Component.DATA)), new File(link.filenameFor(Component.DATA)));
- }
-
- // open the reader early, giving it a FINAL descriptor type so that it is indistinguishable for other consumers
- SegmentedFile ifile = iwriter.builder.openEarly(link.filenameFor(Component.PRIMARY_INDEX));
- SegmentedFile dfile = dbuilder.openEarly(link.filenameFor(Component.DATA));
- SSTableReader sstable = SSTableReader.internalOpen(descriptor.asType(Descriptor.Type.FINAL),
- components, metadata,
- partitioner, ifile,
- dfile, iwriter.summary.build(partitioner, exclusiveUpperBoundOfReadableIndex),
- iwriter.bf, maxDataAge, sstableMetadata, SSTableReader.OpenReason.EARLY);
-
- // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed)
- sstable.first = getMinimalKey(first);
- sstable.last = getMinimalKey(exclusiveUpperBoundOfReadableIndex);
- DecoratedKey inclusiveUpperBoundOfReadableData = iwriter.getMaxReadableKey(1);
- if (inclusiveUpperBoundOfReadableData == null)
- return null;
- int offset = 2;
- while (true)
- {
- RowIndexEntry indexEntry = sstable.getPosition(inclusiveUpperBoundOfReadableData, SSTableReader.Operator.GT);
- if (indexEntry != null && indexEntry.position <= dataFile.getLastFlushOffset())
- break;
- inclusiveUpperBoundOfReadableData = iwriter.getMaxReadableKey(offset++);
- if (inclusiveUpperBoundOfReadableData == null)
- return null;
- }
- sstable.last = getMinimalKey(inclusiveUpperBoundOfReadableData);
- return sstable;
- }
-
- public SSTableReader closeAndOpenReader()
- {
- return closeAndOpenReader(System.currentTimeMillis());
- }
-
- public SSTableReader closeAndOpenReader(long maxDataAge)
- {
- return closeAndOpenReader(maxDataAge, this.repairedAt);
- }
-
- public SSTableReader closeAndOpenReader(long maxDataAge, long repairedAt)
- {
- Pair<Descriptor, StatsMetadata> p = close(repairedAt);
- Descriptor newdesc = p.left;
- StatsMetadata sstableMetadata = p.right;
-
- // finalize in-memory state for the reader
- SegmentedFile ifile = iwriter.builder.complete(newdesc.filenameFor(Component.PRIMARY_INDEX));
- SegmentedFile dfile = dbuilder.complete(newdesc.filenameFor(Component.DATA));
- SSTableReader sstable = SSTableReader.internalOpen(newdesc,
- components,
- metadata,
- partitioner,
- ifile,
- dfile,
- iwriter.summary.build(partitioner),
- iwriter.bf,
- maxDataAge,
- sstableMetadata,
- SSTableReader.OpenReason.NORMAL);
- sstable.first = getMinimalKey(first);
- sstable.last = getMinimalKey(last);
- // try to save the summaries to disk
- sstable.saveSummary(iwriter.builder, dbuilder);
- iwriter = null;
- dbuilder = null;
- return sstable;
- }
-
- // Close the writer and return the descriptor to the new sstable and it's metadata
- public Pair<Descriptor, StatsMetadata> close()
- {
- return close(this.repairedAt);
- }
-
- private Pair<Descriptor, StatsMetadata> close(long repairedAt)
- {
-
- // index and filter
- iwriter.close();
- // main data, close will truncate if necessary
- dataFile.close();
- dataFile.writeFullChecksum(descriptor);
- // write sstable statistics
- Map<MetadataType, MetadataComponent> metadataComponents = sstableMetadataCollector.finalizeMetadata(
- partitioner.getClass().getCanonicalName(),
- metadata.getBloomFilterFpChance(),
- repairedAt);
- writeMetadata(descriptor, metadataComponents);
-
- // save the table of components
- SSTable.appendTOC(descriptor, components);
-
- // remove the 'tmp' marker from all components
- return Pair.create(rename(descriptor, components), (StatsMetadata) metadataComponents.get(MetadataType.STATS));
-
- }
-
-
- private static void writeMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> components)
- {
- SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(Component.STATS)));
- try
- {
- desc.getMetadataSerializer().serialize(components, out.stream);
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, out.getPath());
- }
- finally
- {
- out.close();
- }
- }
-
- static Descriptor rename(Descriptor tmpdesc, Set<Component> components)
- {
- Descriptor newdesc = tmpdesc.asType(Descriptor.Type.FINAL);
- rename(tmpdesc, newdesc, components);
- return newdesc;
- }
-
- public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
- {
- for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY)))
- {
- FileUtils.renameWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component));
- }
-
- // do -Data last because -Data present should mean the sstable was completely renamed before crash
- FileUtils.renameWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA));
-
- // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
- FileUtils.renameWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
- }
-
- public long getFilePointer()
- {
- return dataFile.getFilePointer();
- }
-
- public long getOnDiskFilePointer()
- {
- return dataFile.getOnDiskFilePointer();
- }
-
- /**
- * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
- */
- class IndexWriter implements Closeable
- {
- private final SequentialWriter indexFile;
- public final SegmentedFile.Builder builder;
- public final IndexSummaryBuilder summary;
- public final IFilter bf;
- private FileMark mark;
-
- IndexWriter(long keyCount)
- {
- indexFile = SequentialWriter.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
- builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
- summary = new IndexSummaryBuilder(keyCount, metadata.getMinIndexInterval(), Downsampling.BASE_SAMPLING_LEVEL);
- bf = FilterFactory.getFilter(keyCount, metadata.getBloomFilterFpChance(), true);
- }
-
- // finds the last (-offset) decorated key that can be guaranteed to occur fully in the flushed portion of the index file
- DecoratedKey getMaxReadableKey(int offset)
- {
- long maxIndexLength = indexFile.getLastFlushOffset();
- return summary.getMaxReadableKey(maxIndexLength, offset);
- }
-
- public void append(DecoratedKey key, RowIndexEntry indexEntry)
- {
- bf.add(key.getKey());
- long indexPosition = indexFile.getFilePointer();
- try
- {
- ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile.stream);
- metadata.comparator.rowIndexEntrySerializer().serialize(indexEntry, indexFile.stream);
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, indexFile.getPath());
- }
-
- if (logger.isTraceEnabled())
- logger.trace("wrote index entry: {} at {}", indexEntry, indexPosition);
-
- summary.maybeAddEntry(key, indexPosition);
- builder.addPotentialBoundary(indexPosition);
- }
-
- /**
- * Closes the index and bloomfilter, making the public state of this writer valid for consumption.
- */
- public void close()
- {
- if (components.contains(Component.FILTER))
- {
- String path = descriptor.filenameFor(Component.FILTER);
- try
- {
- // bloom filter
- FileOutputStream fos = new FileOutputStream(path);
- DataOutputStreamAndChannel stream = new DataOutputStreamAndChannel(fos);
- FilterFactory.serialize(bf, stream);
- stream.flush();
- fos.getFD().sync();
- stream.close();
- }
- catch (IOException e)
- {
- throw new FSWriteError(e, path);
- }
- }
-
- // index
- long position = indexFile.getFilePointer();
- indexFile.close(); // calls force
- FileUtils.truncate(indexFile.getPath(), position);
- }
-
- public void mark()
- {
- mark = indexFile.mark();
- }
-
- public void resetAndTruncate()
- {
- // we can't un-set the bloom filter addition, but extra keys in there are harmless.
- // we can't reset dbuilder either, but that is the last thing called in afterappend so
- // we assume that if that worked then we won't be trying to reset.
- indexFile.resetAndTruncate(mark);
- }
-
- @Override
- public String toString()
- {
- return "IndexWriter(" + descriptor + ")";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
new file mode 100644
index 0000000..ca003b6
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.ColumnSerializer;
+import org.apache.cassandra.db.OnDiskAtom;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.FileDataInput;
+
+import java.util.Iterator;
+
+/**
+ * Provides the accessors to data on disk.
+ */
+public interface SSTableFormat
+{
+ static boolean enableSSTableDevelopmentTestMode = Boolean.valueOf(System.getProperty("cassandra.test.sstableformatdevelopment","false"));
+
+
+ Version getLatestVersion();
+ Version getVersion(String version);
+
+ SSTableWriter.Factory getWriterFactory();
+ SSTableReader.Factory getReaderFactory();
+
+ Iterator<OnDiskAtom> getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, CFMetaData cfm, Version version);
+
+ AbstractCompactedRow getCompactedRowWriter(CompactionController controller, ImmutableList<OnDiskAtomIterator> onDiskAtomIterators);
+
+ RowIndexEntry.IndexSerializer<?> getIndexSerializer(CFMetaData cfm);
+
+ public static enum Type
+ {
+ //Used internally to refer to files with no
+ //format flag in the filename
+ LEGACY("big", BigFormat.instance),
+
+ //The original sstable format
+ BIG("big", BigFormat.instance);
+
+ public final SSTableFormat info;
+ public final String name;
+ private Type(String name, SSTableFormat info)
+ {
+ //Since format comes right after generation
+ //we disallow formats with numeric names
+ assert !CharMatcher.DIGIT.matchesAllOf(name);
+
+ this.name = name;
+ this.info = info;
+ }
+
+ public static Type validate(String name)
+ {
+ for (Type valid : Type.values())
+ {
+ //This is used internally for old sstables
+ if (valid == LEGACY)
+ continue;
+
+ if (valid.name.equalsIgnoreCase(name))
+ return valid;
+ }
+
+ throw new IllegalArgumentException("No Type constant " + name);
+ }
+ }
+}
[3/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
new file mode 100644
index 0000000..eb43968
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.AbstractCell;
+import org.apache.cassandra.db.ColumnSerializer;
+import org.apache.cassandra.db.OnDiskAtom;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.db.compaction.CompactionController;
+import org.apache.cassandra.db.compaction.LazilyCompactedRow;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.IndexHelper;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.FileDataInput;
+
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * Legacy bigtable format
+ */
+public class BigFormat implements SSTableFormat
+{
+ public static final BigFormat instance = new BigFormat();
+ public static final BigVersion latestVersion = new BigVersion(BigVersion.current_version);
+ private static final SSTableReader.Factory readerFactory = new ReaderFactory();
+ private static final SSTableWriter.Factory writerFactory = new WriterFactory();
+
+ private BigFormat()
+ {
+
+ }
+
+ @Override
+ public Version getLatestVersion()
+ {
+ return latestVersion;
+ }
+
+ @Override
+ public Version getVersion(String version)
+ {
+ return new BigVersion(version);
+ }
+
+ @Override
+ public SSTableWriter.Factory getWriterFactory()
+ {
+ return writerFactory;
+ }
+
+ @Override
+ public SSTableReader.Factory getReaderFactory()
+ {
+ return readerFactory;
+ }
+
+ @Override
+ public Iterator<OnDiskAtom> getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, CFMetaData cfm, Version version)
+ {
+ return AbstractCell.onDiskIterator(in, flag, expireBefore, version, cfm.comparator);
+ }
+
+ @Override
+ public AbstractCompactedRow getCompactedRowWriter(CompactionController controller, ImmutableList<OnDiskAtomIterator> onDiskAtomIterators)
+ {
+ return new LazilyCompactedRow(controller, onDiskAtomIterators);
+ }
+
+ @Override
+ public RowIndexEntry.IndexSerializer getIndexSerializer(CFMetaData cfMetaData)
+ {
+ return new RowIndexEntry.Serializer(new IndexHelper.IndexInfo.Serializer(cfMetaData.comparator));
+ }
+
+ static class WriterFactory extends SSTableWriter.Factory
+ {
+ @Override
+ public SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector)
+ {
+ return new BigTableWriter(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector);
+ }
+ }
+
+ static class ReaderFactory extends SSTableReader.Factory
+ {
+ @Override
+ public SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, SSTableReader.OpenReason openReason)
+ {
+ return new BigTableReader(descriptor, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
+ }
+ }
+
+ // versions are denoted as [major][minor]. Minor versions must be forward-compatible:
+ // new fields are allowed in e.g. the metadata component, but fields can't be removed
+ // or have their size changed.
+ //
+ // Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that,
+ // we always incremented the major version.
+ static class BigVersion extends Version
+ {
+ public static final String current_version = "la";
+ public static final String earliest_supported_version = "ja";
+
+ // ja (2.0.0): super columns are serialized as composites (note that there is no real format change,
+ // this is mostly a marker to know if we should expect super columns or not. We do need
+ // a major version bump however, because we should not allow streaming of super columns
+ // into this new format)
+ // tracks max local deletiontime in sstable metadata
+ // records bloom_filter_fp_chance in metadata component
+ // remove data size and column count from data file (CASSANDRA-4180)
+ // tracks max/min column values (according to comparator)
+ // jb (2.0.1): switch from crc32 to adler32 for compression checksums
+ // checksum the compressed data
+ // ka (2.1.0): new Statistics.db file format
+ // index summaries can be downsampled and the sampling level is persisted
+ // switch uncompressed checksums to adler32
+ // tracks presense of legacy (local and remote) counter shards
+ // la (3.0.0): new file name format
+
+ private final boolean isLatestVersion;
+ private final boolean hasPostCompressionAdlerChecksums;
+ private final boolean hasSamplingLevel;
+ private final boolean newStatsFile;
+ private final boolean hasAllAdlerChecksums;
+ private final boolean hasRepairedAt;
+ private final boolean tracksLegacyCounterShards;
+ private final boolean newFileName;
+
+ public BigVersion(String version)
+ {
+ super(instance,version);
+
+ isLatestVersion = version.compareTo(current_version) == 0;
+ hasPostCompressionAdlerChecksums = version.compareTo("jb") >= 0;
+ hasSamplingLevel = version.compareTo("ka") >= 0;
+ newStatsFile = version.compareTo("ka") >= 0;
+ hasAllAdlerChecksums = version.compareTo("ka") >= 0;
+ hasRepairedAt = version.compareTo("ka") >= 0;
+ tracksLegacyCounterShards = version.compareTo("ka") >= 0;
+ newFileName = version.compareTo("la") >= 0;
+ }
+
+ @Override
+ public boolean isLatestVersion()
+ {
+ return isLatestVersion;
+ }
+
+ @Override
+ public boolean hasPostCompressionAdlerChecksums()
+ {
+ return hasPostCompressionAdlerChecksums;
+ }
+
+ @Override
+ public boolean hasSamplingLevel()
+ {
+ return hasSamplingLevel;
+ }
+
+ @Override
+ public boolean hasNewStatsFile()
+ {
+ return newStatsFile;
+ }
+
+ @Override
+ public boolean hasAllAdlerChecksums()
+ {
+ return hasAllAdlerChecksums;
+ }
+
+ @Override
+ public boolean hasRepairedAt()
+ {
+ return hasRepairedAt;
+ }
+
+ @Override
+ public boolean tracksLegacyCounterShards()
+ {
+ return tracksLegacyCounterShards;
+ }
+
+ @Override
+ public boolean hasNewFileName()
+ {
+ return newFileName;
+ }
+
+ @Override
+ public boolean isCompatible()
+ {
+ return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
new file mode 100644
index 0000000..2488f86
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.RowPosition;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.compaction.ICompactionScanner;
+import org.apache.cassandra.db.composites.CellName;
+import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+/**
+ * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
+ * Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
+ */
+public class BigTableReader extends SSTableReader
+{
+ private static final Logger logger = LoggerFactory.getLogger(BigTableReader.class);
+
+ BigTableReader(Descriptor desc, Set<Component> components, CFMetaData metadata, IPartitioner partitioner, Long maxDataAge, StatsMetadata sstableMetadata, OpenReason openReason)
+ {
+ super(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
+ }
+
+ public OnDiskAtomIterator iterator(DecoratedKey key, SortedSet<CellName> columns)
+ {
+ return new SSTableNamesIterator(this, key, columns);
+ }
+
+ public OnDiskAtomIterator iterator(FileDataInput input, DecoratedKey key, SortedSet<CellName> columns, RowIndexEntry indexEntry )
+ {
+ return new SSTableNamesIterator(this, input, key, columns, indexEntry);
+ }
+
+ public OnDiskAtomIterator iterator(DecoratedKey key, ColumnSlice[] slices, boolean reverse)
+ {
+ return new SSTableSliceIterator(this, key, slices, reverse);
+ }
+
+ public OnDiskAtomIterator iterator(FileDataInput input, DecoratedKey key, ColumnSlice[] slices, boolean reverse, RowIndexEntry indexEntry)
+ {
+ return new SSTableSliceIterator(this, input, key, slices, reverse, indexEntry);
+ }
+ /**
+ *
+ * @param dataRange filter to use when reading the columns
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public ICompactionScanner getScanner(DataRange dataRange, RateLimiter limiter)
+ {
+ return new BigTableScanner(this, dataRange, limiter);
+ }
+
+
+ /**
+ * Direct I/O SSTableScanner over a defined collection of ranges of tokens.
+ *
+ * @param ranges the range of keys to cover
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public ICompactionScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter)
+ {
+ // We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249)
+ List<Pair<Long, Long>> positions = getPositionsForRanges(Range.normalize(ranges));
+ if (positions.isEmpty())
+ return new EmptyCompactionScanner(getFilename());
+ else
+ return new BigTableScanner(this, ranges, limiter);
+ }
+
+
+ /**
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateCacheAndStats true if updating stats and cache
+ * @return The index entry corresponding to the key, or null if the key is not present
+ */
+ public RowIndexEntry getPosition(RowPosition key, Operator op, boolean updateCacheAndStats)
+ {
+ // first, check bloom filter
+ if (op == Operator.EQ)
+ {
+ assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key
+ if (!bf.isPresent(((DecoratedKey)key).getKey()))
+ {
+ Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.generation);
+ return null;
+ }
+ }
+
+ // next, the key cache (only make sense for valid row key)
+ if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey))
+ {
+ DecoratedKey decoratedKey = (DecoratedKey)key;
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, decoratedKey.getKey());
+ RowIndexEntry cachedPosition = getCachedPosition(cacheKey, updateCacheAndStats);
+ if (cachedPosition != null)
+ {
+ Tracing.trace("Key cache hit for sstable {}", descriptor.generation);
+ return cachedPosition;
+ }
+ }
+
+ // check the smallest and greatest keys in the sstable to see if it can't be present
+ if (first.compareTo(key) > 0 || last.compareTo(key) < 0)
+ {
+ if (op == Operator.EQ && updateCacheAndStats)
+ bloomFilterTracker.addFalsePositive();
+
+ if (op.apply(1) < 0)
+ {
+ Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.generation);
+ return null;
+ }
+ }
+
+ int binarySearchResult = indexSummary.binarySearch(key);
+ long sampledPosition = getIndexScanPositionFromBinarySearchResult(binarySearchResult, indexSummary);
+ int sampledIndex = getIndexSummaryIndexFromBinarySearchResult(binarySearchResult);
+
+ // if we matched the -1th position, we'll start at the first position
+ sampledPosition = sampledPosition == -1 ? 0 : sampledPosition;
+
+ int effectiveInterval = indexSummary.getEffectiveIndexIntervalAfterIndex(sampledIndex);
+
+ // scan the on-disk index, starting at the nearest sampled position.
+ // The check against IndexInterval is to be exit the loop in the EQ case when the key looked for is not present
+ // (bloom filter false positive). But note that for non-EQ cases, we might need to check the first key of the
+ // next index position because the searched key can be greater the last key of the index interval checked if it
+ // is lesser than the first key of next interval (and in that case we must return the position of the first key
+ // of the next interval).
+ int i = 0;
+ Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
+ while (segments.hasNext() && i <= effectiveInterval)
+ {
+ FileDataInput in = segments.next();
+ try
+ {
+ while (!in.isEOF() && i <= effectiveInterval)
+ {
+ i++;
+
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+
+ boolean opSatisfied; // did we find an appropriate position for the op requested
+ boolean exactMatch; // is the current position an exact match for the key, suitable for caching
+
+ // Compare raw keys if possible for performance, otherwise compare decorated keys.
+ if (op == Operator.EQ)
+ {
+ opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey());
+ }
+ else
+ {
+ DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
+ int comparison = indexDecoratedKey.compareTo(key);
+ int v = op.apply(comparison);
+ opSatisfied = (v == 0);
+ exactMatch = (comparison == 0);
+ if (v < 0)
+ {
+ Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation);
+ return null;
+ }
+ }
+
+ if (opSatisfied)
+ {
+ // read data position from index entry
+ RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in, descriptor.version);
+ if (exactMatch && updateCacheAndStats)
+ {
+ assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
+ DecoratedKey decoratedKey = (DecoratedKey)key;
+
+ if (logger.isTraceEnabled())
+ {
+ // expensive sanity check! see CASSANDRA-4687
+ FileDataInput fdi = dfile.getSegment(indexEntry.position);
+ DecoratedKey keyInDisk = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(fdi));
+ if (!keyInDisk.equals(key))
+ throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath()));
+ fdi.close();
+ }
+
+ // store exact match for the key
+ cacheKey(decoratedKey, indexEntry);
+ }
+ if (op == Operator.EQ && updateCacheAndStats)
+ bloomFilterTracker.addTruePositive();
+ Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation);
+ return indexEntry;
+ }
+
+ RowIndexEntry.Serializer.skip(in);
+ }
+ }
+ catch (IOException e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, in.getPath());
+ }
+ finally
+ {
+ FileUtils.closeQuietly(in);
+ }
+ }
+
+ if (op == SSTableReader.Operator.EQ && updateCacheAndStats)
+ bloomFilterTracker.addFalsePositive();
+ Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.generation);
+ return null;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
new file mode 100644
index 0000000..c1fc079
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.util.concurrent.RateLimiter;
+
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.RowPosition;
+import org.apache.cassandra.db.columniterator.IColumnIteratorFactory;
+import org.apache.cassandra.db.columniterator.LazyColumnIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.compaction.ICompactionScanner;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Bounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class BigTableScanner implements ICompactionScanner
+{
+ protected final RandomAccessReader dfile;
+ protected final RandomAccessReader ifile;
+ public final SSTableReader sstable;
+
+ private final Iterator<AbstractBounds<RowPosition>> rangeIterator;
+ private AbstractBounds<RowPosition> currentRange;
+
+ private final DataRange dataRange;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+
+ protected Iterator<OnDiskAtomIterator> iterator;
+
+ /**
+ * @param sstable SSTable to scan; must not be null
+ * @param dataRange a single range to scan; must not be null
+ * @param limiter background i/o RateLimiter; may be null
+ */
+ public BigTableScanner(SSTableReader sstable, DataRange dataRange, RateLimiter limiter)
+ {
+ assert sstable != null;
+
+ this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter);
+ this.ifile = sstable.openIndexReader();
+ this.sstable = sstable;
+ this.dataRange = dataRange;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
+
+ List<AbstractBounds<RowPosition>> boundsList = new ArrayList<>(2);
+ if (dataRange.isWrapAround() && !dataRange.stopKey().isMinimum(sstable.partitioner))
+ {
+ // split the wrapping range into two parts: 1) the part that starts at the beginning of the sstable, and
+ // 2) the part that comes before the wrap-around
+ boundsList.add(new Bounds<>(sstable.partitioner.getMinimumToken().minKeyBound(), dataRange.stopKey(), sstable.partitioner));
+ boundsList.add(new Bounds<>(dataRange.startKey(), sstable.partitioner.getMinimumToken().maxKeyBound(), sstable.partitioner));
+ }
+ else
+ {
+ boundsList.add(new Bounds<>(dataRange.startKey(), dataRange.stopKey(), sstable.partitioner));
+ }
+ this.rangeIterator = boundsList.iterator();
+ }
+
+ /**
+ * @param sstable SSTable to scan; must not be null
+ * @param tokenRanges A set of token ranges to scan
+ * @param limiter background i/o RateLimiter; may be null
+ */
+ public BigTableScanner(SSTableReader sstable, Collection<Range<Token>> tokenRanges, RateLimiter limiter)
+ {
+ assert sstable != null;
+
+ this.dfile = limiter == null ? sstable.openDataReader() : sstable.openDataReader(limiter);
+ this.ifile = sstable.openIndexReader();
+ this.sstable = sstable;
+ this.dataRange = null;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
+
+ List<Range<Token>> normalized = Range.normalize(tokenRanges);
+ List<AbstractBounds<RowPosition>> boundsList = new ArrayList<>(normalized.size());
+ for (Range<Token> range : normalized)
+ boundsList.add(new Range<RowPosition>(range.left.maxKeyBound(sstable.partitioner),
+ range.right.maxKeyBound(sstable.partitioner),
+ sstable.partitioner));
+
+ this.rangeIterator = boundsList.iterator();
+ }
+
+ private void seekToCurrentRangeStart()
+ {
+ if (currentRange.left.isMinimum(sstable.partitioner))
+ return;
+
+ long indexPosition = sstable.getIndexScanPosition(currentRange.left);
+ // -1 means the key is before everything in the sstable. So just start from the beginning.
+ if (indexPosition == -1)
+ {
+ // Note: this method shouldn't assume we're at the start of the sstable already (see #6638) and
+ // the seeks are no-op anyway if we are.
+ ifile.seek(0);
+ dfile.seek(0);
+ return;
+ }
+
+ ifile.seek(indexPosition);
+ try
+ {
+
+ while (!ifile.isEOF())
+ {
+ indexPosition = ifile.getFilePointer();
+ DecoratedKey indexDecoratedKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
+ int comparison = indexDecoratedKey.compareTo(currentRange.left);
+ // because our range start may be inclusive or exclusive, we need to also contains()
+ // instead of just checking (comparison >= 0)
+ if (comparison > 0 || currentRange.contains(indexDecoratedKey))
+ {
+ // Found, just read the dataPosition and seek into index and data files
+ long dataPosition = ifile.readLong();
+ ifile.seek(indexPosition);
+ dfile.seek(dataPosition);
+ break;
+ }
+ else
+ {
+ RowIndexEntry.Serializer.skip(ifile);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+
+ public void close() throws IOException
+ {
+ FileUtils.close(dfile, ifile);
+ }
+
+ public long getLengthInBytes()
+ {
+ return dfile.length();
+ }
+
+ public long getCurrentPosition()
+ {
+ return dfile.getFilePointer();
+ }
+
+ public String getBackingFiles()
+ {
+ return sstable.toString();
+ }
+
+ public boolean hasNext()
+ {
+ if (iterator == null)
+ iterator = createIterator();
+ return iterator.hasNext();
+ }
+
+ public OnDiskAtomIterator next()
+ {
+ if (iterator == null)
+ iterator = createIterator();
+ return iterator.next();
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ private Iterator<OnDiskAtomIterator> createIterator()
+ {
+ return new KeyScanningIterator();
+ }
+
+ protected class KeyScanningIterator extends AbstractIterator<OnDiskAtomIterator>
+ {
+ private DecoratedKey nextKey;
+ private RowIndexEntry nextEntry;
+ private DecoratedKey currentKey;
+ private RowIndexEntry currentEntry;
+
+ protected OnDiskAtomIterator computeNext()
+ {
+ try
+ {
+ if (nextEntry == null)
+ {
+ do
+ {
+ // we're starting the first range or we just passed the end of the previous range
+ if (!rangeIterator.hasNext())
+ return endOfData();
+
+ currentRange = rangeIterator.next();
+ seekToCurrentRangeStart();
+
+ if (ifile.isEOF())
+ return endOfData();
+
+ currentKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
+ currentEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version);
+ } while (!currentRange.contains(currentKey));
+ }
+ else
+ {
+ // we're in the middle of a range
+ currentKey = nextKey;
+ currentEntry = nextEntry;
+ }
+
+ long readEnd;
+ if (ifile.isEOF())
+ {
+ nextEntry = null;
+ nextKey = null;
+ readEnd = dfile.length();
+ }
+ else
+ {
+ // we need the position of the start of the next key, regardless of whether it falls in the current range
+ nextKey = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(ifile));
+ nextEntry = rowIndexEntrySerializer.deserialize(ifile, sstable.descriptor.version);
+ readEnd = nextEntry.position;
+
+ if (!currentRange.contains(nextKey))
+ {
+ nextKey = null;
+ nextEntry = null;
+ }
+ }
+
+ if (dataRange == null || dataRange.selectsFullRowFor(currentKey.getKey()))
+ {
+ dfile.seek(currentEntry.position + currentEntry.headerOffset());
+ ByteBufferUtil.readWithShortLength(dfile); // key
+ return new SSTableIdentityIterator(sstable, dfile, currentKey);
+ }
+
+ return new LazyColumnIterator(currentKey, new IColumnIteratorFactory()
+ {
+ public OnDiskAtomIterator create()
+ {
+ return dataRange.columnFilter(currentKey.getKey()).getSSTableColumnIterator(sstable, dfile, currentKey, currentEntry);
+ }
+ });
+
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ return getClass().getSimpleName() + "(" +
+ "dfile=" + dfile +
+ " ifile=" + ifile +
+ " sstable=" + sstable +
+ ")";
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
new file mode 100644
index 0000000..ec53b4e
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.compress.CompressedSequentialWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
+import org.apache.cassandra.io.util.FileMark;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.SegmentedFile;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FilterFactory;
+import org.apache.cassandra.utils.IFilter;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.StreamingHistogram;
+
+public class BigTableWriter extends SSTableWriter
+{
+ private static final Logger logger = LoggerFactory.getLogger(BigTableWriter.class);
+
+ // not very random, but the only value that can't be mistaken for a legal column-name length
+ public static final int END_OF_ROW = 0x0000;
+
+ private IndexWriter iwriter;
+ private SegmentedFile.Builder dbuilder;
+ private final SequentialWriter dataFile;
+ private DecoratedKey lastWrittenKey;
+ private FileMark dataMark;
+
+ BigTableWriter(Descriptor descriptor, Long keyCount, Long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector)
+ {
+ super(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector);
+
+ iwriter = new IndexWriter(keyCount);
+
+ if (compression)
+ {
+ dataFile = SequentialWriter.open(getFilename(),
+ descriptor.filenameFor(Component.COMPRESSION_INFO),
+ metadata.compressionParameters(),
+ metadataCollector);
+ dbuilder = SegmentedFile.getCompressedBuilder((CompressedSequentialWriter) dataFile);
+ }
+ else
+ {
+ dataFile = SequentialWriter.open(new File(getFilename()), new File(descriptor.filenameFor(Component.CRC)));
+ dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
+ }
+ }
+
+ public void mark()
+ {
+ dataMark = dataFile.mark();
+ iwriter.mark();
+ }
+
+ public void resetAndTruncate()
+ {
+ dataFile.resetAndTruncate(dataMark);
+ iwriter.resetAndTruncate();
+ }
+
+ /**
+ * Perform sanity checks on @param decoratedKey and @return the position in the data file before any data is written
+ */
+ private long beforeAppend(DecoratedKey decoratedKey)
+ {
+ assert decoratedKey != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values
+ if (lastWrittenKey != null && lastWrittenKey.compareTo(decoratedKey) >= 0)
+ throw new RuntimeException("Last written key " + lastWrittenKey + " >= current key " + decoratedKey + " writing into " + getFilename());
+ return (lastWrittenKey == null) ? 0 : dataFile.getFilePointer();
+ }
+
+ private void afterAppend(DecoratedKey decoratedKey, long dataPosition, RowIndexEntry index)
+ {
+ metadataCollector.addKey(decoratedKey.getKey());
+ lastWrittenKey = decoratedKey;
+ last = lastWrittenKey;
+ if (first == null)
+ first = lastWrittenKey;
+
+ if (logger.isTraceEnabled())
+ logger.trace("wrote {} at {}", decoratedKey, dataPosition);
+ iwriter.append(decoratedKey, index);
+ dbuilder.addPotentialBoundary(dataPosition);
+ }
+
+ /**
+ * @param row
+ * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row
+ */
+ public RowIndexEntry append(AbstractCompactedRow row)
+ {
+ long currentPosition = beforeAppend(row.key);
+ RowIndexEntry entry;
+ try
+ {
+ entry = row.write(currentPosition, dataFile);
+ if (entry == null)
+ return null;
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, dataFile.getPath());
+ }
+ metadataCollector.update(dataFile.getFilePointer() - currentPosition, row.columnStats());
+ afterAppend(row.key, currentPosition, entry);
+ return entry;
+ }
+
+ public void append(DecoratedKey decoratedKey, ColumnFamily cf)
+ {
+ long startPosition = beforeAppend(decoratedKey);
+ try
+ {
+ RowIndexEntry entry = rawAppend(cf, startPosition, decoratedKey, dataFile.stream);
+ afterAppend(decoratedKey, startPosition, entry);
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, dataFile.getPath());
+ }
+ metadataCollector.update(dataFile.getFilePointer() - startPosition, cf.getColumnStats());
+ }
+
+ private static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey key, DataOutputPlus out) throws IOException
+ {
+ assert cf.hasColumns() || cf.isMarkedForDelete();
+
+ ColumnIndex.Builder builder = new ColumnIndex.Builder(cf, key.getKey(), out);
+ ColumnIndex index = builder.build(cf);
+
+ out.writeShort(END_OF_ROW);
+ return RowIndexEntry.create(startPosition, cf.deletionInfo().getTopLevelDeletion(), index);
+ }
+
+ /**
+ * @throws IOException if a read from the DataInput fails
+ * @throws FSWriteError if a write to the dataFile fails
+ */
+ public long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Version version) throws IOException
+ {
+ long currentPosition = beforeAppend(key);
+
+ ColumnStats.MaxLongTracker maxTimestampTracker = new ColumnStats.MaxLongTracker(Long.MAX_VALUE);
+ ColumnStats.MinLongTracker minTimestampTracker = new ColumnStats.MinLongTracker(Long.MIN_VALUE);
+ ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE);
+ List<ByteBuffer> minColumnNames = Collections.emptyList();
+ List<ByteBuffer> maxColumnNames = Collections.emptyList();
+ StreamingHistogram tombstones = new StreamingHistogram(TOMBSTONE_HISTOGRAM_BIN_SIZE);
+ boolean hasLegacyCounterShards = false;
+
+ ColumnFamily cf = ArrayBackedSortedColumns.factory.create(metadata);
+ cf.delete(DeletionTime.serializer.deserialize(in));
+
+ ColumnIndex.Builder columnIndexer = new ColumnIndex.Builder(cf, key.getKey(), dataFile.stream);
+
+ if (cf.deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE)
+ {
+ tombstones.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
+ maxDeletionTimeTracker.update(cf.deletionInfo().getTopLevelDeletion().localDeletionTime);
+ minTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
+ maxTimestampTracker.update(cf.deletionInfo().getTopLevelDeletion().markedForDeleteAt);
+ }
+
+ Iterator<RangeTombstone> rangeTombstoneIterator = cf.deletionInfo().rangeIterator();
+ while (rangeTombstoneIterator.hasNext())
+ {
+ RangeTombstone rangeTombstone = rangeTombstoneIterator.next();
+ tombstones.update(rangeTombstone.getLocalDeletionTime());
+ minTimestampTracker.update(rangeTombstone.timestamp());
+ maxTimestampTracker.update(rangeTombstone.timestamp());
+ maxDeletionTimeTracker.update(rangeTombstone.getLocalDeletionTime());
+ minColumnNames = ColumnNameHelper.minComponents(minColumnNames, rangeTombstone.min, metadata.comparator);
+ maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, rangeTombstone.max, metadata.comparator);
+ }
+
+ Iterator<OnDiskAtom> iter = AbstractCell.onDiskIterator(in, ColumnSerializer.Flag.PRESERVE_SIZE, Integer.MIN_VALUE, version, metadata.comparator);
+ try
+ {
+ while (iter.hasNext())
+ {
+ OnDiskAtom atom = iter.next();
+ if (atom == null)
+ break;
+
+ if (atom instanceof CounterCell)
+ {
+ atom = ((CounterCell) atom).markLocalToBeCleared();
+ hasLegacyCounterShards = hasLegacyCounterShards || ((CounterCell) atom).hasLegacyShards();
+ }
+
+ int deletionTime = atom.getLocalDeletionTime();
+ if (deletionTime < Integer.MAX_VALUE)
+ tombstones.update(deletionTime);
+ minTimestampTracker.update(atom.timestamp());
+ maxTimestampTracker.update(atom.timestamp());
+ minColumnNames = ColumnNameHelper.minComponents(minColumnNames, atom.name(), metadata.comparator);
+ maxColumnNames = ColumnNameHelper.maxComponents(maxColumnNames, atom.name(), metadata.comparator);
+ maxDeletionTimeTracker.update(atom.getLocalDeletionTime());
+
+ columnIndexer.add(atom); // This write the atom on disk too
+ }
+
+ columnIndexer.maybeWriteEmptyRowHeader();
+ dataFile.stream.writeShort(END_OF_ROW);
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, dataFile.getPath());
+ }
+
+ metadataCollector.updateMinTimestamp(minTimestampTracker.get())
+ .updateMaxTimestamp(maxTimestampTracker.get())
+ .updateMaxLocalDeletionTime(maxDeletionTimeTracker.get())
+ .addRowSize(dataFile.getFilePointer() - currentPosition)
+ .addColumnCount(columnIndexer.writtenAtomCount())
+ .mergeTombstoneHistogram(tombstones)
+ .updateMinColumnNames(minColumnNames)
+ .updateMaxColumnNames(maxColumnNames)
+ .updateHasLegacyCounterShards(hasLegacyCounterShards);
+
+ afterAppend(key, currentPosition, RowIndexEntry.create(currentPosition, cf.deletionInfo().getTopLevelDeletion(), columnIndexer.build()));
+ return currentPosition;
+ }
+
+ /**
+ * After failure, attempt to close the index writer and data file before deleting all temp components for the sstable
+ */
+ public void abort(boolean closeBf)
+ {
+ assert descriptor.type.isTemporary;
+ if (iwriter == null && dataFile == null)
+ return;
+ if (iwriter != null)
+ {
+ FileUtils.closeQuietly(iwriter.indexFile);
+ if (closeBf)
+ {
+ iwriter.bf.close();
+ }
+ }
+ if (dataFile!= null)
+ FileUtils.closeQuietly(dataFile);
+
+ Set<Component> components = SSTable.componentsFor(descriptor);
+ try
+ {
+ if (!components.isEmpty())
+ SSTable.delete(descriptor, components);
+ }
+ catch (FSWriteError e)
+ {
+ logger.error(String.format("Failed deleting temp components for %s", descriptor), e);
+ throw e;
+ }
+ }
+
+ // we use this method to ensure any managed data we may have retained references to during the write are no
+ // longer referenced, so that we do not need to enclose the expensive call to closeAndOpenReader() in a transaction
+ public void isolateReferences()
+ {
+ // currently we only maintain references to first/last/lastWrittenKey from the data provided; all other
+ // data retention is done through copying
+ first = getMinimalKey(first);
+ last = lastWrittenKey = getMinimalKey(last);
+ }
+
+ public SSTableReader openEarly(long maxDataAge)
+ {
+ StatsMetadata sstableMetadata = (StatsMetadata) metadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName(),
+ metadata.getBloomFilterFpChance(),
+ repairedAt).get(MetadataType.STATS);
+
+ // find the max (exclusive) readable key
+ DecoratedKey exclusiveUpperBoundOfReadableIndex = iwriter.getMaxReadableKey(0);
+ if (exclusiveUpperBoundOfReadableIndex == null)
+ return null;
+
+ // create temp links if they don't already exist
+ Descriptor link = descriptor.asType(Descriptor.Type.TEMPLINK);
+ if (!new File(link.filenameFor(Component.PRIMARY_INDEX)).exists())
+ {
+ FileUtils.createHardLink(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), new File(link.filenameFor(Component.PRIMARY_INDEX)));
+ FileUtils.createHardLink(new File(descriptor.filenameFor(Component.DATA)), new File(link.filenameFor(Component.DATA)));
+ }
+
+ // open the reader early, giving it a FINAL descriptor type so that it is indistinguishable for other consumers
+ SegmentedFile ifile = iwriter.builder.openEarly(link.filenameFor(Component.PRIMARY_INDEX));
+ SegmentedFile dfile = dbuilder.openEarly(link.filenameFor(Component.DATA));
+ SSTableReader sstable = SSTableReader.internalOpen(descriptor.asType(Descriptor.Type.FINAL),
+ components, metadata,
+ partitioner, ifile,
+ dfile, iwriter.summary.build(partitioner, exclusiveUpperBoundOfReadableIndex),
+ iwriter.bf, maxDataAge, sstableMetadata, SSTableReader.OpenReason.EARLY);
+
+ // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed)
+ sstable.first = getMinimalKey(first);
+ sstable.last = getMinimalKey(exclusiveUpperBoundOfReadableIndex);
+ DecoratedKey inclusiveUpperBoundOfReadableData = iwriter.getMaxReadableKey(1);
+ if (inclusiveUpperBoundOfReadableData == null)
+ return null;
+ int offset = 2;
+ while (true)
+ {
+ RowIndexEntry indexEntry = sstable.getPosition(inclusiveUpperBoundOfReadableData, SSTableReader.Operator.GT);
+ if (indexEntry != null && indexEntry.position <= dataFile.getLastFlushOffset())
+ break;
+ inclusiveUpperBoundOfReadableData = iwriter.getMaxReadableKey(offset++);
+ if (inclusiveUpperBoundOfReadableData == null)
+ return null;
+ }
+ sstable.last = getMinimalKey(inclusiveUpperBoundOfReadableData);
+ return sstable;
+ }
+
+ public SSTableReader closeAndOpenReader(long maxDataAge, long repairedAt)
+ {
+ Pair<Descriptor, StatsMetadata> p = close(repairedAt);
+ Descriptor newdesc = p.left;
+ StatsMetadata sstableMetadata = p.right;
+
+ // finalize in-memory state for the reader
+ SegmentedFile ifile = iwriter.builder.complete(newdesc.filenameFor(Component.PRIMARY_INDEX));
+ SegmentedFile dfile = dbuilder.complete(newdesc.filenameFor(Component.DATA));
+ SSTableReader sstable = SSTableReader.internalOpen(newdesc,
+ components,
+ metadata,
+ partitioner,
+ ifile,
+ dfile,
+ iwriter.summary.build(partitioner),
+ iwriter.bf,
+ maxDataAge,
+ sstableMetadata,
+ SSTableReader.OpenReason.NORMAL);
+ sstable.first = getMinimalKey(first);
+ sstable.last = getMinimalKey(last);
+ // try to save the summaries to disk
+ sstable.saveSummary(iwriter.builder, dbuilder);
+ iwriter = null;
+ dbuilder = null;
+ return sstable;
+ }
+
+ // Close the writer and return the descriptor to the new sstable and it's metadata
+ public Pair<Descriptor, StatsMetadata> close()
+ {
+ return close(this.repairedAt);
+ }
+
+ private Pair<Descriptor, StatsMetadata> close(long repairedAt)
+ {
+
+ // index and filter
+ iwriter.close();
+ // main data, close will truncate if necessary
+ dataFile.close();
+ dataFile.writeFullChecksum(descriptor);
+ // write sstable statistics
+ Map<MetadataType, MetadataComponent> metadataComponents = metadataCollector.finalizeMetadata(
+ partitioner.getClass().getCanonicalName(),
+ metadata.getBloomFilterFpChance(),
+ repairedAt);
+ writeMetadata(descriptor, metadataComponents);
+
+ // save the table of components
+ SSTable.appendTOC(descriptor, components);
+
+ // remove the 'tmp' marker from all components
+ return Pair.create(SSTableWriter.rename(descriptor, components), (StatsMetadata) metadataComponents.get(MetadataType.STATS));
+
+ }
+
+
+ private static void writeMetadata(Descriptor desc, Map<MetadataType, MetadataComponent> components)
+ {
+ SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(Component.STATS)));
+ try
+ {
+ desc.getMetadataSerializer().serialize(components, out.stream);
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, out.getPath());
+ }
+ finally
+ {
+ out.close();
+ }
+ }
+
+ public long getFilePointer()
+ {
+ return dataFile.getFilePointer();
+ }
+
+ public long getOnDiskFilePointer()
+ {
+ return dataFile.getOnDiskFilePointer();
+ }
+
+ /**
+ * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
+ */
+ class IndexWriter implements Closeable
+ {
+ private final SequentialWriter indexFile;
+ public final SegmentedFile.Builder builder;
+ public final IndexSummaryBuilder summary;
+ public final IFilter bf;
+ private FileMark mark;
+
+ IndexWriter(long keyCount)
+ {
+ indexFile = SequentialWriter.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
+ builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
+ summary = new IndexSummaryBuilder(keyCount, metadata.getMinIndexInterval(), Downsampling.BASE_SAMPLING_LEVEL);
+ bf = FilterFactory.getFilter(keyCount, metadata.getBloomFilterFpChance(), true);
+ }
+
+ // finds the last (-offset) decorated key that can be guaranteed to occur fully in the flushed portion of the index file
+ DecoratedKey getMaxReadableKey(int offset)
+ {
+ long maxIndexLength = indexFile.getLastFlushOffset();
+ return summary.getMaxReadableKey(maxIndexLength, offset);
+ }
+
+ public void append(DecoratedKey key, RowIndexEntry indexEntry)
+ {
+ bf.add(key.getKey());
+ long indexPosition = indexFile.getFilePointer();
+ try
+ {
+ ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile.stream);
+ rowIndexEntrySerializer.serialize(indexEntry, indexFile.stream);
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, indexFile.getPath());
+ }
+
+ if (logger.isTraceEnabled())
+ logger.trace("wrote index entry: {} at {}", indexEntry, indexPosition);
+
+ summary.maybeAddEntry(key, indexPosition);
+ builder.addPotentialBoundary(indexPosition);
+ }
+
+ /**
+ * Closes the index and bloomfilter, making the public state of this writer valid for consumption.
+ */
+ public void close()
+ {
+ if (components.contains(Component.FILTER))
+ {
+ String path = descriptor.filenameFor(Component.FILTER);
+ try
+ {
+ // bloom filter
+ FileOutputStream fos = new FileOutputStream(path);
+ DataOutputStreamAndChannel stream = new DataOutputStreamAndChannel(fos);
+ FilterFactory.serialize(bf, stream);
+ stream.flush();
+ fos.getFD().sync();
+ stream.close();
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, path);
+ }
+ }
+
+ // index
+ long position = indexFile.getFilePointer();
+ indexFile.close(); // calls force
+ FileUtils.truncate(indexFile.getPath(), position);
+ }
+
+ public void mark()
+ {
+ mark = indexFile.mark();
+ }
+
+ public void resetAndTruncate()
+ {
+ // we can't un-set the bloom filter addition, but extra keys in there are harmless.
+ // we can't reset dbuilder either, but that is the last thing called in afterappend so
+ // we assume that if that worked then we won't be trying to reset.
+ indexFile.resetAndTruncate(mark);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java
new file mode 100644
index 0000000..a69cff9
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexedSliceReader.java
@@ -0,0 +1,500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+
+import com.google.common.collect.AbstractIterator;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.composites.CellNameType;
+import org.apache.cassandra.db.composites.Composite;
+import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.IndexHelper;
+import org.apache.cassandra.io.sstable.IndexHelper.IndexInfo;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileMark;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+/**
+ * This is a reader that finds the block for a starting column and returns blocks before/after it for each next call.
+ * This function assumes that the CF is sorted by name and exploits the name index.
+ */
+class IndexedSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
+{
+ private final ColumnFamily emptyColumnFamily;
+
+ private final SSTableReader sstable;
+ private final List<IndexHelper.IndexInfo> indexes;
+ private final FileDataInput originalInput;
+ private FileDataInput file;
+ private final boolean reversed;
+ private final ColumnSlice[] slices;
+ private final BlockFetcher fetcher;
+ private final Deque<OnDiskAtom> blockColumns = new ArrayDeque<OnDiskAtom>();
+ private final CellNameType comparator;
+
+ // Holds range tombstone in reverse queries. See addColumn()
+ private final Deque<OnDiskAtom> rangeTombstonesReversed;
+
+ /**
+ * This slice reader assumes that slices are sorted correctly, e.g. that for forward lookup slices are in
+ * lexicographic order of start elements and that for reverse lookup they are in reverse lexicographic order of
+ * finish (reverse start) elements. i.e. forward: [a,b],[d,e],[g,h] reverse: [h,g],[e,d],[b,a]. This reader also
+ * assumes that validation has been performed in terms of intervals (no overlapping intervals).
+ */
+ IndexedSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, ColumnSlice[] slices, boolean reversed)
+ {
+ Tracing.trace("Seeking to partition indexed section in data file");
+ this.sstable = sstable;
+ this.originalInput = input;
+ this.reversed = reversed;
+ this.slices = slices;
+ this.comparator = sstable.metadata.comparator;
+ this.rangeTombstonesReversed = reversed ? new ArrayDeque<OnDiskAtom>() : null;
+
+ try
+ {
+ this.indexes = indexEntry.columnsIndex();
+ emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata);
+ if (indexes.isEmpty())
+ {
+ setToRowStart(indexEntry, input);
+ emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file));
+ fetcher = new SimpleBlockFetcher();
+ }
+ else
+ {
+ emptyColumnFamily.delete(indexEntry.deletionTime());
+ fetcher = new IndexedBlockFetcher(indexEntry.position);
+ }
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, file.getPath());
+ }
+ }
+
+ /**
+ * Sets the seek position to the start of the row for column scanning.
+ */
+ private void setToRowStart(RowIndexEntry rowEntry, FileDataInput in) throws IOException
+ {
+ if (in == null)
+ {
+ this.file = sstable.getFileDataInput(rowEntry.position);
+ }
+ else
+ {
+ this.file = in;
+ in.seek(rowEntry.position);
+ }
+ sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file));
+ }
+
+ public ColumnFamily getColumnFamily()
+ {
+ return emptyColumnFamily;
+ }
+
+ public DecoratedKey getKey()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ protected OnDiskAtom computeNext()
+ {
+ while (true)
+ {
+ if (reversed)
+ {
+ // Return all tombstone for the block first (see addColumn() below)
+ OnDiskAtom column = rangeTombstonesReversed.poll();
+ if (column != null)
+ return column;
+ }
+
+ OnDiskAtom column = blockColumns.poll();
+ if (column == null)
+ {
+ if (!fetcher.fetchMoreData())
+ return endOfData();
+ }
+ else
+ {
+ return column;
+ }
+ }
+ }
+
+ public void close() throws IOException
+ {
+ if (originalInput == null && file != null)
+ file.close();
+ }
+
+ protected void addColumn(OnDiskAtom col)
+ {
+ if (reversed)
+ {
+ /*
+ * We put range tomstone markers at the beginning of the range they delete. But for reversed queries,
+ * the caller still need to know about a RangeTombstone before it sees any column that it covers.
+ * To make that simple, we keep said tombstones separate and return them all before any column for
+ * a given block.
+ */
+ if (col instanceof RangeTombstone)
+ rangeTombstonesReversed.addFirst(col);
+ else
+ blockColumns.addFirst(col);
+ }
+ else
+ {
+ blockColumns.addLast(col);
+ }
+ }
+
+ private abstract class BlockFetcher
+ {
+ protected int currentSliceIdx;
+
+ protected BlockFetcher(int sliceIdx)
+ {
+ this.currentSliceIdx = sliceIdx;
+ }
+
+ /*
+ * Return the smallest key selected by the current ColumnSlice.
+ */
+ protected Composite currentStart()
+ {
+ return reversed ? slices[currentSliceIdx].finish : slices[currentSliceIdx].start;
+ }
+
+ /*
+ * Return the biggest key selected by the current ColumnSlice.
+ */
+ protected Composite currentFinish()
+ {
+ return reversed ? slices[currentSliceIdx].start : slices[currentSliceIdx].finish;
+ }
+
+ protected abstract boolean setNextSlice();
+
+ protected abstract boolean fetchMoreData();
+
+ protected boolean isColumnBeforeSliceStart(OnDiskAtom column)
+ {
+ return isBeforeSliceStart(column.name());
+ }
+
+ protected boolean isBeforeSliceStart(Composite name)
+ {
+ Composite start = currentStart();
+ return !start.isEmpty() && comparator.compare(name, start) < 0;
+ }
+
+ protected boolean isColumnBeforeSliceFinish(OnDiskAtom column)
+ {
+ Composite finish = currentFinish();
+ return finish.isEmpty() || comparator.compare(column.name(), finish) <= 0;
+ }
+
+ protected boolean isAfterSliceFinish(Composite name)
+ {
+ Composite finish = currentFinish();
+ return !finish.isEmpty() && comparator.compare(name, finish) > 0;
+ }
+ }
+
+ private class IndexedBlockFetcher extends BlockFetcher
+ {
+ // where this row starts
+ private final long columnsStart;
+
+ // the index entry for the next block to deserialize
+ private int nextIndexIdx = -1;
+
+ // index of the last block we've read from disk;
+ private int lastDeserializedBlock = -1;
+
+ // For reversed, keep columns at the beginning of the last deserialized block that
+ // may still match a slice
+ private final Deque<OnDiskAtom> prefetched;
+
+ public IndexedBlockFetcher(long columnsStart)
+ {
+ super(-1);
+ this.columnsStart = columnsStart;
+ this.prefetched = reversed ? new ArrayDeque<OnDiskAtom>() : null;
+ setNextSlice();
+ }
+
+ protected boolean setNextSlice()
+ {
+ while (++currentSliceIdx < slices.length)
+ {
+ nextIndexIdx = IndexHelper.indexFor(slices[currentSliceIdx].start, indexes, comparator, reversed, nextIndexIdx);
+ if (nextIndexIdx < 0 || nextIndexIdx >= indexes.size())
+ // no index block for that slice
+ continue;
+
+ // Check if we can exclude this slice entirely from the index
+ IndexInfo info = indexes.get(nextIndexIdx);
+ if (reversed)
+ {
+ if (!isBeforeSliceStart(info.lastName))
+ return true;
+ }
+ else
+ {
+ if (!isAfterSliceFinish(info.firstName))
+ return true;
+ }
+ }
+ nextIndexIdx = -1;
+ return false;
+ }
+
+ protected boolean hasMoreSlice()
+ {
+ return currentSliceIdx < slices.length;
+ }
+
+ protected boolean fetchMoreData()
+ {
+ if (!hasMoreSlice())
+ return false;
+
+ // If we read blocks in reversed disk order, we may have columns from the previous block to handle.
+ // Note that prefetched keeps columns in reversed disk order.
+ if (reversed && !prefetched.isEmpty())
+ {
+ boolean gotSome = false;
+ // Avoids some comparison when we know it's not useful
+ boolean inSlice = false;
+
+ OnDiskAtom prefetchedCol;
+ while ((prefetchedCol = prefetched.peek() ) != null)
+ {
+ // col is before slice, we update the slice
+ if (isColumnBeforeSliceStart(prefetchedCol))
+ {
+ inSlice = false;
+ if (!setNextSlice())
+ return false;
+ }
+ // col is within slice, all columns
+ // (we go in reverse, so as soon as we are in a slice, no need to check
+ // we're after the slice until we change slice)
+ else if (inSlice || isColumnBeforeSliceFinish(prefetchedCol))
+ {
+ blockColumns.addLast(prefetched.poll());
+ gotSome = true;
+ inSlice = true;
+ }
+ // if col is after slice, ignore
+ else
+ {
+ prefetched.poll();
+ }
+ }
+ if (gotSome)
+ return true;
+ }
+ try
+ {
+ return getNextBlock();
+ }
+ catch (IOException e)
+ {
+ throw new CorruptSSTableException(e, file.getPath());
+ }
+ }
+
+ private boolean getNextBlock() throws IOException
+ {
+ if (lastDeserializedBlock == nextIndexIdx)
+ {
+ if (reversed)
+ nextIndexIdx--;
+ else
+ nextIndexIdx++;
+ }
+ lastDeserializedBlock = nextIndexIdx;
+
+ // Are we done?
+ if (lastDeserializedBlock < 0 || lastDeserializedBlock >= indexes.size())
+ return false;
+
+ IndexInfo currentIndex = indexes.get(lastDeserializedBlock);
+
+ /* seek to the correct offset to the data, and calculate the data size */
+ long positionToSeek = columnsStart + currentIndex.offset;
+
+ // With new promoted indexes, our first seek in the data file will happen at that point.
+ if (file == null)
+ file = originalInput == null ? sstable.getFileDataInput(positionToSeek) : originalInput;
+
+ AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
+
+ file.seek(positionToSeek);
+ FileMark mark = file.mark();
+
+ // We remenber when we are whithin a slice to avoid some comparison
+ boolean inSlice = false;
+
+ // scan from index start
+ while (file.bytesPastMark(mark) < currentIndex.width || deserializer.hasUnprocessed())
+ {
+ // col is before slice
+ // (If in slice, don't bother checking that until we change slice)
+ Composite start = currentStart();
+ if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0)
+ {
+ if (reversed)
+ {
+ // the next slice select columns that are before the current one, so it may
+ // match this column, so keep it around.
+ prefetched.addFirst(deserializer.readNext());
+ }
+ else
+ {
+ deserializer.skipNext();
+ }
+ }
+ // col is within slice
+ else
+ {
+ Composite finish = currentFinish();
+ if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0)
+ {
+ inSlice = true;
+ addColumn(deserializer.readNext());
+ }
+ // col is after slice.
+ else
+ {
+ // When reading forward, if we hit a column that sorts after the current slice, it means we're done with this slice.
+ // For reversed, this may either mean that we're done with the current slice, or that we need to read the previous
+ // index block. However, we can be sure that we are in the first case though (the current slice is done) if the first
+ // columns of the block were not part of the current slice, i.e. if we have columns in prefetched.
+ if (reversed && prefetched.isEmpty())
+ break;
+
+ if (!setNextSlice())
+ break;
+
+ inSlice = false;
+
+ // The next index block now corresponds to the first block that may have columns for the newly set slice.
+ // So if it's different from the current block, we're done with this block. And in that case, we know
+ // that our prefetched columns won't match.
+ if (nextIndexIdx != lastDeserializedBlock)
+ {
+ if (reversed)
+ prefetched.clear();
+ break;
+ }
+
+ // Even if the next slice may have column in this blocks, if we're reversed, those columns have been
+ // prefetched and we're done with that block
+ if (reversed)
+ break;
+
+ // otherwise, we will deal with that column at the next iteration
+ }
+ }
+ }
+ return true;
+ }
+ }
+
+ private class SimpleBlockFetcher extends BlockFetcher
+ {
+ public SimpleBlockFetcher() throws IOException
+ {
+ // Since we have to deserialize in order and will read all slices might as well reverse the slices and
+ // behave as if it was not reversed
+ super(reversed ? slices.length - 1 : 0);
+
+ // We remenber when we are whithin a slice to avoid some comparison
+ boolean inSlice = false;
+
+ AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
+ while (deserializer.hasNext())
+ {
+ // col is before slice
+ // (If in slice, don't bother checking that until we change slice)
+ Composite start = currentStart();
+ if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0)
+ {
+ deserializer.skipNext();
+ continue;
+ }
+
+ // col is within slice
+ Composite finish = currentFinish();
+ if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0)
+ {
+ inSlice = true;
+ addColumn(deserializer.readNext());
+ }
+ // col is after slice. more slices?
+ else
+ {
+ inSlice = false;
+ if (!setNextSlice())
+ break;
+ }
+ }
+ }
+
+ protected boolean setNextSlice()
+ {
+ if (reversed)
+ {
+ if (currentSliceIdx <= 0)
+ return false;
+
+ currentSliceIdx--;
+ }
+ else
+ {
+ if (currentSliceIdx >= slices.length - 1)
+ return false;
+
+ currentSliceIdx++;
+ }
+ return true;
+ }
+
+ protected boolean fetchMoreData()
+ {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java
new file mode 100644
index 0000000..07dc59a
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableNamesIterator.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.*;
+
+import com.google.common.collect.AbstractIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.composites.CellName;
+import org.apache.cassandra.db.composites.CellNameType;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.IndexHelper;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.io.util.FileMark;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+class SSTableNamesIterator extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
+{
+ private ColumnFamily cf;
+ private final SSTableReader sstable;
+ private FileDataInput fileToClose;
+ private Iterator<OnDiskAtom> iter;
+ public final SortedSet<CellName> columns;
+ public final DecoratedKey key;
+
+ public SSTableNamesIterator(SSTableReader sstable, DecoratedKey key, SortedSet<CellName> columns)
+ {
+ assert columns != null;
+ this.sstable = sstable;
+ this.columns = columns;
+ this.key = key;
+
+ RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
+ if (indexEntry == null)
+ return;
+
+ try
+ {
+ read(sstable, null, indexEntry);
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ finally
+ {
+ if (fileToClose != null)
+ FileUtils.closeQuietly(fileToClose);
+ }
+ }
+
+ public SSTableNamesIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, SortedSet<CellName> columns, RowIndexEntry indexEntry)
+ {
+ assert columns != null;
+ this.sstable = sstable;
+ this.columns = columns;
+ this.key = key;
+
+ try
+ {
+ read(sstable, file, indexEntry);
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+
+ private FileDataInput createFileDataInput(long position)
+ {
+ fileToClose = sstable.getFileDataInput(position);
+ return fileToClose;
+ }
+
+ private void read(SSTableReader sstable, FileDataInput file, RowIndexEntry indexEntry)
+ throws IOException
+ {
+ List<IndexHelper.IndexInfo> indexList;
+
+ // If the entry is not indexed or the index is not promoted, read from the row start
+ if (!indexEntry.isIndexed())
+ {
+ if (file == null)
+ file = createFileDataInput(indexEntry.position);
+ else
+ file.seek(indexEntry.position);
+
+ DecoratedKey keyInDisk = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file));
+ assert keyInDisk.equals(key) : String.format("%s != %s in %s", keyInDisk, key, file.getPath());
+ }
+
+ indexList = indexEntry.columnsIndex();
+
+ if (!indexEntry.isIndexed())
+ {
+ ColumnFamilySerializer serializer = ColumnFamily.serializer;
+ try
+ {
+ cf = ArrayBackedSortedColumns.factory.create(sstable.metadata);
+ cf.delete(DeletionTime.serializer.deserialize(file));
+ }
+ catch (Exception e)
+ {
+ throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e);
+ }
+ }
+ else
+ {
+ cf = ArrayBackedSortedColumns.factory.create(sstable.metadata);
+ cf.delete(indexEntry.deletionTime());
+ }
+
+ List<OnDiskAtom> result = new ArrayList<OnDiskAtom>();
+ if (indexList.isEmpty())
+ {
+ readSimpleColumns(file, columns, result);
+ }
+ else
+ {
+ readIndexedColumns(sstable.metadata, file, columns, indexList, indexEntry.position, result);
+ }
+
+ // create an iterator view of the columns we read
+ iter = result.iterator();
+ }
+
+ private void readSimpleColumns(FileDataInput file, SortedSet<CellName> columnNames, List<OnDiskAtom> result)
+ {
+ Iterator<OnDiskAtom> atomIterator = cf.metadata().getOnDiskIterator(file, sstable.descriptor.version);
+ int n = 0;
+ while (atomIterator.hasNext())
+ {
+ OnDiskAtom column = atomIterator.next();
+ if (column instanceof Cell)
+ {
+ if (columnNames.contains(column.name()))
+ {
+ result.add(column);
+ if (++n >= columns.size())
+ break;
+ }
+ }
+ else
+ {
+ result.add(column);
+ }
+ }
+ }
+
+ private void readIndexedColumns(CFMetaData metadata,
+ FileDataInput file,
+ SortedSet<CellName> columnNames,
+ List<IndexHelper.IndexInfo> indexList,
+ long basePosition,
+ List<OnDiskAtom> result)
+ throws IOException
+ {
+ /* get the various column ranges we have to read */
+ CellNameType comparator = metadata.comparator;
+ List<IndexHelper.IndexInfo> ranges = new ArrayList<IndexHelper.IndexInfo>();
+ int lastIndexIdx = -1;
+ for (CellName name : columnNames)
+ {
+ int index = IndexHelper.indexFor(name, indexList, comparator, false, lastIndexIdx);
+ if (index < 0 || index == indexList.size())
+ continue;
+ IndexHelper.IndexInfo indexInfo = indexList.get(index);
+ // Check the index block does contain the column names and that we haven't inserted this block yet.
+ if (comparator.compare(name, indexInfo.firstName) < 0 || index == lastIndexIdx)
+ continue;
+
+ ranges.add(indexInfo);
+ lastIndexIdx = index;
+ }
+
+ if (ranges.isEmpty())
+ return;
+
+ Iterator<CellName> toFetch = columnNames.iterator();
+ CellName nextToFetch = toFetch.next();
+ for (IndexHelper.IndexInfo indexInfo : ranges)
+ {
+ long positionToSeek = basePosition + indexInfo.offset;
+
+ // With new promoted indexes, our first seek in the data file will happen at that point.
+ if (file == null)
+ file = createFileDataInput(positionToSeek);
+
+ AtomDeserializer deserializer = cf.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
+ file.seek(positionToSeek);
+ FileMark mark = file.mark();
+ while (file.bytesPastMark(mark) < indexInfo.width && nextToFetch != null)
+ {
+ int cmp = deserializer.compareNextTo(nextToFetch);
+ if (cmp == 0)
+ {
+ nextToFetch = toFetch.hasNext() ? toFetch.next() : null;
+ result.add(deserializer.readNext());
+ continue;
+ }
+
+ deserializer.skipNext();
+ if (cmp > 0)
+ nextToFetch = toFetch.hasNext() ? toFetch.next() : null;
+ }
+ }
+ }
+
+ public DecoratedKey getKey()
+ {
+ return key;
+ }
+
+ public ColumnFamily getColumnFamily()
+ {
+ return cf;
+ }
+
+ protected OnDiskAtom computeNext()
+ {
+ if (iter == null || !iter.hasNext())
+ return endOfData();
+ return iter.next();
+ }
+
+ public void close() throws IOException { }
+}
[4/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
new file mode 100644
index 0000000..c3062f7
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -0,0 +1,1881 @@
+package org.apache.cassandra.io.sstable.format;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.cache.CachingOptions;
+import org.apache.cassandra.cache.InstrumentingCache;
+import org.apache.cassandra.cache.KeyCacheKey;
+import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
+import org.apache.cassandra.config.*;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.commitlog.ReplayPosition;
+import org.apache.cassandra.db.compaction.ICompactionScanner;
+import org.apache.cassandra.db.composites.CellName;
+import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.db.index.SecondaryIndex;
+import org.apache.cassandra.dht.*;
+import org.apache.cassandra.io.compress.CompressedRandomAccessReader;
+import org.apache.cassandra.io.compress.CompressedThrottledReader;
+import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.metadata.*;
+import org.apache.cassandra.io.util.*;
+import org.apache.cassandra.metrics.RestorableMeter;
+import org.apache.cassandra.metrics.StorageMetrics;
+import org.apache.cassandra.service.ActiveRepairService;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
+
+
+/**
+ * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
+ * Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
+ */
+public abstract class SSTableReader extends SSTable
+{
+ private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
+
+ private static final ScheduledThreadPoolExecutor syncExecutor = new ScheduledThreadPoolExecutor(1);
+ private static final RateLimiter meterSyncThrottle = RateLimiter.create(100.0);
+
+ public static final Comparator<SSTableReader> maxTimestampComparator = new Comparator<SSTableReader>()
+ {
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ long ts1 = o1.getMaxTimestamp();
+ long ts2 = o2.getMaxTimestamp();
+ return (ts1 > ts2 ? -1 : (ts1 == ts2 ? 0 : 1));
+ }
+ };
+
+ public static final Comparator<SSTableReader> sstableComparator = new Comparator<SSTableReader>()
+ {
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ return o1.first.compareTo(o2.first);
+ }
+ };
+
+ public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
+
+ /**
+ * maxDataAge is a timestamp in local server time (e.g. System.currentTimeMilli) which represents an upper bound
+ * to the newest piece of data stored in the sstable. In other words, this sstable does not contain items created
+ * later than maxDataAge.
+ *
+ * The field is not serialized to disk, so relying on it for more than what truncate does is not advised.
+ *
+ * When a new sstable is flushed, maxDataAge is set to the time of creation.
+ * When a sstable is created from compaction, maxDataAge is set to max of all merged sstables.
+ *
+ * The age is in milliseconds since epoc and is local to this host.
+ */
+ public final long maxDataAge;
+
+ public enum OpenReason
+ {
+ NORMAL,
+ EARLY,
+ METADATA_CHANGE
+ }
+
+ public final OpenReason openReason;
+
+ // indexfile and datafile: might be null before a call to load()
+ protected SegmentedFile ifile;
+ protected SegmentedFile dfile;
+
+ protected IndexSummary indexSummary;
+ protected IFilter bf;
+
+ protected final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+
+ protected InstrumentingCache<KeyCacheKey, RowIndexEntry> keyCache;
+
+ protected final BloomFilterTracker bloomFilterTracker = new BloomFilterTracker();
+
+ protected final AtomicInteger references = new AtomicInteger(1);
+ // technically isCompacted is not necessary since it should never be unreferenced unless it is also compacted,
+ // but it seems like a good extra layer of protection against reference counting bugs to not delete data based on that alone
+ protected final AtomicBoolean isCompacted = new AtomicBoolean(false);
+ protected final AtomicBoolean isSuspect = new AtomicBoolean(false);
+
+ // not final since we need to be able to change level on a file.
+ protected volatile StatsMetadata sstableMetadata;
+
+ protected final AtomicLong keyCacheHit = new AtomicLong(0);
+ protected final AtomicLong keyCacheRequest = new AtomicLong(0);
+
+ /**
+ * To support replacing this sstablereader with another object that represents that same underlying sstable, but with different associated resources,
+ * we build a linked-list chain of replacement, which we synchronise using a shared object to make maintenance of the list across multiple threads simple.
+ * On close we check if any of the closeable resources differ between any chains either side of us; any that are in neither of the adjacent links (if any) are closed.
+ * Once we've made this decision we remove ourselves from the linked list, so that anybody behind/ahead will compare against only other still opened resources.
+ */
+ protected Object replaceLock = new Object();
+ protected SSTableReader replacedBy;
+ private SSTableReader replaces;
+ private SSTableDeletingTask deletingTask;
+ private Runnable runOnClose;
+
+ @VisibleForTesting
+ public RestorableMeter readMeter;
+ protected ScheduledFuture readMeterSyncFuture;
+
+ /**
+ * Calculate approximate key count.
+ * If cardinality estimator is available on all given sstables, then this method use them to estimate
+ * key count.
+ * If not, then this uses index summaries.
+ *
+ * @param sstables SSTables to calculate key count
+ * @return estimated key count
+ */
+ public static long getApproximateKeyCount(Collection<SSTableReader> sstables)
+ {
+ long count = -1;
+
+ // check if cardinality estimator is available for all SSTables
+ boolean cardinalityAvailable = !sstables.isEmpty() && Iterators.all(sstables.iterator(), new Predicate<SSTableReader>()
+ {
+ public boolean apply(SSTableReader sstable)
+ {
+ return sstable.descriptor.version.hasNewStatsFile();
+ }
+ });
+
+ // if it is, load them to estimate key count
+ if (cardinalityAvailable)
+ {
+ boolean failed = false;
+ ICardinality cardinality = null;
+ for (SSTableReader sstable : sstables)
+ {
+ try
+ {
+ CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
+ if (cardinality == null)
+ cardinality = metadata.cardinalityEstimator;
+ else
+ cardinality = cardinality.merge(metadata.cardinalityEstimator);
+ }
+ catch (IOException e)
+ {
+ logger.warn("Reading cardinality from Statistics.db failed.", e);
+ failed = true;
+ break;
+ }
+ catch (CardinalityMergeException e)
+ {
+ logger.warn("Cardinality merge failed.", e);
+ failed = true;
+ break;
+ }
+ }
+ if (cardinality != null && !failed)
+ count = cardinality.cardinality();
+ }
+
+ // if something went wrong above or cardinality is not available, calculate using index summary
+ if (count < 0)
+ {
+ for (SSTableReader sstable : sstables)
+ count += sstable.estimatedKeys();
+ }
+ return count;
+ }
+
+ public static SSTableReader open(Descriptor descriptor) throws IOException
+ {
+ CFMetaData metadata;
+ if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
+ {
+ int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
+ String parentName = descriptor.cfname.substring(0, i);
+ CFMetaData parent = Schema.instance.getCFMetaData(descriptor.ksname, parentName);
+ ColumnDefinition def = parent.getColumnDefinitionForIndex(descriptor.cfname.substring(i + 1));
+ metadata = CFMetaData.newIndexMetadata(parent, def, SecondaryIndex.getIndexComparator(parent, def));
+ }
+ else
+ {
+ metadata = Schema.instance.getCFMetaData(descriptor.ksname, descriptor.cfname);
+ }
+ return open(descriptor, metadata);
+ }
+
+ public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException
+ {
+ IPartitioner p = desc.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR)
+ ? new LocalPartitioner(metadata.getKeyValidator())
+ : StorageService.getPartitioner();
+ return open(desc, componentsFor(desc), metadata, p);
+ }
+
+ public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner) throws IOException
+ {
+ return open(descriptor, components, metadata, partitioner, true);
+ }
+
+ public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
+ {
+ return open(descriptor, components, metadata, StorageService.getPartitioner(), false);
+ }
+
+ /**
+ * Open SSTable reader to be used in batch mode(such as sstableloader).
+ *
+ * @param descriptor
+ * @param components
+ * @param metadata
+ * @param partitioner
+ * @return opened SSTableReader
+ * @throws IOException
+ */
+ public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner) throws IOException
+ {
+ // Minimum components without which we can't do anything
+ assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor;
+ assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
+
+ Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
+ EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS));
+ ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
+ StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
+
+ // Check if sstable is created using same partitioner.
+ // Partitioner can be null, which indicates older version of sstable or no stats available.
+ // In that case, we skip the check.
+ String partitionerName = partitioner.getClass().getCanonicalName();
+ if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
+ {
+ logger.error(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
+ descriptor, validationMetadata.partitioner, partitionerName));
+ System.exit(1);
+ }
+
+ logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(Component.DATA)).length());
+ SSTableReader sstable = internalOpen(descriptor, components, metadata, partitioner, System.currentTimeMillis(),
+ statsMetadata, OpenReason.NORMAL);
+
+ // special implementation of load to use non-pooled SegmentedFile builders
+ SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
+ SegmentedFile.Builder dbuilder = sstable.compression
+ ? new CompressedSegmentedFile.Builder(null)
+ : new BufferedSegmentedFile.Builder();
+ if (!sstable.loadSummary(ibuilder, dbuilder))
+ sstable.buildSummary(false, ibuilder, dbuilder, false, Downsampling.BASE_SAMPLING_LEVEL);
+ sstable.ifile = ibuilder.complete(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
+ sstable.dfile = dbuilder.complete(sstable.descriptor.filenameFor(Component.DATA));
+ sstable.bf = FilterFactory.AlwaysPresent;
+
+ return sstable;
+ }
+
+ private static SSTableReader open(Descriptor descriptor,
+ Set<Component> components,
+ CFMetaData metadata,
+ IPartitioner partitioner,
+ boolean validate) throws IOException
+ {
+ // Minimum components without which we can't do anything
+ assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor;
+ assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
+
+ Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
+ EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS));
+ ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
+ StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
+
+ // Check if sstable is created using same partitioner.
+ // Partitioner can be null, which indicates older version of sstable or no stats available.
+ // In that case, we skip the check.
+ String partitionerName = partitioner.getClass().getCanonicalName();
+ if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
+ {
+ logger.error(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
+ descriptor, validationMetadata.partitioner, partitionerName));
+ System.exit(1);
+ }
+
+ logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(Component.DATA)).length());
+ SSTableReader sstable = internalOpen(descriptor, components, metadata, partitioner, System.currentTimeMillis(),
+ statsMetadata, OpenReason.NORMAL);
+
+ // load index and filter
+ long start = System.nanoTime();
+ sstable.load(validationMetadata);
+ logger.debug("INDEX LOAD TIME for {}: {} ms.", descriptor, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
+
+ if (validate)
+ sstable.validate();
+
+ if (sstable.getKeyCache() != null)
+ logger.debug("key cache contains {}/{} keys", sstable.getKeyCache().size(), sstable.getKeyCache().getCapacity());
+
+ return sstable;
+ }
+
+ public static void logOpenException(Descriptor descriptor, IOException e)
+ {
+ if (e instanceof FileNotFoundException)
+ logger.error("Missing sstable component in {}; skipped because of {}", descriptor, e.getMessage());
+ else
+ logger.error("Corrupt sstable {}; skipped", descriptor, e);
+ }
+
+ public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries,
+ final CFMetaData metadata,
+ final IPartitioner partitioner)
+ {
+ final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>();
+
+ ExecutorService executor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("SSTableBatchOpen", FBUtilities.getAvailableProcessors());
+ for (final Map.Entry<Descriptor, Set<Component>> entry : entries)
+ {
+ Runnable runnable = new Runnable()
+ {
+ public void run()
+ {
+ SSTableReader sstable;
+ try
+ {
+ sstable = open(entry.getKey(), entry.getValue(), metadata, partitioner);
+ }
+ catch (IOException ex)
+ {
+ logger.error("Corrupt sstable {}; skipped", entry, ex);
+ return;
+ }
+ sstables.add(sstable);
+ }
+ };
+ executor.submit(runnable);
+ }
+
+ executor.shutdown();
+ try
+ {
+ executor.awaitTermination(7, TimeUnit.DAYS);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
+
+ return sstables;
+
+ }
+
+ /**
+ * Open a RowIndexedReader which already has its state initialized (by SSTableWriter).
+ */
+ public static SSTableReader internalOpen(Descriptor desc,
+ Set<Component> components,
+ CFMetaData metadata,
+ IPartitioner partitioner,
+ SegmentedFile ifile,
+ SegmentedFile dfile,
+ IndexSummary isummary,
+ IFilter bf,
+ long maxDataAge,
+ StatsMetadata sstableMetadata,
+ OpenReason openReason)
+ {
+ assert desc != null && partitioner != null && ifile != null && dfile != null && isummary != null && bf != null && sstableMetadata != null;
+
+ SSTableReader reader = internalOpen(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
+
+ reader.bf = bf;
+ reader.ifile = ifile;
+ reader.dfile = dfile;
+ reader.indexSummary = isummary;
+
+ return reader;
+ }
+
+
+ private static SSTableReader internalOpen(final Descriptor descriptor,
+ Set<Component> components,
+ CFMetaData metadata,
+ IPartitioner partitioner,
+ Long maxDataAge,
+ StatsMetadata sstableMetadata,
+ OpenReason openReason)
+ {
+ Factory readerFactory = descriptor.getFormat().getReaderFactory();
+
+ return readerFactory.open(descriptor, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
+ }
+
+ protected SSTableReader(final Descriptor desc,
+ Set<Component> components,
+ CFMetaData metadata,
+ IPartitioner partitioner,
+ long maxDataAge,
+ StatsMetadata sstableMetadata,
+ OpenReason openReason)
+ {
+ super(desc, components, metadata, partitioner);
+ this.sstableMetadata = sstableMetadata;
+ this.maxDataAge = maxDataAge;
+ this.openReason = openReason;
+
+ this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata);
+
+ deletingTask = new SSTableDeletingTask(this);
+
+ // Don't track read rates for tables in the system keyspace. Also don't track reads for special operations (like early open)
+ // this is to avoid overflowing the executor queue (see CASSANDRA-8066)
+ if (Keyspace.SYSTEM_KS.equals(desc.ksname) || openReason != OpenReason.NORMAL)
+ {
+ readMeter = null;
+ readMeterSyncFuture = null;
+ return;
+ }
+
+ readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
+ // sync the average read rate to system.sstable_activity every five minutes, starting one minute from now
+ readMeterSyncFuture = syncExecutor.scheduleAtFixedRate(new Runnable()
+ {
+ public void run()
+ {
+ if (!isCompacted.get())
+ {
+ meterSyncThrottle.acquire();
+ SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.generation, readMeter);
+ }
+ }
+ }, 1, 5, TimeUnit.MINUTES);
+ }
+
+ public static long getTotalBytes(Iterable<SSTableReader> sstables)
+ {
+ long sum = 0;
+ for (SSTableReader sstable : sstables)
+ {
+ sum += sstable.onDiskLength();
+ }
+ return sum;
+ }
+
+ private void tidy(boolean release)
+ {
+ if (readMeterSyncFuture != null)
+ readMeterSyncFuture.cancel(false);
+
+ if (references.get() != 0)
+ {
+ throw new IllegalStateException("SSTable is not fully released (" + references.get() + " references)");
+ }
+
+ synchronized (replaceLock)
+ {
+ boolean closeBf = true, closeSummary = true, closeFiles = true, deleteFiles = false;
+
+ if (replacedBy != null)
+ {
+ closeBf = replacedBy.bf != bf;
+ closeSummary = replacedBy.indexSummary != indexSummary;
+ closeFiles = replacedBy.dfile != dfile;
+ // if the replacement sstablereader uses a different path, clean up our paths
+ deleteFiles = !dfile.path.equals(replacedBy.dfile.path);
+ }
+
+ if (replaces != null)
+ {
+ closeBf &= replaces.bf != bf;
+ closeSummary &= replaces.indexSummary != indexSummary;
+ closeFiles &= replaces.dfile != dfile;
+ deleteFiles &= !dfile.path.equals(replaces.dfile.path);
+ }
+
+ boolean deleteAll = false;
+ if (release && isCompacted.get())
+ {
+ assert replacedBy == null;
+ if (replaces != null)
+ {
+ replaces.replacedBy = null;
+ replaces.deletingTask = deletingTask;
+ replaces.markObsolete();
+ }
+ else
+ {
+ deleteAll = true;
+ }
+ }
+ else
+ {
+ if (replaces != null)
+ replaces.replacedBy = replacedBy;
+ if (replacedBy != null)
+ replacedBy.replaces = replaces;
+ }
+
+ scheduleTidy(closeBf, closeSummary, closeFiles, deleteFiles, deleteAll);
+ }
+ }
+
+ private void scheduleTidy(final boolean closeBf, final boolean closeSummary, final boolean closeFiles, final boolean deleteFiles, final boolean deleteAll)
+ {
+ if (references.get() != 0)
+ throw new IllegalStateException("SSTable is not fully released (" + references.get() + " references)");
+
+ final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+ final OpOrder.Barrier barrier;
+ if (cfs != null)
+ {
+ barrier = cfs.readOrdering.newBarrier();
+ barrier.issue();
+ }
+ else
+ barrier = null;
+
+ StorageService.tasks.execute(new Runnable()
+ {
+ public void run()
+ {
+ if (barrier != null)
+ barrier.await();
+ if (closeBf)
+ bf.close();
+ if (closeSummary)
+ indexSummary.close();
+ if (closeFiles)
+ {
+ ifile.cleanup();
+ dfile.cleanup();
+ }
+ if (runOnClose != null)
+ runOnClose.run();
+ if (deleteAll)
+ {
+ /**
+ * Do the OS a favour and suggest (using fadvice call) that we
+ * don't want to see pages of this SSTable in memory anymore.
+ *
+ * NOTE: We can't use madvice in java because it requires the address of
+ * the mapping, so instead we always open a file and run fadvice(fd, 0, 0) on it
+ */
+ dropPageCache();
+ deletingTask.run();
+ }
+ else if (deleteFiles)
+ {
+ FileUtils.deleteWithConfirm(new File(dfile.path));
+ FileUtils.deleteWithConfirm(new File(ifile.path));
+ }
+ }
+ });
+ }
+
+ public boolean equals(Object that)
+ {
+ return that instanceof SSTableReader && ((SSTableReader) that).descriptor.equals(this.descriptor);
+ }
+
+ public int hashCode()
+ {
+ return this.descriptor.hashCode();
+ }
+
+ public String getFilename()
+ {
+ return dfile.path;
+ }
+
+ public String getIndexFilename()
+ {
+ return ifile.path;
+ }
+
+ public void setTrackedBy(DataTracker tracker)
+ {
+ deletingTask.setTracker(tracker);
+ // under normal operation we can do this at any time, but SSTR is also used outside C* proper,
+ // e.g. by BulkLoader, which does not initialize the cache. As a kludge, we set up the cache
+ // here when we know we're being wired into the rest of the server infrastructure.
+ keyCache = CacheService.instance.keyCache;
+ }
+
+ private void load(ValidationMetadata validation) throws IOException
+ {
+ if (metadata.getBloomFilterFpChance() == 1.0)
+ {
+ // bf is disabled.
+ load(false, true);
+ bf = FilterFactory.AlwaysPresent;
+ }
+ else if (!components.contains(Component.FILTER) || validation == null)
+ {
+ // bf is enabled, but filter component is missing.
+ load(true, true);
+ }
+ else if (validation.bloomFilterFPChance != metadata.getBloomFilterFpChance())
+ {
+ // bf fp chance in sstable metadata and it has changed since compaction.
+ load(true, true);
+ }
+ else
+ {
+ // bf is enabled and fp chance matches the currently configured value.
+ load(false, true);
+ loadBloomFilter();
+ }
+ }
+
+ /**
+ * Load bloom filter from Filter.db file.
+ *
+ * @throws IOException
+ */
+ private void loadBloomFilter() throws IOException
+ {
+ DataInputStream stream = null;
+ try
+ {
+ stream = new DataInputStream(new BufferedInputStream(new FileInputStream(descriptor.filenameFor(Component.FILTER))));
+ bf = FilterFactory.deserialize(stream, true);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(stream);
+ }
+ }
+
+ /**
+ * Loads ifile, dfile and indexSummary, and optionally recreates the bloom filter.
+ * @param saveSummaryIfCreated for bulk loading purposes, if the summary was absent and needed to be built, you can
+ * avoid persisting it to disk by setting this to false
+ */
+ private void load(boolean recreateBloomFilter, boolean saveSummaryIfCreated) throws IOException
+ {
+ SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
+ SegmentedFile.Builder dbuilder = compression
+ ? SegmentedFile.getCompressedBuilder()
+ : SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
+
+ boolean summaryLoaded = loadSummary(ibuilder, dbuilder);
+ if (recreateBloomFilter || !summaryLoaded)
+ buildSummary(recreateBloomFilter, ibuilder, dbuilder, summaryLoaded, Downsampling.BASE_SAMPLING_LEVEL);
+
+ ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX));
+ dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA));
+ if (saveSummaryIfCreated && (recreateBloomFilter || !summaryLoaded)) // save summary information to disk
+ saveSummary(ibuilder, dbuilder);
+ }
+
+ /**
+ * Build index summary(and optionally bloom filter) by reading through Index.db file.
+ *
+ * @param recreateBloomFilter true if recreate bloom filter
+ * @param ibuilder
+ * @param dbuilder
+ * @param summaryLoaded true if index summary is already loaded and not need to build again
+ * @throws IOException
+ */
+ private void buildSummary(boolean recreateBloomFilter, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded, int samplingLevel) throws IOException
+ {
+ // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
+ RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
+
+ try
+ {
+ long indexSize = primaryIndex.length();
+ long histogramCount = sstableMetadata.estimatedRowSize.count();
+ long estimatedKeys = histogramCount > 0 && !sstableMetadata.estimatedRowSize.isOverflowed()
+ ? histogramCount
+ : estimateRowsFromIndex(primaryIndex); // statistics is supposed to be optional
+
+ if (recreateBloomFilter)
+ bf = FilterFactory.getFilter(estimatedKeys, metadata.getBloomFilterFpChance(), true);
+
+ IndexSummaryBuilder summaryBuilder = null;
+ if (!summaryLoaded)
+ summaryBuilder = new IndexSummaryBuilder(estimatedKeys, metadata.getMinIndexInterval(), samplingLevel);
+
+ long indexPosition;
+ RowIndexEntry.IndexSerializer rowIndexSerializer = descriptor.getFormat().getIndexSerializer(metadata);
+
+ while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
+ {
+ ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
+ RowIndexEntry indexEntry = rowIndexSerializer.deserialize(primaryIndex, descriptor.version);
+ DecoratedKey decoratedKey = partitioner.decorateKey(key);
+ if (first == null)
+ first = decoratedKey;
+ last = decoratedKey;
+
+ if (recreateBloomFilter)
+ bf.add(decoratedKey.getKey());
+
+ // if summary was already read from disk we don't want to re-populate it using primary index
+ if (!summaryLoaded)
+ {
+ summaryBuilder.maybeAddEntry(decoratedKey, indexPosition);
+ ibuilder.addPotentialBoundary(indexPosition);
+ dbuilder.addPotentialBoundary(indexEntry.position);
+ }
+ }
+
+ if (!summaryLoaded)
+ indexSummary = summaryBuilder.build(partitioner);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(primaryIndex);
+ }
+
+ first = getMinimalKey(first);
+ last = getMinimalKey(last);
+ }
+
+ /**
+ * Load index summary from Summary.db file if it exists.
+ *
+ * if loaded index summary has different index interval from current value stored in schema,
+ * then Summary.db file will be deleted and this returns false to rebuild summary.
+ *
+ * @param ibuilder
+ * @param dbuilder
+ * @return true if index summary is loaded successfully from Summary.db file.
+ */
+ public boolean loadSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder)
+ {
+ File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
+ if (!summariesFile.exists())
+ return false;
+
+ DataInputStream iStream = null;
+ try
+ {
+ iStream = new DataInputStream(new FileInputStream(summariesFile));
+ indexSummary = IndexSummary.serializer.deserialize(iStream, partitioner, descriptor.version.hasSamplingLevel(), metadata.getMinIndexInterval(), metadata.getMaxIndexInterval());
+ first = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
+ last = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
+ ibuilder.deserializeBounds(iStream);
+ dbuilder.deserializeBounds(iStream);
+ }
+ catch (IOException e)
+ {
+ logger.debug("Cannot deserialize SSTable Summary File {}: {}", summariesFile.getPath(), e.getMessage());
+ // corrupted; delete it and fall back to creating a new summary
+ FileUtils.closeQuietly(iStream);
+ // delete it and fall back to creating a new summary
+ FileUtils.deleteWithConfirm(summariesFile);
+ return false;
+ }
+ finally
+ {
+ FileUtils.closeQuietly(iStream);
+ }
+
+ return true;
+ }
+
+ /**
+ * Save index summary to Summary.db file.
+ *
+ * @param ibuilder
+ * @param dbuilder
+ */
+ public void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder)
+ {
+ saveSummary(ibuilder, dbuilder, indexSummary);
+ }
+
+ private void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, IndexSummary summary)
+ {
+ File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
+ if (summariesFile.exists())
+ FileUtils.deleteWithConfirm(summariesFile);
+
+ DataOutputStreamAndChannel oStream = null;
+ try
+ {
+ oStream = new DataOutputStreamAndChannel(new FileOutputStream(summariesFile));
+ IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel());
+ ByteBufferUtil.writeWithLength(first.getKey(), oStream);
+ ByteBufferUtil.writeWithLength(last.getKey(), oStream);
+ ibuilder.serializeBounds(oStream);
+ dbuilder.serializeBounds(oStream);
+ }
+ catch (IOException e)
+ {
+ logger.debug("Cannot save SSTable Summary: ", e);
+
+ // corrupted hence delete it and let it load it now.
+ if (summariesFile.exists())
+ FileUtils.deleteWithConfirm(summariesFile);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(oStream);
+ }
+ }
+
+ public void setReplacedBy(SSTableReader replacement)
+ {
+ synchronized (replaceLock)
+ {
+ assert replacedBy == null;
+ replacedBy = replacement;
+ replacement.replaces = this;
+ replacement.replaceLock = replaceLock;
+ }
+ }
+
+ public SSTableReader cloneWithNewStart(DecoratedKey newStart, final Runnable runOnClose)
+ {
+ synchronized (replaceLock)
+ {
+ assert replacedBy == null;
+
+ if (newStart.compareTo(this.first) > 0)
+ {
+ if (newStart.compareTo(this.last) > 0)
+ {
+ this.runOnClose = new Runnable()
+ {
+ public void run()
+ {
+ CLibrary.trySkipCache(dfile.path, 0, 0);
+ CLibrary.trySkipCache(ifile.path, 0, 0);
+ runOnClose.run();
+ }
+ };
+ }
+ else
+ {
+ final long dataStart = getPosition(newStart, Operator.GE).position;
+ final long indexStart = getIndexScanPosition(newStart);
+ this.runOnClose = new Runnable()
+ {
+ public void run()
+ {
+ CLibrary.trySkipCache(dfile.path, 0, dataStart);
+ CLibrary.trySkipCache(ifile.path, 0, indexStart);
+ runOnClose.run();
+ }
+ };
+ }
+ }
+
+ SSTableReader replacement = internalOpen(descriptor, components, metadata, partitioner, ifile, dfile, indexSummary.readOnlyClone(), bf, maxDataAge, sstableMetadata,
+ openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
+ replacement.readMeterSyncFuture = this.readMeterSyncFuture;
+ replacement.readMeter = this.readMeter;
+ replacement.first = this.last.compareTo(newStart) > 0 ? newStart : this.last;
+ replacement.last = this.last;
+ setReplacedBy(replacement);
+ return replacement;
+ }
+ }
+
+ /**
+ * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will
+ * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have
+ * its DeletingTask removed, and have its periodic read-meter sync task cancelled.
+ * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader
+ * @return a new SSTableReader
+ * @throws IOException
+ */
+ public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
+ {
+ synchronized (replaceLock)
+ {
+ assert replacedBy == null;
+
+ int minIndexInterval = metadata.getMinIndexInterval();
+ int maxIndexInterval = metadata.getMaxIndexInterval();
+ double effectiveInterval = indexSummary.getEffectiveIndexInterval();
+
+ IndexSummary newSummary;
+ long oldSize = bytesOnDisk();
+
+ // We have to rebuild the summary from the on-disk primary index in three cases:
+ // 1. The sampling level went up, so we need to read more entries off disk
+ // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary
+ // at full sampling (and consequently at any other sampling level)
+ // 3. The max_index_interval was lowered, forcing us to raise the sampling level
+ if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval)
+ {
+ newSummary = buildSummaryAtLevel(samplingLevel);
+ }
+ else if (samplingLevel < indexSummary.getSamplingLevel())
+ {
+ // we can use the existing index summary to make a smaller one
+ newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, partitioner);
+
+ SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
+ SegmentedFile.Builder dbuilder = compression
+ ? SegmentedFile.getCompressedBuilder()
+ : SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
+ saveSummary(ibuilder, dbuilder, newSummary);
+ }
+ else
+ {
+ throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " +
+ "no adjustments to min/max_index_interval");
+ }
+
+ long newSize = bytesOnDisk();
+ StorageMetrics.load.inc(newSize - oldSize);
+ parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize);
+
+ SSTableReader replacement = internalOpen(descriptor, components, metadata, partitioner, ifile, dfile, newSummary, bf, maxDataAge, sstableMetadata,
+ openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
+ replacement.readMeterSyncFuture = this.readMeterSyncFuture;
+ replacement.readMeter = this.readMeter;
+ replacement.first = this.first;
+ replacement.last = this.last;
+ setReplacedBy(replacement);
+ return replacement;
+ }
+ }
+
+ private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
+ {
+ // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
+ RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
+ try
+ {
+ long indexSize = primaryIndex.length();
+ IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel);
+
+ long indexPosition;
+ while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
+ {
+ summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
+ RowIndexEntry.Serializer.skip(primaryIndex);
+ }
+
+ return summaryBuilder.build(partitioner);
+ }
+ finally
+ {
+ FileUtils.closeQuietly(primaryIndex);
+ }
+ }
+
+ public int getIndexSummarySamplingLevel()
+ {
+ return indexSummary.getSamplingLevel();
+ }
+
+ public long getIndexSummaryOffHeapSize()
+ {
+ return indexSummary.getOffHeapSize();
+ }
+
+ public int getMinIndexInterval()
+ {
+ return indexSummary.getMinIndexInterval();
+ }
+
+ public double getEffectiveIndexInterval()
+ {
+ return indexSummary.getEffectiveIndexInterval();
+ }
+
+ public void releaseSummary() throws IOException
+ {
+ indexSummary.close();
+ indexSummary = null;
+ }
+
+ private void validate()
+ {
+ if (this.first.compareTo(this.last) > 0)
+ throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
+ }
+
+ /**
+ * Gets the position in the index file to start scanning to find the given key (at most indexInterval keys away,
+ * modulo downsampling of the index summary).
+ */
+ public long getIndexScanPosition(RowPosition key)
+ {
+ return getIndexScanPositionFromBinarySearchResult(indexSummary.binarySearch(key), indexSummary);
+ }
+
+ protected static long getIndexScanPositionFromBinarySearchResult(int binarySearchResult, IndexSummary referencedIndexSummary)
+ {
+ if (binarySearchResult == -1)
+ return -1;
+ else
+ return referencedIndexSummary.getPosition(getIndexSummaryIndexFromBinarySearchResult(binarySearchResult));
+ }
+
+ protected static int getIndexSummaryIndexFromBinarySearchResult(int binarySearchResult)
+ {
+ if (binarySearchResult < 0)
+ {
+ // binary search gives us the first index _greater_ than the key searched for,
+ // i.e., its insertion position
+ int greaterThan = (binarySearchResult + 1) * -1;
+ if (greaterThan == 0)
+ return -1;
+ return greaterThan - 1;
+ }
+ else
+ {
+ return binarySearchResult;
+ }
+ }
+
+ /**
+ * Returns the compression metadata for this sstable.
+ * @throws IllegalStateException if the sstable is not compressed
+ */
+ public CompressionMetadata getCompressionMetadata()
+ {
+ if (!compression)
+ throw new IllegalStateException(this + " is not compressed");
+
+ CompressionMetadata cmd = ((ICompressedFile) dfile).getMetadata();
+
+ //We need the parent cf metadata
+ String cfName = metadata.isSecondaryIndex() ? metadata.getParentColumnFamilyName() : metadata.cfName;
+ cmd.parameters.setLiveMetadata(Schema.instance.getCFMetaData(metadata.ksName, cfName));
+
+ return cmd;
+ }
+
+ /**
+ * For testing purposes only.
+ */
+ public void forceFilterFailures()
+ {
+ bf = FilterFactory.AlwaysPresent;
+ }
+
+ public IFilter getBloomFilter()
+ {
+ return bf;
+ }
+
+ public long getBloomFilterSerializedSize()
+ {
+ return bf.serializedSize();
+ }
+
+ /**
+ * @return An estimate of the number of keys in this SSTable based on the index summary.
+ */
+ public long estimatedKeys()
+ {
+ return indexSummary.getEstimatedKeyCount();
+ }
+
+ /**
+ * @param ranges
+ * @return An estimate of the number of keys for given ranges in this SSTable.
+ */
+ public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
+ {
+ long sampleKeyCount = 0;
+ List<Pair<Integer, Integer>> sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges);
+ for (Pair<Integer, Integer> sampleIndexRange : sampleIndexes)
+ sampleKeyCount += (sampleIndexRange.right - sampleIndexRange.left + 1);
+
+ // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
+ long estimatedKeys = sampleKeyCount * (Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
+ return Math.max(1, estimatedKeys);
+ }
+
+ /**
+ * Returns the number of entries in the IndexSummary. At full sampling, this is approximately 1/INDEX_INTERVALth of
+ * the keys in this SSTable.
+ */
+ public int getIndexSummarySize()
+ {
+ return indexSummary.size();
+ }
+
+ /**
+ * Returns the approximate number of entries the IndexSummary would contain if it were at full sampling.
+ */
+ public int getMaxIndexSummarySize()
+ {
+ return indexSummary.getMaxNumberOfEntries();
+ }
+
+ /**
+ * Returns the key for the index summary entry at `index`.
+ */
+ public byte[] getIndexSummaryKey(int index)
+ {
+ return indexSummary.getKey(index);
+ }
+
+ private static List<Pair<Integer,Integer>> getSampleIndexesForRanges(IndexSummary summary, Collection<Range<Token>> ranges)
+ {
+ // use the index to determine a minimal section for each range
+ List<Pair<Integer,Integer>> positions = new ArrayList<>();
+
+ for (Range<Token> range : Range.normalize(ranges))
+ {
+ RowPosition leftPosition = range.left.maxKeyBound();
+ RowPosition rightPosition = range.right.maxKeyBound();
+
+ int left = summary.binarySearch(leftPosition);
+ if (left < 0)
+ left = (left + 1) * -1;
+ else
+ // left range are start exclusive
+ left = left + 1;
+ if (left == summary.size())
+ // left is past the end of the sampling
+ continue;
+
+ int right = Range.isWrapAround(range.left, range.right)
+ ? summary.size() - 1
+ : summary.binarySearch(rightPosition);
+ if (right < 0)
+ {
+ // range are end inclusive so we use the previous index from what binarySearch give us
+ // since that will be the last index we will return
+ right = (right + 1) * -1;
+ if (right == 0)
+ // Means the first key is already stricly greater that the right bound
+ continue;
+ right--;
+ }
+
+ if (left > right)
+ // empty range
+ continue;
+ positions.add(Pair.create(left, right));
+ }
+ return positions;
+ }
+
+ public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
+ {
+ final List<Pair<Integer, Integer>> indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range));
+
+ if (indexRanges.isEmpty())
+ return Collections.emptyList();
+
+ return new Iterable<DecoratedKey>()
+ {
+ public Iterator<DecoratedKey> iterator()
+ {
+ return new Iterator<DecoratedKey>()
+ {
+ private Iterator<Pair<Integer, Integer>> rangeIter = indexRanges.iterator();
+ private Pair<Integer, Integer> current;
+ private int idx;
+
+ public boolean hasNext()
+ {
+ if (current == null || idx > current.right)
+ {
+ if (rangeIter.hasNext())
+ {
+ current = rangeIter.next();
+ idx = current.left;
+ return true;
+ }
+ return false;
+ }
+
+ return true;
+ }
+
+ public DecoratedKey next()
+ {
+ byte[] bytes = indexSummary.getKey(idx++);
+ return partitioner.decorateKey(ByteBuffer.wrap(bytes));
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ }
+
+ /**
+ * Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges.
+ * @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable.
+ */
+ public List<Pair<Long,Long>> getPositionsForRanges(Collection<Range<Token>> ranges)
+ {
+ // use the index to determine a minimal section for each range
+ List<Pair<Long,Long>> positions = new ArrayList<>();
+ for (Range<Token> range : Range.normalize(ranges))
+ {
+ AbstractBounds<RowPosition> keyRange = range.toRowBounds();
+ RowIndexEntry idxLeft = getPosition(keyRange.left, Operator.GT);
+ long left = idxLeft == null ? -1 : idxLeft.position;
+ if (left == -1)
+ // left is past the end of the file
+ continue;
+ RowIndexEntry idxRight = getPosition(keyRange.right, Operator.GT);
+ long right = idxRight == null ? -1 : idxRight.position;
+ if (right == -1 || Range.isWrapAround(range.left, range.right))
+ // right is past the end of the file, or it wraps
+ right = uncompressedLength();
+ if (left == right)
+ // empty range
+ continue;
+ positions.add(Pair.create(left, right));
+ }
+ return positions;
+ }
+
+ public void invalidateCacheKey(DecoratedKey key)
+ {
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, key.getKey());
+ keyCache.remove(cacheKey);
+ }
+
+ public void cacheKey(DecoratedKey key, RowIndexEntry info)
+ {
+ CachingOptions caching = metadata.getCaching();
+
+ if (!caching.keyCache.isEnabled()
+ || keyCache == null
+ || keyCache.getCapacity() == 0)
+ {
+ return;
+ }
+
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, key.getKey());
+ logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
+ keyCache.put(cacheKey, info);
+ }
+
+ public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
+ {
+ return getCachedPosition(new KeyCacheKey(metadata.cfId, descriptor, key.getKey()), updateStats);
+ }
+
+ protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
+ {
+ if (keyCache != null && keyCache.getCapacity() > 0) {
+ if (updateStats)
+ {
+ RowIndexEntry cachedEntry = keyCache.get(unifiedKey);
+ keyCacheRequest.incrementAndGet();
+ if (cachedEntry != null)
+ keyCacheHit.incrementAndGet();
+ return cachedEntry;
+ }
+ else
+ {
+ return keyCache.getInternal(unifiedKey);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get position updating key cache and stats.
+ * @see #getPosition(org.apache.cassandra.db.RowPosition, SSTableReader.Operator, boolean)
+ */
+ public RowIndexEntry getPosition(RowPosition key, Operator op)
+ {
+ return getPosition(key, op, true);
+ }
+
+ /**
+ * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
+ * allow key selection by token bounds but only if op != * EQ
+ * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
+ * @param updateCacheAndStats true if updating stats and cache
+ * @return The index entry corresponding to the key, or null if the key is not present
+ */
+ public abstract RowIndexEntry getPosition(RowPosition key, Operator op, boolean updateCacheAndStats);
+
+ //Corresponds to a name column
+ public abstract OnDiskAtomIterator iterator(DecoratedKey key, SortedSet<CellName> columns);
+ public abstract OnDiskAtomIterator iterator(FileDataInput file, DecoratedKey key, SortedSet<CellName> columns, RowIndexEntry indexEntry);
+
+ //Corresponds to a slice query
+ public abstract OnDiskAtomIterator iterator(DecoratedKey key, ColumnSlice[] slices, boolean reverse);
+ public abstract OnDiskAtomIterator iterator(FileDataInput file, DecoratedKey key, ColumnSlice[] slices, boolean reversed, RowIndexEntry indexEntry);
+
+ /**
+ * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
+ */
+ public DecoratedKey firstKeyBeyond(RowPosition token)
+ {
+ long sampledPosition = getIndexScanPosition(token);
+ if (sampledPosition == -1)
+ sampledPosition = 0;
+
+ Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
+ while (segments.hasNext())
+ {
+ FileDataInput in = segments.next();
+ try
+ {
+ while (!in.isEOF())
+ {
+ ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
+ DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
+ if (indexDecoratedKey.compareTo(token) > 0)
+ return indexDecoratedKey;
+
+ RowIndexEntry.Serializer.skip(in);
+ }
+ }
+ catch (IOException e)
+ {
+ markSuspect();
+ throw new CorruptSSTableException(e, in.getPath());
+ }
+ finally
+ {
+ FileUtils.closeQuietly(in);
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * @return The length in bytes of the data for this SSTable. For
+ * compressed files, this is not the same thing as the on disk size (see
+ * onDiskLength())
+ */
+ public long uncompressedLength()
+ {
+ return dfile.length;
+ }
+
+ /**
+ * @return The length in bytes of the on disk size for this SSTable. For
+ * compressed files, this is not the same thing as the data length (see
+ * length())
+ */
+ public long onDiskLength()
+ {
+ return dfile.onDiskLength;
+ }
+
+ public boolean acquireReference()
+ {
+ while (true)
+ {
+ int n = references.get();
+ if (n <= 0)
+ return false;
+ if (references.compareAndSet(n, n + 1))
+ return true;
+ }
+ }
+
+ /**
+ * Release reference to this SSTableReader.
+ * If there is no one referring to this SSTable, and is marked as compacted,
+ * all resources are cleaned up and files are deleted eventually.
+ */
+ public void releaseReference()
+ {
+ if (references.decrementAndGet() == 0)
+ tidy(true);
+ assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
+ }
+
+ /**
+ * Mark the sstable as obsolete, i.e., compacted into newer sstables.
+ *
+ * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere
+ * except for threads holding a reference.
+ *
+ * @return true if the this is the first time the file was marked obsolete. Calling this
+ * multiple times is usually buggy (see exceptions in DataTracker.unmarkCompacting and removeOldSSTablesSize).
+ */
+ public boolean markObsolete()
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Marking {} compacted", getFilename());
+
+ synchronized (replaceLock)
+ {
+ assert replacedBy == null;
+ }
+ return !isCompacted.getAndSet(true);
+ }
+
+ public boolean isMarkedCompacted()
+ {
+ return isCompacted.get();
+ }
+
+ public void markSuspect()
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Marking {} as a suspect for blacklisting.", getFilename());
+
+ isSuspect.getAndSet(true);
+ }
+
+ public boolean isMarkedSuspect()
+ {
+ return isSuspect.get();
+ }
+
+
+ /**
+ * I/O SSTableScanner
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public ICompactionScanner getScanner()
+ {
+ return getScanner((RateLimiter) null);
+ }
+
+ public ICompactionScanner getScanner(RateLimiter limiter)
+ {
+ return getScanner(DataRange.allData(partitioner), limiter);
+ }
+
+ /**
+ *
+ * @param dataRange filter to use when reading the columns
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public ICompactionScanner getScanner(DataRange dataRange)
+ {
+ return getScanner(dataRange, null);
+ }
+
+ /**
+ * Direct I/O SSTableScanner over a defined range of tokens.
+ *
+ * @param range the range of keys to cover
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public ICompactionScanner getScanner(Range<Token> range, RateLimiter limiter)
+ {
+ if (range == null)
+ return getScanner(limiter);
+ return getScanner(Collections.singletonList(range), limiter);
+ }
+
+ /**
+ * Direct I/O SSTableScanner over a defined collection of ranges of tokens.
+ *
+ * @param ranges the range of keys to cover
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public abstract ICompactionScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter);
+
+ /**
+ *
+ * @param dataRange filter to use when reading the columns
+ * @return A Scanner for seeking over the rows of the SSTable.
+ */
+ public abstract ICompactionScanner getScanner(DataRange dataRange, RateLimiter limiter);
+
+
+
+ public FileDataInput getFileDataInput(long position)
+ {
+ return dfile.getSegment(position);
+ }
+
+ /**
+ * Tests if the sstable contains data newer than the given age param (in localhost currentMilli time).
+ * This works in conjunction with maxDataAge which is an upper bound on the create of data in this sstable.
+ * @param age The age to compare the maxDataAre of this sstable. Measured in millisec since epoc on this host
+ * @return True iff this sstable contains data that's newer than the given age parameter.
+ */
+ public boolean newSince(long age)
+ {
+ return maxDataAge > age;
+ }
+
+ public void createLinks(String snapshotDirectoryPath)
+ {
+ for (Component component : components)
+ {
+ File sourceFile = new File(descriptor.filenameFor(component));
+ File targetLink = new File(snapshotDirectoryPath, sourceFile.getName());
+ FileUtils.createHardLink(sourceFile, targetLink);
+ }
+ }
+
+ public boolean isRepaired()
+ {
+ return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
+ }
+
+ public SSTableReader getCurrentReplacement()
+ {
+ synchronized (replaceLock)
+ {
+ SSTableReader cur = this, next = replacedBy;
+ while (next != null)
+ {
+ cur = next;
+ next = next.replacedBy;
+ }
+ return cur;
+ }
+ }
+
+ /**
+ * TODO: Move someplace reusable
+ */
+ public abstract static class Operator
+ {
+ public static final Operator EQ = new Equals();
+ public static final Operator GE = new GreaterThanOrEqualTo();
+ public static final Operator GT = new GreaterThan();
+
+ /**
+ * @param comparison The result of a call to compare/compareTo, with the desired field on the rhs.
+ * @return less than 0 if the operator cannot match forward, 0 if it matches, greater than 0 if it might match forward.
+ */
+ public abstract int apply(int comparison);
+
+ final static class Equals extends Operator
+ {
+ public int apply(int comparison) { return -comparison; }
+ }
+
+ final static class GreaterThanOrEqualTo extends Operator
+ {
+ public int apply(int comparison) { return comparison >= 0 ? 0 : -comparison; }
+ }
+
+ final static class GreaterThan extends Operator
+ {
+ public int apply(int comparison) { return comparison > 0 ? 0 : 1; }
+ }
+ }
+
+ public long getBloomFilterFalsePositiveCount()
+ {
+ return bloomFilterTracker.getFalsePositiveCount();
+ }
+
+ public long getRecentBloomFilterFalsePositiveCount()
+ {
+ return bloomFilterTracker.getRecentFalsePositiveCount();
+ }
+
+ public long getBloomFilterTruePositiveCount()
+ {
+ return bloomFilterTracker.getTruePositiveCount();
+ }
+
+ public long getRecentBloomFilterTruePositiveCount()
+ {
+ return bloomFilterTracker.getRecentTruePositiveCount();
+ }
+
+ public InstrumentingCache<KeyCacheKey, RowIndexEntry> getKeyCache()
+ {
+ return keyCache;
+ }
+
+ public EstimatedHistogram getEstimatedRowSize()
+ {
+ return sstableMetadata.estimatedRowSize;
+ }
+
+ public EstimatedHistogram getEstimatedColumnCount()
+ {
+ return sstableMetadata.estimatedColumnCount;
+ }
+
+ public double getEstimatedDroppableTombstoneRatio(int gcBefore)
+ {
+ return sstableMetadata.getEstimatedDroppableTombstoneRatio(gcBefore);
+ }
+
+ public double getDroppableTombstonesBefore(int gcBefore)
+ {
+ return sstableMetadata.getDroppableTombstonesBefore(gcBefore);
+ }
+
+ public double getCompressionRatio()
+ {
+ return sstableMetadata.compressionRatio;
+ }
+
+ public ReplayPosition getReplayPosition()
+ {
+ return sstableMetadata.replayPosition;
+ }
+
+ public long getMinTimestamp()
+ {
+ return sstableMetadata.minTimestamp;
+ }
+
+ public long getMaxTimestamp()
+ {
+ return sstableMetadata.maxTimestamp;
+ }
+
+ public Set<Integer> getAncestors()
+ {
+ try
+ {
+ CompactionMetadata compactionMetadata = (CompactionMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.COMPACTION);
+ return compactionMetadata.ancestors;
+ }
+ catch (IOException e)
+ {
+ SSTableReader.logOpenException(descriptor, e);
+ return Collections.emptySet();
+ }
+ }
+
+ public int getSSTableLevel()
+ {
+ return sstableMetadata.sstableLevel;
+ }
+
+ /**
+ * Reloads the sstable metadata from disk.
+ *
+ * Called after level is changed on sstable, for example if the sstable is dropped to L0
+ *
+ * Might be possible to remove in future versions
+ *
+ * @throws IOException
+ */
+ public void reloadSSTableMetadata() throws IOException
+ {
+ this.sstableMetadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS);
+ }
+
+ public StatsMetadata getSSTableMetadata()
+ {
+ return sstableMetadata;
+ }
+
+ public RandomAccessReader openDataReader(RateLimiter limiter)
+ {
+ assert limiter != null;
+ return compression
+ ? CompressedThrottledReader.open(getFilename(), getCompressionMetadata(), limiter)
+ : ThrottledReader.open(new File(getFilename()), limiter);
+ }
+
+ public RandomAccessReader openDataReader()
+ {
+ return compression
+ ? CompressedRandomAccessReader.open(getFilename(), getCompressionMetadata())
+ : RandomAccessReader.open(new File(getFilename()));
+ }
+
+ public RandomAccessReader openIndexReader()
+ {
+ return RandomAccessReader.open(new File(getIndexFilename()));
+ }
+
+ /**
+ * @param component component to get timestamp.
+ * @return last modified time for given component. 0 if given component does not exist or IO error occurs.
+ */
+ public long getCreationTimeFor(Component component)
+ {
+ return new File(descriptor.filenameFor(component)).lastModified();
+ }
+
+ /**
+ * @return Number of key cache hit
+ */
+ public long getKeyCacheHit()
+ {
+ return keyCacheHit.get();
+ }
+
+ /**
+ * @return Number of key cache request
+ */
+ public long getKeyCacheRequest()
+ {
+ return keyCacheRequest.get();
+ }
+
+ /**
+ * @param sstables
+ * @return true if all desired references were acquired. Otherwise, it will unreference any partial acquisition, and return false.
+ */
+ public static boolean acquireReferences(Iterable<SSTableReader> sstables)
+ {
+ SSTableReader failed = null;
+ for (SSTableReader sstable : sstables)
+ {
+ if (!sstable.acquireReference())
+ {
+ failed = sstable;
+ break;
+ }
+ }
+
+ if (failed == null)
+ return true;
+
+ for (SSTableReader sstable : sstables)
+ {
+ if (sstable == failed)
+ break;
+ sstable.releaseReference();
+ }
+ return false;
+ }
+
+ public static void releaseReferences(Iterable<SSTableReader> sstables)
+ {
+ for (SSTableReader sstable : sstables)
+ {
+ sstable.releaseReference();
+ }
+ }
+
+ private void dropPageCache()
+ {
+ dropPageCache(dfile.path);
+ dropPageCache(ifile.path);
+ }
+
+ private void dropPageCache(String filePath)
+ {
+ RandomAccessFile file = null;
+
+ try
+ {
+ file = new RandomAccessFile(filePath, "r");
+
+ int fd = CLibrary.getfd(file.getFD());
+
+ if (fd > 0)
+ {
+ if (logger.isDebugEnabled())
+ logger.debug(String.format("Dropping page cache of file %s.", filePath));
+
+ CLibrary.trySkipCache(fd, 0, 0);
+ }
+ }
+ catch (IOException e)
+ {
+ // we don't care if cache cleanup fails
+ }
+ finally
+ {
+ FileUtils.closeQuietly(file);
+ }
+ }
+
+ /**
+ * Increment the total row read count and read rate for this SSTable. This should not be incremented for range
+ * slice queries, row cache hits, or non-query reads, like compaction.
+ */
+ public void incrementReadCount()
+ {
+ if (readMeter != null)
+ readMeter.mark();
+ }
+
+ protected class EmptyCompactionScanner implements ICompactionScanner
+ {
+ private final String filename;
+
+ public EmptyCompactionScanner(String filename)
+ {
+ this.filename = filename;
+ }
+
+ public long getLengthInBytes()
+ {
+ return 0;
+ }
+
+ public long getCurrentPosition()
+ {
+ return 0;
+ }
+
+ public String getBackingFiles()
+ {
+ return filename;
+ }
+
+ public boolean hasNext()
+ {
+ return false;
+ }
+
+ public OnDiskAtomIterator next()
+ {
+ return null;
+ }
+
+ public void close() throws IOException { }
+
+ public void remove() { }
+ }
+
+ public static class SizeComparator implements Comparator<SSTableReader>
+ {
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ return Longs.compare(o1.onDiskLength(), o2.onDiskLength());
+ }
+ }
+
+ public static abstract class Factory
+ {
+ public abstract SSTableReader open(final Descriptor descriptor,
+ Set<Component> components,
+ CFMetaData metadata,
+ IPartitioner partitioner,
+ Long maxDataAge,
+ StatsMetadata sstableMetadata,
+ OpenReason openReason);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
new file mode 100644
index 0000000..c017530
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable.format;
+
+import com.google.common.collect.Sets;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.compaction.AbstractCompactedRow;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This is the API all table writers must implement.
+ *
+ * TableWriter.create() is the primary way to create a writer for a particular format.
+ * The format information is part of the Descriptor.
+ */
+public abstract class SSTableWriter extends SSTable
+{
+ private static final Logger logger = LoggerFactory.getLogger(SSTableWriter.class);
+
+ protected final long repairedAt;
+ protected final long keyCount;
+ protected final MetadataCollector metadataCollector;
+ protected final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
+
+ protected SSTableWriter(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector)
+ {
+ super(descriptor, components(metadata), metadata, partitioner);
+ this.keyCount = keyCount;
+ this.repairedAt = repairedAt;
+ this.metadataCollector = metadataCollector;
+ this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata);
+ }
+
+ public static SSTableWriter create(Descriptor descriptor, Long keyCount, Long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector)
+ {
+ Factory writerFactory = descriptor.getFormat().getWriterFactory();
+ return writerFactory.open(descriptor, keyCount, repairedAt, metadata, partitioner, metadataCollector);
+ }
+
+ public static SSTableWriter create(Descriptor descriptor, long keyCount, long repairedAt)
+ {
+ return create(descriptor, keyCount, repairedAt, 0);
+ }
+
+ public static SSTableWriter create(Descriptor descriptor, long keyCount, long repairedAt, int sstableLevel)
+ {
+ CFMetaData metadata = Schema.instance.getCFMetaData(descriptor);
+ MetadataCollector collector = new MetadataCollector(metadata.comparator).sstableLevel(sstableLevel);
+
+ return create(descriptor, keyCount, repairedAt, metadata, DatabaseDescriptor.getPartitioner(), collector);
+ }
+
+ public static SSTableWriter create(String filename, long keyCount, long repairedAt, int sstableLevel)
+ {
+ return create(Descriptor.fromFilename(filename), keyCount, repairedAt, sstableLevel);
+ }
+
+ public static SSTableWriter create(String filename, long keyCount, long repairedAt)
+ {
+ return create(Descriptor.fromFilename(filename), keyCount, repairedAt, 0);
+ }
+
+ private static Set<Component> components(CFMetaData metadata)
+ {
+ Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA,
+ Component.PRIMARY_INDEX,
+ Component.STATS,
+ Component.SUMMARY,
+ Component.TOC,
+ Component.DIGEST));
+
+ if (metadata.getBloomFilterFpChance() < 1.0)
+ components.add(Component.FILTER);
+
+ if (metadata.compressionParameters().sstableCompressor != null)
+ {
+ components.add(Component.COMPRESSION_INFO);
+ }
+ else
+ {
+ // it would feel safer to actually add this component later in maybeWriteDigest(),
+ // but the components are unmodifiable after construction
+ components.add(Component.CRC);
+ }
+ return components;
+ }
+
+
+ public abstract void mark();
+
+
+ /**
+ * @param row
+ * @return null if the row was compacted away entirely; otherwise, the PK index entry for this row
+ */
+ public abstract RowIndexEntry append(AbstractCompactedRow row);
+
+ public abstract void append(DecoratedKey decoratedKey, ColumnFamily cf);
+
+ public abstract long appendFromStream(DecoratedKey key, CFMetaData metadata, DataInput in, Version version) throws IOException;
+
+ public abstract long getFilePointer();
+
+ public abstract long getOnDiskFilePointer();
+
+ public abstract void isolateReferences();
+
+ public abstract void resetAndTruncate();
+
+ public SSTableReader closeAndOpenReader()
+ {
+ return closeAndOpenReader(System.currentTimeMillis());
+ }
+
+ public SSTableReader closeAndOpenReader(long maxDataAge)
+ {
+ return closeAndOpenReader(maxDataAge, repairedAt);
+ }
+
+ public abstract SSTableReader closeAndOpenReader(long maxDataAge, long repairedAt);
+
+ public abstract SSTableReader openEarly(long maxDataAge);
+
+ // Close the writer and return the descriptor to the new sstable and it's metadata
+ public abstract Pair<Descriptor, StatsMetadata> close();
+
+
+
+ public static Descriptor rename(Descriptor tmpdesc, Set<Component> components)
+ {
+ Descriptor newdesc = tmpdesc.asType(Descriptor.Type.FINAL);
+ rename(tmpdesc, newdesc, components);
+ return newdesc;
+ }
+
+ public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set<Component> components)
+ {
+ for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY)))
+ {
+ FileUtils.renameWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component));
+ }
+
+ // do -Data last because -Data present should mean the sstable was completely renamed before crash
+ FileUtils.renameWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA));
+
+ // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader
+ FileUtils.renameWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY));
+ }
+
+
+ /**
+ * After failure, attempt to close the index writer and data file before deleting all temp components for the sstable
+ */
+ public void abort()
+ {
+ abort(true);
+ }
+
+ public abstract void abort(boolean closeBf);
+
+
+ public static abstract class Factory
+ {
+ public abstract SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, CFMetaData metadata, IPartitioner partitioner, MetadataCollector metadataCollector);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
new file mode 100644
index 0000000..5da0cb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format;
+
+/**
+ * A set of feature flags associated with a SSTable format
+ *
+ * versions are denoted as [major][minor]. Minor versions must be forward-compatible:
+ * new fields are allowed in e.g. the metadata component, but fields can't be removed
+ * or have their size changed.
+ *
+ * Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that,
+ * we always incremented the major version.
+ *
+ */
+public abstract class Version
+{
+ protected final String version;
+ protected final SSTableFormat format;
+ protected Version(SSTableFormat format, String version)
+ {
+ this.format = format;
+ this.version = version;
+ }
+
+ public abstract boolean isLatestVersion();
+
+ public abstract boolean hasPostCompressionAdlerChecksums();
+
+ public abstract boolean hasSamplingLevel();
+
+ public abstract boolean hasNewStatsFile();
+
+ public abstract boolean hasAllAdlerChecksums();
+
+ public abstract boolean hasRepairedAt();
+
+ public abstract boolean tracksLegacyCounterShards();
+
+ public abstract boolean hasNewFileName();
+
+ public String getVersion()
+ {
+ return version;
+ }
+
+ public SSTableFormat getSSTableFormat()
+ {
+ return format;
+ }
+
+ /**
+ * @param ver SSTable version
+ * @return True if the given version string matches the format.
+ * @see #version
+ */
+ public static boolean validate(String ver)
+ {
+ return ver != null && ver.matches("[a-z]+");
+ }
+
+ abstract public boolean isCompatible();
+
+ @Override
+ public String toString()
+ {
+ return version;
+ }
+
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Version version1 = (Version) o;
+
+ if (version != null ? !version.equals(version1.version) : version1.version != null) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return version != null ? version.hashCode() : 0;
+ }
+}
[2/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java
new file mode 100644
index 0000000..07d867d
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableSliceIterator.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.OnDiskAtom;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.db.filter.ColumnSlice;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileDataInput;
+
+/**
+ * A Cell Iterator over SSTable
+ */
+class SSTableSliceIterator implements OnDiskAtomIterator
+{
+ private final OnDiskAtomIterator reader;
+ private final DecoratedKey key;
+
+ public SSTableSliceIterator(SSTableReader sstable, DecoratedKey key, ColumnSlice[] slices, boolean reversed)
+ {
+ this.key = key;
+ RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
+ this.reader = indexEntry == null ? null : createReader(sstable, indexEntry, null, slices, reversed);
+ }
+
+ /**
+ * An iterator for a slice within an SSTable
+ * @param sstable Keyspace for the CFS we are reading from
+ * @param file Optional parameter that input is read from. If null is passed, this class creates an appropriate one automatically.
+ * If this class creates, it will close the underlying file when #close() is called.
+ * If a caller passes a non-null argument, this class will NOT close the underlying file when the iterator is closed (i.e. the caller is responsible for closing the file)
+ * In all cases the caller should explicitly #close() this iterator.
+ * @param key The key the requested slice resides under
+ * @param slices the column slices
+ * @param reversed Results are returned in reverse order iff reversed is true.
+ * @param indexEntry position of the row
+ */
+ public SSTableSliceIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, ColumnSlice[] slices, boolean reversed, RowIndexEntry indexEntry)
+ {
+ this.key = key;
+ reader = createReader(sstable, indexEntry, file, slices, reversed);
+ }
+
+ private static OnDiskAtomIterator createReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput file, ColumnSlice[] slices, boolean reversed)
+ {
+ return slices.length == 1 && slices[0].start.isEmpty() && !reversed
+ ? new SimpleSliceReader(sstable, indexEntry, file, slices[0].finish)
+ : new IndexedSliceReader(sstable, indexEntry, file, slices, reversed);
+ }
+
+ public DecoratedKey getKey()
+ {
+ return key;
+ }
+
+ public ColumnFamily getColumnFamily()
+ {
+ return reader == null ? null : reader.getColumnFamily();
+ }
+
+ public boolean hasNext()
+ {
+ return reader != null && reader.hasNext();
+ }
+
+ public OnDiskAtom next()
+ {
+ return reader.next();
+ }
+
+ public void remove()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ public void close() throws IOException
+ {
+ if (reader != null)
+ reader.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java
new file mode 100644
index 0000000..9fec303
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/SimpleSliceReader.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable.format.big;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import com.google.common.collect.AbstractIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.composites.CellNameType;
+import org.apache.cassandra.db.composites.Composite;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.util.FileDataInput;
+import org.apache.cassandra.tracing.Tracing;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+class SimpleSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
+{
+ private static final Logger logger = LoggerFactory.getLogger(SimpleSliceReader.class);
+
+ private final FileDataInput file;
+ private final boolean needsClosing;
+ private final Composite finishColumn;
+ private final CellNameType comparator;
+ private final ColumnFamily emptyColumnFamily;
+ private final Iterator<OnDiskAtom> atomIterator;
+
+ SimpleSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, Composite finishColumn)
+ {
+ Tracing.trace("Seeking to partition beginning in data file");
+ this.finishColumn = finishColumn;
+ this.comparator = sstable.metadata.comparator;
+ try
+ {
+ if (input == null)
+ {
+ this.file = sstable.getFileDataInput(indexEntry.position);
+ this.needsClosing = true;
+ }
+ else
+ {
+ this.file = input;
+ input.seek(indexEntry.position);
+ this.needsClosing = false;
+ }
+
+ // Skip key and data size
+ ByteBufferUtil.skipShortLength(file);
+
+ emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata);
+ emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file));
+ atomIterator = emptyColumnFamily.metadata().getOnDiskIterator(file, sstable.descriptor.version);
+ }
+ catch (IOException e)
+ {
+ sstable.markSuspect();
+ throw new CorruptSSTableException(e, sstable.getFilename());
+ }
+ }
+
+ protected OnDiskAtom computeNext()
+ {
+ if (!atomIterator.hasNext())
+ return endOfData();
+
+ OnDiskAtom column = atomIterator.next();
+ if (!finishColumn.isEmpty() && comparator.compare(column.name(), finishColumn) > 0)
+ return endOfData();
+
+ return column;
+ }
+
+ public ColumnFamily getColumnFamily()
+ {
+ return emptyColumnFamily;
+ }
+
+ public void close() throws IOException
+ {
+ if (needsClosing)
+ file.close();
+ }
+
+ public DecoratedKey getKey()
+ {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index f801dac..dd879c4 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -27,6 +27,7 @@ import com.clearspring.analytics.stream.cardinality.ICardinality;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -91,7 +92,7 @@ public class CompactionMetadata extends MetadataComponent
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
- public CompactionMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
+ public CompactionMetadata deserialize(Version version, DataInput in) throws IOException
{
int nbAncestors = in.readInt();
Set<Integer> ancestors = new HashSet<>(nbAncestors);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
index 49ae378..018d4a0 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataComponentSerializer.java
@@ -21,6 +21,7 @@ import java.io.DataInput;
import java.io.IOException;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -55,5 +56,5 @@ public interface IMetadataComponentSerializer<T extends MetadataComponent>
* @return Deserialized component
* @throws IOException
*/
- T deserialize(Descriptor.Version version, DataInput in) throws IOException;
+ T deserialize(Version version, DataInput in) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 7ba2895..152614d 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -37,7 +37,7 @@ import org.apache.cassandra.io.sstable.ColumnNameHelper;
import org.apache.cassandra.io.sstable.ColumnStats;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.MurmurHash;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index a557b88..a501518 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
@@ -235,7 +236,7 @@ public class StatsMetadata extends MetadataComponent
out.writeBoolean(component.hasLegacyCounterShards);
}
- public StatsMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
+ public StatsMetadata deserialize(Version version, DataInput in) throws IOException
{
EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
@@ -247,7 +248,7 @@ public class StatsMetadata extends MetadataComponent
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
long repairedAt = 0;
- if (version.hasRepairedAt)
+ if (version.hasRepairedAt())
repairedAt = in.readLong();
int colCount = in.readInt();
@@ -261,7 +262,7 @@ public class StatsMetadata extends MetadataComponent
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
boolean hasLegacyCounterShards = true;
- if (version.tracksLegacyCounterShards)
+ if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
return new StatsMetadata(rowSizes,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
index e00c55c..aed6820 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/ValidationMetadata.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
@@ -82,7 +83,7 @@ public class ValidationMetadata extends MetadataComponent
out.writeDouble(component.bloomFilterFPChance);
}
- public ValidationMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
+ public ValidationMetadata deserialize(Version version, DataInput in) throws IOException
{
return new ValidationMetadata(in.readUTF(), in.readDouble());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/AbstractDataInput.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/AbstractDataInput.java b/src/java/org/apache/cassandra/io/util/AbstractDataInput.java
index 2815260..588540d 100644
--- a/src/java/org/apache/cassandra/io/util/AbstractDataInput.java
+++ b/src/java/org/apache/cassandra/io/util/AbstractDataInput.java
@@ -21,9 +21,9 @@ import java.io.*;
public abstract class AbstractDataInput extends InputStream implements DataInput
{
- protected abstract void seek(long position) throws IOException;
- protected abstract long getPosition();
- protected abstract long getPositionLimit();
+ public abstract void seek(long position) throws IOException;
+ public abstract long getPosition();
+ public abstract long getPositionLimit();
public int skipBytes(int n) throws IOException
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
index 797b964..4140c95 100644
--- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
+++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
@@ -51,7 +51,7 @@ public class DataIntegrityMetadata
public ChecksumValidator(Descriptor descriptor) throws IOException
{
this.descriptor = descriptor;
- checksum = descriptor.version.hasAllAdlerChecksums ? new Adler32() : new PureJavaCrc32();
+ checksum = descriptor.version.hasAllAdlerChecksums() ? new Adler32() : new PureJavaCrc32();
reader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC)));
chunkSize = reader.readInt();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/FileDataInput.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileDataInput.java b/src/java/org/apache/cassandra/io/util/FileDataInput.java
index d94075c..55809ad 100644
--- a/src/java/org/apache/cassandra/io/util/FileDataInput.java
+++ b/src/java/org/apache/cassandra/io/util/FileDataInput.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.io.util;
import java.io.Closeable;
import java.io.DataInput;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
public interface FileDataInput extends DataInput, Closeable
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index e590918..3567465 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -17,13 +17,7 @@
*/
package org.apache.cassandra.io.util;
-import java.io.Closeable;
-import java.io.DataInput;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
+import java.io.*;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.file.AtomicMoveNotSupportedException;
@@ -442,4 +436,25 @@ public class FileUtils
}
return length;
}
+
+
+ public static void copyTo(DataInput in, OutputStream out, int length) throws IOException
+ {
+ byte[] buffer = new byte[64 * 1024];
+ int copiedBytes = 0;
+
+ while (copiedBytes + buffer.length < length)
+ {
+ in.readFully(buffer);
+ out.write(buffer);
+ copiedBytes += buffer.length;
+ }
+
+ if (copiedBytes < length)
+ {
+ int left = length - copiedBytes;
+ in.readFully(buffer, 0, left);
+ out.write(buffer, 0, left);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java b/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
index 0479256..574a7fb 100644
--- a/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
+++ b/src/java/org/apache/cassandra/io/util/MappedFileDataInput.java
@@ -65,12 +65,12 @@ public class MappedFileDataInput extends AbstractDataInput implements FileDataIn
return segmentOffset + position;
}
- protected long getPosition()
+ public long getPosition()
{
return segmentOffset + position;
}
- protected long getPositionLimit()
+ public long getPositionLimit()
{
return segmentOffset + buffer.capacity();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/util/MemoryInputStream.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java
index 73ccc1b..45261e0 100644
--- a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java
+++ b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java
@@ -41,17 +41,17 @@ public class MemoryInputStream extends AbstractDataInput implements DataInput
position += count;
}
- protected void seek(long pos)
+ public void seek(long pos)
{
position = (int) pos;
}
- protected long getPosition()
+ public long getPosition()
{
return position;
}
- protected long getPositionLimit()
+ public long getPositionLimit()
{
return mem.size();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index 8ab432e..4e6f856 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.utils.EstimatedHistogram;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/net/MessagingService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index de3c125..72d8d6e 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -74,7 +74,8 @@ public final class MessagingService implements MessagingServiceMBean
public static final int VERSION_12 = 6;
public static final int VERSION_20 = 7;
public static final int VERSION_21 = 8;
- public static final int current_version = VERSION_21;
+ public static final int VERSION_30 = 9;
+ public static final int current_version = VERSION_30;
public static final String FAILURE_CALLBACK_PARAM = "CAL_BAC";
public static final byte[] ONE_BYTE = new byte[1];
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
index e4aff96..15230ea 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.notifications;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
public class SSTableAddedNotification implements INotification
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/notifications/SSTableDeletingNotification.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/notifications/SSTableDeletingNotification.java b/src/java/org/apache/cassandra/notifications/SSTableDeletingNotification.java
index 8b0f597..dcaa3b5 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableDeletingNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableDeletingNotification.java
@@ -17,7 +17,7 @@
*/
package org.apache.cassandra.notifications;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
/**
* Fired right before removing an SSTable.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
index c36583c..7ca574b 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableListChangedNotification.java
@@ -19,9 +19,8 @@ package org.apache.cassandra.notifications;
import java.util.Collection;
-import org.apache.cassandra.io.sstable.SSTableReader;
-
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
public class SSTableListChangedNotification implements INotification
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/notifications/SSTableRepairStatusChanged.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/notifications/SSTableRepairStatusChanged.java b/src/java/org/apache/cassandra/notifications/SSTableRepairStatusChanged.java
index a473a43..d1398bc 100644
--- a/src/java/org/apache/cassandra/notifications/SSTableRepairStatusChanged.java
+++ b/src/java/org/apache/cassandra/notifications/SSTableRepairStatusChanged.java
@@ -20,7 +20,7 @@ package org.apache.cassandra.notifications;
import java.util.Collection;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
public class SSTableRepairStatusChanged implements INotification
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
index 04a27af..2ad8dc2 100644
--- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
+++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
@@ -24,6 +24,7 @@ import java.util.UUID;
import java.util.concurrent.Future;
import com.google.common.base.Predicate;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +36,6 @@ import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.MessageIn;
import org.apache.cassandra.net.MessageOut;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/service/ActiveRepairService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java
index 670aa0b..fa354e6 100644
--- a/src/java/org/apache/cassandra/service/ActiveRepairService.java
+++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java
@@ -29,6 +29,8 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +43,6 @@ import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.net.IAsyncCallbackWithFailure;
import org.apache.cassandra.net.MessageIn;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/service/CacheService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java
index 1b93c2c..2ffd954 100644
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@ -33,6 +33,7 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import com.google.common.util.concurrent.Futures;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,6 @@ import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@@ -472,7 +472,8 @@ public class CacheService implements CacheServiceMBean
out.writeInt(desc.generation);
out.writeBoolean(true);
CFMetaData cfm = Schema.instance.getCFMetaData(key.desc.ksname, key.desc.cfname);
- cfm.comparator.rowIndexEntrySerializer().serialize(entry, out);
+
+ key.desc.getFormat().getIndexSerializer(cfm).serialize(entry, out);
}
public Future<Pair<KeyCacheKey, RowIndexEntry>> deserialize(DataInputStream input, ColumnFamilyStore cfs) throws IOException
@@ -492,7 +493,7 @@ public class CacheService implements CacheServiceMBean
RowIndexEntry.Serializer.skipPromotedIndex(input);
return null;
}
- RowIndexEntry entry = reader.metadata.comparator.rowIndexEntrySerializer().deserialize(input, reader.descriptor.version);
+ RowIndexEntry entry = reader.descriptor.getFormat().getIndexSerializer(reader.metadata).deserialize(input, reader.descriptor.version);
return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata.cfId, reader.descriptor, key), entry));
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamLockfile.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamLockfile.java b/src/java/org/apache/cassandra/streaming/StreamLockfile.java
index d00842a..a0cf5fc 100644
--- a/src/java/org/apache/cassandra/streaming/StreamLockfile.java
+++ b/src/java/org/apache/cassandra/streaming/StreamLockfile.java
@@ -28,12 +28,12 @@ import java.util.List;
import java.util.UUID;
import com.google.common.base.Charsets;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.util.FileUtils;
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReader.java b/src/java/org/apache/cassandra/streaming/StreamReader.java
index 34cbf02..8c4efcd 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReader.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReader.java
@@ -17,16 +17,18 @@
*/
package org.apache.cassandra.streaming;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.Collection;
import java.util.UUID;
import com.google.common.base.Throwables;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.FileUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,15 +39,14 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.messages.FileMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.BytesReadTracker;
import org.apache.cassandra.utils.Pair;
+
/**
* StreamReader reads from stream and writes to SSTable.
*/
@@ -56,8 +57,9 @@ public class StreamReader
protected final long estimatedKeys;
protected final Collection<Pair<Long, Long>> sections;
protected final StreamSession session;
- protected final Descriptor.Version inputVersion;
+ protected final Version inputVersion;
protected final long repairedAt;
+ protected final SSTableFormat.Type format;
protected final int sstableLevel;
protected Descriptor desc;
@@ -68,8 +70,9 @@ public class StreamReader
this.cfId = header.cfId;
this.estimatedKeys = header.estimatedKeys;
this.sections = header.sections;
- this.inputVersion = new Descriptor.Version(header.version);
+ this.inputVersion = header.format.info.getVersion(header.version);
this.repairedAt = header.repairedAt;
+ this.format = header.format;
this.sstableLevel = header.sstableLevel;
}
@@ -91,7 +94,8 @@ public class StreamReader
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
- SSTableWriter writer = createWriter(cfs, totalSize, repairedAt);
+ SSTableWriter writer = createWriter(cfs, totalSize, repairedAt, format);
+
DataInputStream dis = new DataInputStream(new LZFInputStream(Channels.newInputStream(channel)));
BytesReadTracker in = new BytesReadTracker(dis);
try
@@ -99,12 +103,12 @@ public class StreamReader
while (in.getBytesRead() < totalSize)
{
writeRow(writer, in, cfs);
+
// TODO move this to BytesReadTracker
session.progress(desc, ProgressInfo.Direction.IN, in.getBytesRead(), totalSize);
}
return writer;
- }
- catch (Throwable e)
+ } catch (Throwable e)
{
writer.abort();
drain(dis, in.getBytesRead());
@@ -115,13 +119,14 @@ public class StreamReader
}
}
- protected SSTableWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt) throws IOException
+ protected SSTableWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, SSTableFormat.Type format) throws IOException
{
Directories.DataDirectory localDir = cfs.directories.getWriteableLocation();
if (localDir == null)
throw new IOException("Insufficient disk space to store " + totalSize + " bytes");
- desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir)));
- return new SSTableWriter(desc.filenameFor(Component.DATA), estimatedKeys, repairedAt, sstableLevel);
+ desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir), format));
+
+ return SSTableWriter.create(desc, estimatedKeys, repairedAt, sstableLevel);
}
protected void drain(InputStream dis, long bytesRead) throws IOException
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
index f26e439..a6db58c 100644
--- a/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
@@ -28,8 +28,10 @@ import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamSession.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java
index 48b88c4..8bb34a6 100644
--- a/src/java/org/apache/cassandra/streaming/StreamSession.java
+++ b/src/java/org/apache/cassandra/streaming/StreamSession.java
@@ -25,6 +25,7 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.collect.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +39,6 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.metrics.StreamingMetrics;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.messages.*;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
index 18058c1..a3dd10f 100644
--- a/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
+++ b/src/java/org/apache/cassandra/streaming/StreamTransferTask.java
@@ -23,7 +23,7 @@ import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.cassandra.concurrent.NamedThreadFactory;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.streaming.messages.OutgoingFileMessage;
import org.apache.cassandra.utils.Pair;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/StreamWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/StreamWriter.java b/src/java/org/apache/cassandra/streaming/StreamWriter.java
index 43bc26a..93903a7 100644
--- a/src/java/org/apache/cassandra/streaming/StreamWriter.java
+++ b/src/java/org/apache/cassandra/streaming/StreamWriter.java
@@ -27,7 +27,7 @@ import java.util.Collection;
import com.ning.compress.lzf.LZFOutputStream;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataIntegrityMetadata;
import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator;
import org.apache.cassandra.io.util.FileUtils;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
index fb2599f..0595e0c 100644
--- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
+++ b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
@@ -18,12 +18,15 @@
package org.apache.cassandra.streaming.compress;
import java.io.DataInputStream;
+
import java.io.IOException;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import com.google.common.base.Throwables;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,7 +34,6 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamReader;
import org.apache.cassandra.streaming.StreamSession;
@@ -72,21 +74,25 @@ public class CompressedStreamReader extends StreamReader
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
- SSTableWriter writer = createWriter(cfs, totalSize, repairedAt);
+ SSTableWriter writer = createWriter(cfs, totalSize, repairedAt, format);
- CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo, inputVersion.hasPostCompressionAdlerChecksums);
+ CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo, inputVersion.hasPostCompressionAdlerChecksums());
BytesReadTracker in = new BytesReadTracker(new DataInputStream(cis));
try
{
for (Pair<Long, Long> section : sections)
{
- long length = section.right - section.left;
+ assert in.getBytesRead() < totalSize;
+ int sectionLength = (int) (section.right - section.left);
+
// skip to beginning of section inside chunk
cis.position(section.left);
in.reset(0);
- while (in.getBytesRead() < length)
+
+ while (in.getBytesRead() < sectionLength)
{
writeRow(writer, in, cfs);
+
// when compressed, report total bytes of compressed chunks read since remoteFile.size is the sum of chunks transferred
session.progress(desc, ProgressInfo.Direction.IN, cis.getTotalCompressedBytesRead(), totalSize);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
index 001c927..786ff23 100644
--- a/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
+++ b/src/java/org/apache/cassandra/streaming/compress/CompressedStreamWriter.java
@@ -25,7 +25,7 @@ import java.util.Collection;
import java.util.List;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.streaming.ProgressInfo;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
index 5e378bc..5266e45 100644
--- a/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
+++ b/src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.compress.CompressionMetadata;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.streaming.compress.CompressionInfo;
@@ -43,6 +44,9 @@ public class FileMessageHeader
public final int sequenceNumber;
/** SSTable version */
public final String version;
+
+ /** SSTable format **/
+ public final SSTableFormat.Type format;
public final long estimatedKeys;
public final List<Pair<Long, Long>> sections;
public final CompressionInfo compressionInfo;
@@ -52,6 +56,7 @@ public class FileMessageHeader
public FileMessageHeader(UUID cfId,
int sequenceNumber,
String version,
+ SSTableFormat.Type format,
long estimatedKeys,
List<Pair<Long, Long>> sections,
CompressionInfo compressionInfo,
@@ -61,6 +66,7 @@ public class FileMessageHeader
this.cfId = cfId;
this.sequenceNumber = sequenceNumber;
this.version = version;
+ this.format = format;
this.estimatedKeys = estimatedKeys;
this.sections = sections;
this.compressionInfo = compressionInfo;
@@ -95,6 +101,7 @@ public class FileMessageHeader
sb.append("cfId: ").append(cfId);
sb.append(", #").append(sequenceNumber);
sb.append(", version: ").append(version);
+ sb.append(", format: ").append(format);
sb.append(", estimated keys: ").append(estimatedKeys);
sb.append(", transfer size: ").append(size());
sb.append(", compressed?: ").append(compressionInfo != null);
@@ -128,8 +135,15 @@ public class FileMessageHeader
UUIDSerializer.serializer.serialize(header.cfId, out, version);
out.writeInt(header.sequenceNumber);
out.writeUTF(header.version);
- out.writeLong(header.estimatedKeys);
+ //We can't stream to a node that doesn't understand a new sstable format
+ if (version < StreamMessage.VERSION_30 && header.format != SSTableFormat.Type.LEGACY && header.format != SSTableFormat.Type.BIG)
+ throw new UnsupportedOperationException("Can't stream non-legacy sstables to nodes < 3.0");
+
+ if (version >= StreamMessage.VERSION_30)
+ out.writeUTF(header.format.name);
+
+ out.writeLong(header.estimatedKeys);
out.writeInt(header.sections.size());
for (Pair<Long, Long> section : header.sections)
{
@@ -146,6 +160,11 @@ public class FileMessageHeader
UUID cfId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
int sequenceNumber = in.readInt();
String sstableVersion = in.readUTF();
+
+ SSTableFormat.Type format = SSTableFormat.Type.LEGACY;
+ if (version >= StreamMessage.VERSION_30)
+ format = SSTableFormat.Type.validate(in.readUTF());
+
long estimatedKeys = in.readLong();
int count = in.readInt();
List<Pair<Long, Long>> sections = new ArrayList<>(count);
@@ -154,7 +173,7 @@ public class FileMessageHeader
CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, MessagingService.current_version);
long repairedAt = in.readLong();
int sstableLevel = in.readInt();
- return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel);
+ return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel);
}
public long serializedSize(FileMessageHeader header, int version)
@@ -162,6 +181,10 @@ public class FileMessageHeader
long size = UUIDSerializer.serializer.serializedSize(header.cfId, version);
size += TypeSizes.NATIVE.sizeof(header.sequenceNumber);
size += TypeSizes.NATIVE.sizeof(header.version);
+
+ if (version >= StreamMessage.VERSION_30)
+ size += TypeSizes.NATIVE.sizeof(header.format.name);
+
size += TypeSizes.NATIVE.sizeof(header.estimatedKeys);
size += TypeSizes.NATIVE.sizeof(header.sections.size());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
index 8569b88..237fb70 100644
--- a/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/IncomingFileMessage.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
-import org.apache.cassandra.io.sstable.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
import org.apache.cassandra.streaming.StreamReader;
import org.apache.cassandra.streaming.StreamSession;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
index 13af987..ed4c4ce 100644
--- a/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingFileMessage.java
@@ -22,7 +22,7 @@ import java.nio.channels.ReadableByteChannel;
import java.util.List;
import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.StreamWriter;
@@ -74,6 +74,7 @@ public class OutgoingFileMessage extends StreamMessage
this.header = new FileMessageHeader(sstable.metadata.cfId,
sequenceNumber,
sstable.descriptor.version.toString(),
+ sstable.descriptor.formatType,
estimatedKeys,
sections,
compressionInfo,
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
index 372fdd3..20490db 100644
--- a/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
+++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
@@ -32,7 +32,9 @@ import org.apache.cassandra.streaming.StreamSession;
public abstract class StreamMessage
{
/** Streaming protocol version */
- public static final int CURRENT_VERSION = 3;
+ public static final int VERSION_20 = 2;
+ public static final int VERSION_30 = 3;
+ public static final int CURRENT_VERSION = VERSION_30;
public static void serialize(StreamMessage message, DataOutputStreamAndChannel out, int version, StreamSession session) throws IOException
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
index 1febbe8..1ab56ce 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExport.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.*;
+import org.apache.cassandra.db.compaction.ICompactionScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.CFMetaData;
@@ -115,7 +117,7 @@ public class SSTableExport
}
/**
- * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Format is
+ * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings. Type is
*
* human_readable_name, value, timestamp, [flag, [options]]
*
@@ -318,10 +320,10 @@ public class SSTableExport
Set<String> excludeSet = new HashSet<String>();
if (excludes != null)
- excludeSet = new HashSet<String>(Arrays.asList(excludes));
+ excludeSet = new HashSet<>(Arrays.asList(excludes));
SSTableIdentityIterator row;
- SSTableScanner scanner = reader.getScanner();
+ ICompactionScanner scanner = reader.getScanner();
try
{
outs.println("[");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/SSTableImport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableImport.java b/src/java/org/apache/cassandra/tools/SSTableImport.java
index 05b9dcb..fc4470e 100644
--- a/src/java/org/apache/cassandra/tools/SSTableImport.java
+++ b/src/java/org/apache/cassandra/tools/SSTableImport.java
@@ -26,6 +26,8 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.Option;
@@ -42,7 +44,6 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -302,7 +303,7 @@ public class SSTableImport
Object[] data = parser.readValueAs(new TypeReference<Object[]>(){});
keyCountToImport = (keyCountToImport == null) ? data.length : keyCountToImport;
- SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(ssTablePath), keyCountToImport, ActiveRepairService.UNREPAIRED_SSTABLE, 0);
System.out.printf("Importing %s keys...%n", keyCountToImport);
@@ -375,7 +376,7 @@ public class SSTableImport
System.out.printf("Importing %s keys...%n", keyCountToImport);
parser = getParser(jsonFile); // renewing parser
- SSTableWriter writer = new SSTableWriter(ssTablePath, keyCountToImport);
+ SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(ssTablePath), keyCountToImport, ActiveRepairService.UNREPAIRED_SSTABLE);
int lineNumber = 1;
DecoratedKey prevStoredKey = null;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
index b37d3b4..1f596ad 100644
--- a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
@@ -77,7 +77,7 @@ public class SSTableRepairedAtSetter
for (String fname: fileNames)
{
Descriptor descriptor = Descriptor.fromFilename(fname);
- if (descriptor.version.hasRepairedAt)
+ if (descriptor.version.hasRepairedAt())
{
if (setIsRepaired)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 4dffa74..2c92e60 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.util.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -66,7 +67,7 @@ public class StandaloneScrubber
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
- List<SSTableReader> sstables = new ArrayList<SSTableReader>();
+ List<SSTableReader> sstables = new ArrayList<>();
// Scrub sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index e078c3b..8c7e704 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.tools;
import java.io.File;
import java.util.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -109,7 +110,7 @@ public class StandaloneSplitter
String snapshotName = "pre-split-" + System.currentTimeMillis();
- List<SSTableReader> sstables = new ArrayList<SSTableReader>();
+ List<SSTableReader> sstables = new ArrayList<>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet())
{
try
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 1a8a89f..9aa02f2 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.tools;
import java.util.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;
@@ -61,7 +62,7 @@ public class StandaloneUpgrader
else
lister.includeBackups(false);
- Collection<SSTableReader> readers = new ArrayList<SSTableReader>();
+ Collection<SSTableReader> readers = new ArrayList<>();
// Upgrade sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
@@ -73,7 +74,7 @@ public class StandaloneUpgrader
try
{
SSTableReader sstable = SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata);
- if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT))
+ if (sstable.descriptor.version.equals(DatabaseDescriptor.getSSTableFormat().info.getLatestVersion()))
continue;
readers.add(sstable);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/utils/vint/EncodedDataInputStream.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/vint/EncodedDataInputStream.java b/src/java/org/apache/cassandra/utils/vint/EncodedDataInputStream.java
index 6385e5c..bee8ab0 100644
--- a/src/java/org/apache/cassandra/utils/vint/EncodedDataInputStream.java
+++ b/src/java/org/apache/cassandra/utils/vint/EncodedDataInputStream.java
@@ -47,17 +47,17 @@ public class EncodedDataInputStream extends AbstractDataInput implements DataInp
return input.readByte() & 0xFF;
}
- protected void seek(long position)
+ public void seek(long position)
{
throw new UnsupportedOperationException();
}
- protected long getPosition()
+ public long getPosition()
{
throw new UnsupportedOperationException();
}
- protected long getPositionLimit()
+ public long getPositionLimit()
{
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
index 7bc8ef5..28ec975 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -24,6 +24,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
@@ -32,7 +33,6 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.Util;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableUtils;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -92,7 +92,7 @@ public class LongCompactionsTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
- ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
+ ArrayList<SSTableReader> sstables = new ArrayList<>();
for (int k = 0; k < sstableCount; k++)
{
SortedMap<String,ColumnFamily> rows = new TreeMap<String,ColumnFamily>();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index 159b641..0530d84 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -29,7 +30,6 @@ import org.apache.cassandra.Util;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.FBUtilities;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/Util.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index fd53170..2d5e6fc 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -49,7 +49,7 @@ import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
index 3da4555..c1869b9 100644
--- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
+++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.cassandra.cache;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -29,7 +30,6 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/CleanupTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java
index 213c5b8..21de96f 100644
--- a/test/unit/org/apache/cassandra/db/CleanupTest.java
+++ b/test/unit/org/apache/cassandra/db/CleanupTest.java
@@ -39,7 +39,7 @@ import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.dht.BytesToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 386be01..be64a6f 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -43,6 +43,9 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.BeforeClass;
@@ -74,12 +77,6 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.IncludingExcludingBounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableDeletingTask;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableSimpleWriter;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.SimpleStrategy;
@@ -1780,12 +1777,12 @@ public class ColumnFamilyStoreTest
{
MetadataCollector collector = new MetadataCollector(cfmeta.comparator);
collector.addAncestor(sstable1.descriptor.generation); // add ancestor from previously written sstable
- return new SSTableWriter(makeFilename(directory, metadata.ksName, metadata.cfName),
- 0,
- ActiveRepairService.UNREPAIRED_SSTABLE,
- metadata,
- StorageService.getPartitioner(),
- collector);
+ return SSTableWriter.create(Descriptor.fromFilename(makeFilename(directory, metadata.ksName, metadata.cfName, DatabaseDescriptor.getSSTableFormat())),
+ 0L,
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ metadata,
+ DatabaseDescriptor.getPartitioner(),
+ collector);
}
};
writer.newRow(key);
@@ -1837,12 +1834,12 @@ public class ColumnFamilyStoreTest
for (int ancestor : ancestors)
collector.addAncestor(ancestor);
String file = new Descriptor(directory, ks, cf, 3, Descriptor.Type.TEMP).filenameFor(Component.DATA);
- return new SSTableWriter(file,
- 0,
- ActiveRepairService.UNREPAIRED_SSTABLE,
- metadata,
- StorageService.getPartitioner(),
- collector);
+ return SSTableWriter.create(Descriptor.fromFilename(file),
+ 0L,
+ ActiveRepairService.UNREPAIRED_SSTABLE,
+ metadata,
+ StorageService.getPartitioner(),
+ collector);
}
};
writer.newRow(key);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index ad3a6bc..629b414 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra.db;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -25,6 +24,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,7 +37,6 @@ import org.apache.cassandra.db.composites.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
@@ -179,7 +178,7 @@ public class KeyCacheTest
for (SSTableReader reader : readers)
reader.releaseReference();
- Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);;
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);
while (StorageService.tasks.getActiveCount() + StorageService.tasks.getQueue().size() > 0);
// after releasing the reference this should drop to 2
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/KeyspaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index 27a4e20..01038c4 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -26,6 +26,7 @@ import java.util.*;
import java.io.IOException;
import com.google.common.collect.Iterables;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.lang3.StringUtils;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,7 +50,6 @@ import static org.apache.cassandra.Util.column;
import static org.apache.cassandra.Util.expiringColumn;
import static org.apache.cassandra.Util.cellname;
import org.apache.cassandra.Util;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index 2a6b952..d2f63cc 100644
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ExecutionException;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -52,12 +53,10 @@ import org.apache.cassandra.db.index.SecondaryIndexSearcher;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.concurrent.OpOrder;
-import org.apache.cassandra.utils.memory.MemtableAllocator;
import static org.apache.cassandra.Util.dk;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
index 3282b0a..e880d95 100644
--- a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
+++ b/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java
@@ -28,6 +28,7 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.composites.CellNames;
import org.apache.cassandra.db.composites.SimpleDenseCellNameType;
import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.io.sstable.IndexHelper;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -39,10 +40,10 @@ public class RowIndexEntryTest extends SchemaLoader
@Test
public void testSerializedSize() throws IOException
{
- final RowIndexEntry simple = new RowIndexEntry(123);
+ final RowIndexEntry<IndexHelper.IndexInfo> simple = new RowIndexEntry<>(123);
DataOutputBuffer buffer = new DataOutputBuffer();
- RowIndexEntry.Serializer serializer = new RowIndexEntry.Serializer(new SimpleDenseCellNameType(UTF8Type.instance));
+ RowIndexEntry.Serializer serializer = new RowIndexEntry.Serializer(new IndexHelper.IndexInfo.Serializer(new SimpleDenseCellNameType(UTF8Type.instance)));
serializer.serialize(simple, buffer);
@@ -70,7 +71,7 @@ public class RowIndexEntryTest extends SchemaLoader
}}.build();
- RowIndexEntry withIndex = RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);
+ RowIndexEntry<IndexHelper.IndexInfo> withIndex = RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex);
serializer.serialize(withIndex, buffer);
Assert.assertEquals(buffer.getLength(), serializer.serializedSize(withIndex));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index 24a6787..814b498 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -37,6 +37,8 @@ import org.apache.cassandra.db.marshal.CounterColumnType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.commons.lang3.StringUtils;
@@ -56,7 +58,6 @@ import org.apache.cassandra.db.compaction.Scrubber;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.Util.cellname;
@@ -230,7 +231,7 @@ public class ScrubTest
assert root != null;
File rootDir = new File(root);
assert rootDir.isDirectory();
- Descriptor desc = new Descriptor(new Descriptor.Version("jb"), rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL);
+ Descriptor desc = new Descriptor("jb", rootDir, KEYSPACE, columnFamily, 1, Descriptor.Type.FINAL, SSTableFormat.Type.LEGACY);
CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
try
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
index cd5dc7f..e6e5d55 100644
--- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
@@ -29,6 +29,14 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutionException;
+import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.locator.SimpleStrategy;
+import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Test;
+
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.KSMetaData;
@@ -41,9 +49,6 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableScanner;
-import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -110,7 +115,7 @@ public class AntiCompactionTest
int nonRepairedKeys = 0;
for (SSTableReader sstable : store.getSSTables())
{
- SSTableScanner scanner = sstable.getScanner();
+ ICompactionScanner scanner = sstable.getScanner();
while (scanner.hasNext())
{
SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
@@ -187,7 +192,7 @@ public class AntiCompactionTest
int nonRepairedKeys = 0;
for (SSTableReader sstable : store.getSSTables())
{
- SSTableScanner scanner = sstable.getScanner();
+ ICompactionScanner scanner = sstable.getScanner();
while (scanner.hasNext())
{
SSTableIdentityIterator row = (SSTableIdentityIterator) scanner.next();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
index 85f5330..1116c9e 100644
--- a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
@@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -34,7 +35,6 @@ import org.apache.cassandra.Util;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
index 00923b1..5f87ff3 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java
@@ -28,6 +28,7 @@ import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.*;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.SimpleStrategy;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -35,7 +36,6 @@ import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.Util;
import static org.junit.Assert.assertEquals;
[6/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
deleted file mode 100644
index 1bc7a24..0000000
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ /dev/null
@@ -1,2055 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.io.sstable;
-
-import java.io.BufferedInputStream;
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Predicate;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Longs;
-import com.google.common.util.concurrent.RateLimiter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
-import com.clearspring.analytics.stream.cardinality.ICardinality;
-import org.apache.cassandra.cache.CachingOptions;
-import org.apache.cassandra.cache.InstrumentingCache;
-import org.apache.cassandra.cache.KeyCacheKey;
-import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Config;
-import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.DataRange;
-import org.apache.cassandra.db.DataTracker;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.RowPosition;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
-import org.apache.cassandra.db.commitlog.ReplayPosition;
-import org.apache.cassandra.db.compaction.ICompactionScanner;
-import org.apache.cassandra.db.index.SecondaryIndex;
-import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.dht.LocalPartitioner;
-import org.apache.cassandra.dht.Range;
-import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.io.compress.CompressedRandomAccessReader;
-import org.apache.cassandra.io.compress.CompressedThrottledReader;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.metadata.CompactionMetadata;
-import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
-import org.apache.cassandra.io.util.BufferedSegmentedFile;
-import org.apache.cassandra.io.util.CompressedSegmentedFile;
-import org.apache.cassandra.io.util.DataOutputStreamAndChannel;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.ICompressedFile;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SegmentedFile;
-import org.apache.cassandra.io.util.ThrottledReader;
-import org.apache.cassandra.metrics.RestorableMeter;
-import org.apache.cassandra.metrics.StorageMetrics;
-import org.apache.cassandra.service.ActiveRepairService;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.CLibrary;
-import org.apache.cassandra.utils.EstimatedHistogram;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.FilterFactory;
-import org.apache.cassandra.utils.IFilter;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.OpOrder;
-
-import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR;
-
-/**
- * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen.
- * Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead.
- */
-public class SSTableReader extends SSTable
-{
- private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class);
-
- private static final ScheduledThreadPoolExecutor syncExecutor = new ScheduledThreadPoolExecutor(1);
- private static final RateLimiter meterSyncThrottle = RateLimiter.create(100.0);
-
- public static final Comparator<SSTableReader> maxTimestampComparator = new Comparator<SSTableReader>()
- {
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- long ts1 = o1.getMaxTimestamp();
- long ts2 = o2.getMaxTimestamp();
- return (ts1 > ts2 ? -1 : (ts1 == ts2 ? 0 : 1));
- }
- };
-
- public static final Comparator<SSTableReader> sstableComparator = new Comparator<SSTableReader>()
- {
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- return o1.first.compareTo(o2.first);
- }
- };
-
- public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
-
- /**
- * maxDataAge is a timestamp in local server time (e.g. System.currentTimeMilli) which represents an upper bound
- * to the newest piece of data stored in the sstable. In other words, this sstable does not contain items created
- * later than maxDataAge.
- *
- * The field is not serialized to disk, so relying on it for more than what truncate does is not advised.
- *
- * When a new sstable is flushed, maxDataAge is set to the time of creation.
- * When a sstable is created from compaction, maxDataAge is set to max of all merged sstables.
- *
- * The age is in milliseconds since epoc and is local to this host.
- */
- public final long maxDataAge;
-
- public enum OpenReason
- {
- NORMAL,
- EARLY,
- METADATA_CHANGE
- }
-
- public final OpenReason openReason;
-
- // indexfile and datafile: might be null before a call to load()
- private SegmentedFile ifile;
- private SegmentedFile dfile;
-
- private IndexSummary indexSummary;
- private IFilter bf;
-
- private InstrumentingCache<KeyCacheKey, RowIndexEntry> keyCache;
-
- private final BloomFilterTracker bloomFilterTracker = new BloomFilterTracker();
-
- private final AtomicInteger references = new AtomicInteger(1);
- // technically isCompacted is not necessary since it should never be unreferenced unless it is also compacted,
- // but it seems like a good extra layer of protection against reference counting bugs to not delete data based on that alone
- private final AtomicBoolean isCompacted = new AtomicBoolean(false);
- private final AtomicBoolean isSuspect = new AtomicBoolean(false);
-
- // not final since we need to be able to change level on a file.
- private volatile StatsMetadata sstableMetadata;
-
- private final AtomicLong keyCacheHit = new AtomicLong(0);
- private final AtomicLong keyCacheRequest = new AtomicLong(0);
-
- /**
- * To support replacing this sstablereader with another object that represents that same underlying sstable, but with different associated resources,
- * we build a linked-list chain of replacement, which we synchronise using a shared object to make maintenance of the list across multiple threads simple.
- * On close we check if any of the closeable resources differ between any chains either side of us; any that are in neither of the adjacent links (if any) are closed.
- * Once we've made this decision we remove ourselves from the linked list, so that anybody behind/ahead will compare against only other still opened resources.
- */
- private Object replaceLock = new Object();
- private SSTableReader replacedBy;
- private SSTableReader replaces;
- private SSTableDeletingTask deletingTask;
- private Runnable runOnClose;
-
- @VisibleForTesting
- public RestorableMeter readMeter;
- private ScheduledFuture readMeterSyncFuture;
-
- /**
- * Calculate approximate key count.
- * If cardinality estimator is available on all given sstables, then this method use them to estimate
- * key count.
- * If not, then this uses index summaries.
- *
- * @param sstables SSTables to calculate key count
- * @return estimated key count
- */
- public static long getApproximateKeyCount(Collection<SSTableReader> sstables)
- {
- long count = -1;
-
- // check if cardinality estimator is available for all SSTables
- boolean cardinalityAvailable = !sstables.isEmpty() && Iterators.all(sstables.iterator(), new Predicate<SSTableReader>()
- {
- public boolean apply(SSTableReader sstable)
- {
- return sstable.descriptor.version.newStatsFile;
- }
- });
-
- // if it is, load them to estimate key count
- if (cardinalityAvailable)
- {
- boolean failed = false;
- ICardinality cardinality = null;
- for (SSTableReader sstable : sstables)
- {
- try
- {
- CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
- if (cardinality == null)
- cardinality = metadata.cardinalityEstimator;
- else
- cardinality = cardinality.merge(metadata.cardinalityEstimator);
- }
- catch (IOException e)
- {
- logger.warn("Reading cardinality from Statistics.db failed.", e);
- failed = true;
- break;
- }
- catch (CardinalityMergeException e)
- {
- logger.warn("Cardinality merge failed.", e);
- failed = true;
- break;
- }
- }
- if (cardinality != null && !failed)
- count = cardinality.cardinality();
- }
-
- // if something went wrong above or cardinality is not available, calculate using index summary
- if (count < 0)
- {
- for (SSTableReader sstable : sstables)
- count += sstable.estimatedKeys();
- }
- return count;
- }
-
- public static SSTableReader open(Descriptor descriptor) throws IOException
- {
- CFMetaData metadata;
- if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR))
- {
- int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
- String parentName = descriptor.cfname.substring(0, i);
- CFMetaData parent = Schema.instance.getCFMetaData(descriptor.ksname, parentName);
- ColumnDefinition def = parent.getColumnDefinitionForIndex(descriptor.cfname.substring(i + 1));
- metadata = CFMetaData.newIndexMetadata(parent, def, SecondaryIndex.getIndexComparator(parent, def));
- }
- else
- {
- metadata = Schema.instance.getCFMetaData(descriptor.ksname, descriptor.cfname);
- }
- return open(descriptor, metadata);
- }
-
- public static SSTableReader open(Descriptor desc, CFMetaData metadata) throws IOException
- {
- IPartitioner p = desc.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR)
- ? new LocalPartitioner(metadata.getKeyValidator())
- : StorageService.getPartitioner();
- return open(desc, componentsFor(desc), metadata, p);
- }
-
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner) throws IOException
- {
- return open(descriptor, components, metadata, partitioner, true);
- }
-
- public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
- {
- return open(descriptor, components, metadata, StorageService.getPartitioner(), false);
- }
-
- /**
- * Open SSTable reader to be used in batch mode(such as sstableloader).
- *
- * @param descriptor
- * @param components
- * @param metadata
- * @param partitioner
- * @return opened SSTableReader
- * @throws IOException
- */
- public static SSTableReader openForBatch(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner) throws IOException
- {
- // Minimum components without which we can't do anything
- assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor;
- assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
-
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
- EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS));
- ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
- StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
-
- // Check if sstable is created using same partitioner.
- // Partitioner can be null, which indicates older version of sstable or no stats available.
- // In that case, we skip the check.
- String partitionerName = partitioner.getClass().getCanonicalName();
- if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
- {
- logger.error(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
- descriptor, validationMetadata.partitioner, partitionerName));
- System.exit(1);
- }
-
- logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(Component.DATA)).length());
- SSTableReader sstable = new SSTableReader(descriptor,
- components,
- metadata,
- partitioner,
- System.currentTimeMillis(),
- statsMetadata,
- OpenReason.NORMAL);
-
- // special implementation of load to use non-pooled SegmentedFile builders
- SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
- SegmentedFile.Builder dbuilder = sstable.compression
- ? new CompressedSegmentedFile.Builder(null)
- : new BufferedSegmentedFile.Builder();
- if (!sstable.loadSummary(ibuilder, dbuilder))
- sstable.buildSummary(false, ibuilder, dbuilder, false, Downsampling.BASE_SAMPLING_LEVEL);
- sstable.ifile = ibuilder.complete(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX));
- sstable.dfile = dbuilder.complete(sstable.descriptor.filenameFor(Component.DATA));
- sstable.bf = FilterFactory.AlwaysPresent;
-
- return sstable;
- }
-
- private static SSTableReader open(Descriptor descriptor,
- Set<Component> components,
- CFMetaData metadata,
- IPartitioner partitioner,
- boolean validate) throws IOException
- {
- // Minimum components without which we can't do anything
- assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor;
- assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
-
- Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor,
- EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS));
- ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
- StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
-
- // Check if sstable is created using same partitioner.
- // Partitioner can be null, which indicates older version of sstable or no stats available.
- // In that case, we skip the check.
- String partitionerName = partitioner.getClass().getCanonicalName();
- if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner))
- {
- logger.error(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.",
- descriptor, validationMetadata.partitioner, partitionerName));
- System.exit(1);
- }
-
- logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(Component.DATA)).length());
- SSTableReader sstable = new SSTableReader(descriptor,
- components,
- metadata,
- partitioner,
- System.currentTimeMillis(),
- statsMetadata,
- OpenReason.NORMAL);
-
- // load index and filter
- long start = System.nanoTime();
- sstable.load(validationMetadata);
- logger.debug("INDEX LOAD TIME for {}: {} ms.", descriptor, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
-
- if (validate)
- sstable.validate();
-
- if (sstable.getKeyCache() != null)
- logger.debug("key cache contains {}/{} keys", sstable.getKeyCache().size(), sstable.getKeyCache().getCapacity());
-
- return sstable;
- }
-
- public static void logOpenException(Descriptor descriptor, IOException e)
- {
- if (e instanceof FileNotFoundException)
- logger.error("Missing sstable component in {}; skipped because of {}", descriptor, e.getMessage());
- else
- logger.error("Corrupt sstable {}; skipped", descriptor, e);
- }
-
- public static Collection<SSTableReader> openAll(Set<Map.Entry<Descriptor, Set<Component>>> entries,
- final CFMetaData metadata,
- final IPartitioner partitioner)
- {
- final Collection<SSTableReader> sstables = new LinkedBlockingQueue<>();
-
- ExecutorService executor = DebuggableThreadPoolExecutor.createWithFixedPoolSize("SSTableBatchOpen", FBUtilities.getAvailableProcessors());
- for (final Map.Entry<Descriptor, Set<Component>> entry : entries)
- {
- Runnable runnable = new Runnable()
- {
- public void run()
- {
- SSTableReader sstable;
- try
- {
- sstable = open(entry.getKey(), entry.getValue(), metadata, partitioner);
- }
- catch (IOException ex)
- {
- logger.error("Corrupt sstable {}; skipped", entry, ex);
- return;
- }
- sstables.add(sstable);
- }
- };
- executor.submit(runnable);
- }
-
- executor.shutdown();
- try
- {
- executor.awaitTermination(7, TimeUnit.DAYS);
- }
- catch (InterruptedException e)
- {
- throw new AssertionError(e);
- }
-
- return sstables;
-
- }
-
- /**
- * Open a RowIndexedReader which already has its state initialized (by SSTableWriter).
- */
- static SSTableReader internalOpen(Descriptor desc,
- Set<Component> components,
- CFMetaData metadata,
- IPartitioner partitioner,
- SegmentedFile ifile,
- SegmentedFile dfile,
- IndexSummary isummary,
- IFilter bf,
- long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason)
- {
- assert desc != null && partitioner != null && ifile != null && dfile != null && isummary != null && bf != null && sstableMetadata != null;
- return new SSTableReader(desc,
- components,
- metadata,
- partitioner,
- ifile, dfile,
- isummary,
- bf,
- maxDataAge,
- sstableMetadata,
- openReason);
- }
-
-
- private SSTableReader(final Descriptor desc,
- Set<Component> components,
- CFMetaData metadata,
- IPartitioner partitioner,
- long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason)
- {
- super(desc, components, metadata, partitioner);
- this.sstableMetadata = sstableMetadata;
- this.maxDataAge = maxDataAge;
- this.openReason = openReason;
-
- deletingTask = new SSTableDeletingTask(this);
-
- // Don't track read rates for tables in the system keyspace. Also don't track reads for special operations (like early open)
- // this is to avoid overflowing the executor queue (see CASSANDRA-8066)
- if (Keyspace.SYSTEM_KS.equals(desc.ksname) || openReason != OpenReason.NORMAL)
- {
- readMeter = null;
- readMeterSyncFuture = null;
- return;
- }
-
- readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
- // sync the average read rate to system.sstable_activity every five minutes, starting one minute from now
- readMeterSyncFuture = syncExecutor.scheduleAtFixedRate(new Runnable()
- {
- public void run()
- {
- if (!isCompacted.get())
- {
- meterSyncThrottle.acquire();
- SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.generation, readMeter);
- }
- }
- }, 1, 5, TimeUnit.MINUTES);
- }
-
- private SSTableReader(Descriptor desc,
- Set<Component> components,
- CFMetaData metadata,
- IPartitioner partitioner,
- SegmentedFile ifile,
- SegmentedFile dfile,
- IndexSummary indexSummary,
- IFilter bloomFilter,
- long maxDataAge,
- StatsMetadata sstableMetadata,
- OpenReason openReason)
- {
- this(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
-
- this.ifile = ifile;
- this.dfile = dfile;
- this.indexSummary = indexSummary;
- this.bf = bloomFilter;
- }
-
- public static long getTotalBytes(Iterable<SSTableReader> sstables)
- {
- long sum = 0;
- for (SSTableReader sstable : sstables)
- {
- sum += sstable.onDiskLength();
- }
- return sum;
- }
-
- private void tidy(boolean release)
- {
- if (readMeterSyncFuture != null)
- readMeterSyncFuture.cancel(false);
-
- if (references.get() != 0)
- {
- throw new IllegalStateException("SSTable is not fully released (" + references.get() + " references)");
- }
-
- synchronized (replaceLock)
- {
- boolean closeBf = true, closeSummary = true, closeFiles = true, deleteFiles = false;
-
- if (replacedBy != null)
- {
- closeBf = replacedBy.bf != bf;
- closeSummary = replacedBy.indexSummary != indexSummary;
- closeFiles = replacedBy.dfile != dfile;
- // if the replacement sstablereader uses a different path, clean up our paths
- deleteFiles = !dfile.path.equals(replacedBy.dfile.path);
- }
-
- if (replaces != null)
- {
- closeBf &= replaces.bf != bf;
- closeSummary &= replaces.indexSummary != indexSummary;
- closeFiles &= replaces.dfile != dfile;
- deleteFiles &= !dfile.path.equals(replaces.dfile.path);
- }
-
- boolean deleteAll = false;
- if (release && isCompacted.get())
- {
- assert replacedBy == null;
- if (replaces != null)
- {
- replaces.replacedBy = null;
- replaces.deletingTask = deletingTask;
- replaces.markObsolete();
- }
- else
- {
- deleteAll = true;
- }
- }
- else
- {
- if (replaces != null)
- replaces.replacedBy = replacedBy;
- if (replacedBy != null)
- replacedBy.replaces = replaces;
- }
-
- scheduleTidy(closeBf, closeSummary, closeFiles, deleteFiles, deleteAll);
- }
- }
-
- private void scheduleTidy(final boolean closeBf, final boolean closeSummary, final boolean closeFiles, final boolean deleteFiles, final boolean deleteAll)
- {
- if (references.get() != 0)
- throw new IllegalStateException("SSTable is not fully released (" + references.get() + " references)");
-
- final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
- final OpOrder.Barrier barrier;
- if (cfs != null)
- {
- barrier = cfs.readOrdering.newBarrier();
- barrier.issue();
- }
- else
- barrier = null;
-
- StorageService.tasks.execute(new Runnable()
- {
- public void run()
- {
- if (barrier != null)
- barrier.await();
- if (closeBf)
- bf.close();
- if (closeSummary)
- indexSummary.close();
- if (closeFiles)
- {
- ifile.cleanup();
- dfile.cleanup();
- }
- if (runOnClose != null)
- runOnClose.run();
- if (deleteAll)
- {
- /**
- * Do the OS a favour and suggest (using fadvice call) that we
- * don't want to see pages of this SSTable in memory anymore.
- *
- * NOTE: We can't use madvice in java because it requires the address of
- * the mapping, so instead we always open a file and run fadvice(fd, 0, 0) on it
- */
- dropPageCache();
- deletingTask.run();
- }
- else if (deleteFiles)
- {
- FileUtils.deleteWithConfirm(new File(dfile.path));
- FileUtils.deleteWithConfirm(new File(ifile.path));
- }
- }
- });
- }
-
- public boolean equals(Object that)
- {
- return that instanceof SSTableReader && ((SSTableReader) that).descriptor.equals(this.descriptor);
- }
-
- public int hashCode()
- {
- return this.descriptor.hashCode();
- }
-
- public String getFilename()
- {
- return dfile.path;
- }
-
- public String getIndexFilename()
- {
- return ifile.path;
- }
-
- public void setTrackedBy(DataTracker tracker)
- {
- deletingTask.setTracker(tracker);
- // under normal operation we can do this at any time, but SSTR is also used outside C* proper,
- // e.g. by BulkLoader, which does not initialize the cache. As a kludge, we set up the cache
- // here when we know we're being wired into the rest of the server infrastructure.
- keyCache = CacheService.instance.keyCache;
- }
-
- private void load(ValidationMetadata validation) throws IOException
- {
- if (metadata.getBloomFilterFpChance() == 1.0)
- {
- // bf is disabled.
- load(false, true);
- bf = FilterFactory.AlwaysPresent;
- }
- else if (!components.contains(Component.FILTER) || validation == null)
- {
- // bf is enabled, but filter component is missing.
- load(true, true);
- }
- else if (validation.bloomFilterFPChance != metadata.getBloomFilterFpChance())
- {
- // bf fp chance in sstable metadata and it has changed since compaction.
- load(true, true);
- }
- else
- {
- // bf is enabled and fp chance matches the currently configured value.
- load(false, true);
- loadBloomFilter();
- }
- }
-
- /**
- * Load bloom filter from Filter.db file.
- *
- * @throws IOException
- */
- private void loadBloomFilter() throws IOException
- {
- DataInputStream stream = null;
- try
- {
- stream = new DataInputStream(new BufferedInputStream(new FileInputStream(descriptor.filenameFor(Component.FILTER))));
- bf = FilterFactory.deserialize(stream, true);
- }
- finally
- {
- FileUtils.closeQuietly(stream);
- }
- }
-
- /**
- * Loads ifile, dfile and indexSummary, and optionally recreates the bloom filter.
- * @param saveSummaryIfCreated for bulk loading purposes, if the summary was absent and needed to be built, you can
- * avoid persisting it to disk by setting this to false
- */
- private void load(boolean recreateBloomFilter, boolean saveSummaryIfCreated) throws IOException
- {
- SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
- SegmentedFile.Builder dbuilder = compression
- ? SegmentedFile.getCompressedBuilder()
- : SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
-
- boolean summaryLoaded = loadSummary(ibuilder, dbuilder);
- if (recreateBloomFilter || !summaryLoaded)
- buildSummary(recreateBloomFilter, ibuilder, dbuilder, summaryLoaded, Downsampling.BASE_SAMPLING_LEVEL);
-
- ifile = ibuilder.complete(descriptor.filenameFor(Component.PRIMARY_INDEX));
- dfile = dbuilder.complete(descriptor.filenameFor(Component.DATA));
- if (saveSummaryIfCreated && (recreateBloomFilter || !summaryLoaded)) // save summary information to disk
- saveSummary(ibuilder, dbuilder);
- }
-
- /**
- * Build index summary(and optionally bloom filter) by reading through Index.db file.
- *
- * @param recreateBloomFilter true if recreate bloom filter
- * @param ibuilder
- * @param dbuilder
- * @param summaryLoaded true if index summary is already loaded and not need to build again
- * @throws IOException
- */
- private void buildSummary(boolean recreateBloomFilter, SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, boolean summaryLoaded, int samplingLevel) throws IOException
- {
- // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
- RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
-
- try
- {
- long indexSize = primaryIndex.length();
- long histogramCount = sstableMetadata.estimatedRowSize.count();
- long estimatedKeys = histogramCount > 0 && !sstableMetadata.estimatedRowSize.isOverflowed()
- ? histogramCount
- : estimateRowsFromIndex(primaryIndex); // statistics is supposed to be optional
-
- if (recreateBloomFilter)
- bf = FilterFactory.getFilter(estimatedKeys, metadata.getBloomFilterFpChance(), true);
-
- IndexSummaryBuilder summaryBuilder = null;
- if (!summaryLoaded)
- summaryBuilder = new IndexSummaryBuilder(estimatedKeys, metadata.getMinIndexInterval(), samplingLevel);
-
- long indexPosition;
- while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
- {
- ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex);
- RowIndexEntry indexEntry = metadata.comparator.rowIndexEntrySerializer().deserialize(primaryIndex, descriptor.version);
- DecoratedKey decoratedKey = partitioner.decorateKey(key);
- if (first == null)
- first = decoratedKey;
- last = decoratedKey;
-
- if (recreateBloomFilter)
- bf.add(decoratedKey.getKey());
-
- // if summary was already read from disk we don't want to re-populate it using primary index
- if (!summaryLoaded)
- {
- summaryBuilder.maybeAddEntry(decoratedKey, indexPosition);
- ibuilder.addPotentialBoundary(indexPosition);
- dbuilder.addPotentialBoundary(indexEntry.position);
- }
- }
-
- if (!summaryLoaded)
- indexSummary = summaryBuilder.build(partitioner);
- }
- finally
- {
- FileUtils.closeQuietly(primaryIndex);
- }
-
- first = getMinimalKey(first);
- last = getMinimalKey(last);
- }
-
- /**
- * Load index summary from Summary.db file if it exists.
- *
- * if loaded index summary has different index interval from current value stored in schema,
- * then Summary.db file will be deleted and this returns false to rebuild summary.
- *
- * @param ibuilder
- * @param dbuilder
- * @return true if index summary is loaded successfully from Summary.db file.
- */
- public boolean loadSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder)
- {
- File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
- if (!summariesFile.exists())
- return false;
-
- DataInputStream iStream = null;
- try
- {
- iStream = new DataInputStream(new FileInputStream(summariesFile));
- indexSummary = IndexSummary.serializer.deserialize(iStream, partitioner, descriptor.version.hasSamplingLevel, metadata.getMinIndexInterval(), metadata.getMaxIndexInterval());
- first = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
- last = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
- ibuilder.deserializeBounds(iStream);
- dbuilder.deserializeBounds(iStream);
- }
- catch (IOException e)
- {
- logger.debug("Cannot deserialize SSTable Summary File {}: {}", summariesFile.getPath(), e.getMessage());
- // corrupted; delete it and fall back to creating a new summary
- FileUtils.closeQuietly(iStream);
- // delete it and fall back to creating a new summary
- FileUtils.deleteWithConfirm(summariesFile);
- return false;
- }
- finally
- {
- FileUtils.closeQuietly(iStream);
- }
-
- return true;
- }
-
- /**
- * Save index summary to Summary.db file.
- *
- * @param ibuilder
- * @param dbuilder
- */
- public void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder)
- {
- saveSummary(ibuilder, dbuilder, indexSummary);
- }
-
- private void saveSummary(SegmentedFile.Builder ibuilder, SegmentedFile.Builder dbuilder, IndexSummary summary)
- {
- File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY));
- if (summariesFile.exists())
- FileUtils.deleteWithConfirm(summariesFile);
-
- DataOutputStreamAndChannel oStream = null;
- try
- {
- oStream = new DataOutputStreamAndChannel(new FileOutputStream(summariesFile));
- IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel);
- ByteBufferUtil.writeWithLength(first.getKey(), oStream);
- ByteBufferUtil.writeWithLength(last.getKey(), oStream);
- ibuilder.serializeBounds(oStream);
- dbuilder.serializeBounds(oStream);
- }
- catch (IOException e)
- {
- logger.debug("Cannot save SSTable Summary: ", e);
-
- // corrupted hence delete it and let it load it now.
- if (summariesFile.exists())
- FileUtils.deleteWithConfirm(summariesFile);
- }
- finally
- {
- FileUtils.closeQuietly(oStream);
- }
- }
-
- public void setReplacedBy(SSTableReader replacement)
- {
- synchronized (replaceLock)
- {
- assert replacedBy == null;
- replacedBy = replacement;
- replacement.replaces = this;
- replacement.replaceLock = replaceLock;
- }
- }
-
- public SSTableReader cloneWithNewStart(DecoratedKey newStart, final Runnable runOnClose)
- {
- synchronized (replaceLock)
- {
- assert replacedBy == null;
-
- if (newStart.compareTo(this.first) > 0)
- {
- if (newStart.compareTo(this.last) > 0)
- {
- this.runOnClose = new Runnable()
- {
- public void run()
- {
- CLibrary.trySkipCache(dfile.path, 0, 0);
- CLibrary.trySkipCache(ifile.path, 0, 0);
- runOnClose.run();
- }
- };
- }
- else
- {
- final long dataStart = getPosition(newStart, Operator.GE).position;
- final long indexStart = getIndexScanPosition(newStart);
- this.runOnClose = new Runnable()
- {
- public void run()
- {
- CLibrary.trySkipCache(dfile.path, 0, dataStart);
- CLibrary.trySkipCache(ifile.path, 0, indexStart);
- runOnClose.run();
- }
- };
- }
- }
-
- SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, indexSummary.readOnlyClone(), bf, maxDataAge, sstableMetadata,
- openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
- replacement.readMeterSyncFuture = this.readMeterSyncFuture;
- replacement.readMeter = this.readMeter;
- replacement.first = this.last.compareTo(newStart) > 0 ? newStart : this.last;
- replacement.last = this.last;
- setReplacedBy(replacement);
- return replacement;
- }
- }
-
- /**
- * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will
- * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have
- * its DeletingTask removed, and have its periodic read-meter sync task cancelled.
- * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader
- * @return a new SSTableReader
- * @throws IOException
- */
- public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
- {
- synchronized (replaceLock)
- {
- assert replacedBy == null;
-
- int minIndexInterval = metadata.getMinIndexInterval();
- int maxIndexInterval = metadata.getMaxIndexInterval();
- double effectiveInterval = indexSummary.getEffectiveIndexInterval();
-
- IndexSummary newSummary;
- long oldSize = bytesOnDisk();
-
- // We have to rebuild the summary from the on-disk primary index in three cases:
- // 1. The sampling level went up, so we need to read more entries off disk
- // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary
- // at full sampling (and consequently at any other sampling level)
- // 3. The max_index_interval was lowered, forcing us to raise the sampling level
- if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval)
- {
- newSummary = buildSummaryAtLevel(samplingLevel);
- }
- else if (samplingLevel < indexSummary.getSamplingLevel())
- {
- // we can use the existing index summary to make a smaller one
- newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, partitioner);
-
- SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
- SegmentedFile.Builder dbuilder = compression
- ? SegmentedFile.getCompressedBuilder()
- : SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
- saveSummary(ibuilder, dbuilder, newSummary);
- }
- else
- {
- throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " +
- "no adjustments to min/max_index_interval");
- }
-
- long newSize = bytesOnDisk();
- StorageMetrics.load.inc(newSize - oldSize);
- parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize);
-
- SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, newSummary, bf, maxDataAge, sstableMetadata,
- openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
- replacement.readMeterSyncFuture = this.readMeterSyncFuture;
- replacement.readMeter = this.readMeter;
- replacement.first = this.first;
- replacement.last = this.last;
- setReplacedBy(replacement);
- return replacement;
- }
- }
-
- private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
- {
- // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
- RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
- try
- {
- long indexSize = primaryIndex.length();
- IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel);
-
- long indexPosition;
- while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
- {
- summaryBuilder.maybeAddEntry(partitioner.decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition);
- RowIndexEntry.Serializer.skip(primaryIndex);
- }
-
- return summaryBuilder.build(partitioner);
- }
- finally
- {
- FileUtils.closeQuietly(primaryIndex);
- }
- }
-
- public int getIndexSummarySamplingLevel()
- {
- return indexSummary.getSamplingLevel();
- }
-
- public long getIndexSummaryOffHeapSize()
- {
- return indexSummary.getOffHeapSize();
- }
-
- public int getMinIndexInterval()
- {
- return indexSummary.getMinIndexInterval();
- }
-
- public double getEffectiveIndexInterval()
- {
- return indexSummary.getEffectiveIndexInterval();
- }
-
- public void releaseSummary()
- {
- indexSummary.close();
- indexSummary = null;
- }
-
- private void validate()
- {
- if (this.first.compareTo(this.last) > 0)
- throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last));
- }
-
- /**
- * Gets the position in the index file to start scanning to find the given key (at most indexInterval keys away,
- * modulo downsampling of the index summary).
- */
- public long getIndexScanPosition(RowPosition key)
- {
- return getIndexScanPositionFromBinarySearchResult(indexSummary.binarySearch(key), indexSummary);
- }
-
- private static long getIndexScanPositionFromBinarySearchResult(int binarySearchResult, IndexSummary referencedIndexSummary)
- {
- if (binarySearchResult == -1)
- return -1;
- else
- return referencedIndexSummary.getPosition(getIndexSummaryIndexFromBinarySearchResult(binarySearchResult));
- }
-
- private static int getIndexSummaryIndexFromBinarySearchResult(int binarySearchResult)
- {
- if (binarySearchResult < 0)
- {
- // binary search gives us the first index _greater_ than the key searched for,
- // i.e., its insertion position
- int greaterThan = (binarySearchResult + 1) * -1;
- if (greaterThan == 0)
- return -1;
- return greaterThan - 1;
- }
- else
- {
- return binarySearchResult;
- }
- }
-
- /**
- * Returns the compression metadata for this sstable.
- * @throws IllegalStateException if the sstable is not compressed
- */
- public CompressionMetadata getCompressionMetadata()
- {
- if (!compression)
- throw new IllegalStateException(this + " is not compressed");
-
- CompressionMetadata cmd = ((ICompressedFile) dfile).getMetadata();
-
- //We need the parent cf metadata
- String cfName = metadata.isSecondaryIndex() ? metadata.getParentColumnFamilyName() : metadata.cfName;
- cmd.parameters.setLiveMetadata(Schema.instance.getCFMetaData(metadata.ksName, cfName));
-
- return cmd;
- }
-
- /**
- * For testing purposes only.
- */
- public void forceFilterFailures()
- {
- bf = FilterFactory.AlwaysPresent;
- }
-
- public IFilter getBloomFilter()
- {
- return bf;
- }
-
- public long getBloomFilterSerializedSize()
- {
- return bf.serializedSize();
- }
-
- /**
- * @return An estimate of the number of keys in this SSTable based on the index summary.
- */
- public long estimatedKeys()
- {
- return indexSummary.getEstimatedKeyCount();
- }
-
- /**
- * @param ranges
- * @return An estimate of the number of keys for given ranges in this SSTable.
- */
- public long estimatedKeysForRanges(Collection<Range<Token>> ranges)
- {
- long sampleKeyCount = 0;
- List<Pair<Integer, Integer>> sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges);
- for (Pair<Integer, Integer> sampleIndexRange : sampleIndexes)
- sampleKeyCount += (sampleIndexRange.right - sampleIndexRange.left + 1);
-
- // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
- long estimatedKeys = sampleKeyCount * (Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
- return Math.max(1, estimatedKeys);
- }
-
- /**
- * Returns the number of entries in the IndexSummary. At full sampling, this is approximately 1/INDEX_INTERVALth of
- * the keys in this SSTable.
- */
- public int getIndexSummarySize()
- {
- return indexSummary.size();
- }
-
- /**
- * Returns the approximate number of entries the IndexSummary would contain if it were at full sampling.
- */
- public int getMaxIndexSummarySize()
- {
- return indexSummary.getMaxNumberOfEntries();
- }
-
- /**
- * Returns the key for the index summary entry at `index`.
- */
- public byte[] getIndexSummaryKey(int index)
- {
- return indexSummary.getKey(index);
- }
-
- private static List<Pair<Integer,Integer>> getSampleIndexesForRanges(IndexSummary summary, Collection<Range<Token>> ranges)
- {
- // use the index to determine a minimal section for each range
- List<Pair<Integer,Integer>> positions = new ArrayList<>();
-
- for (Range<Token> range : Range.normalize(ranges))
- {
- RowPosition leftPosition = range.left.maxKeyBound();
- RowPosition rightPosition = range.right.maxKeyBound();
-
- int left = summary.binarySearch(leftPosition);
- if (left < 0)
- left = (left + 1) * -1;
- else
- // left range are start exclusive
- left = left + 1;
- if (left == summary.size())
- // left is past the end of the sampling
- continue;
-
- int right = Range.isWrapAround(range.left, range.right)
- ? summary.size() - 1
- : summary.binarySearch(rightPosition);
- if (right < 0)
- {
- // range are end inclusive so we use the previous index from what binarySearch give us
- // since that will be the last index we will return
- right = (right + 1) * -1;
- if (right == 0)
- // Means the first key is already stricly greater that the right bound
- continue;
- right--;
- }
-
- if (left > right)
- // empty range
- continue;
- positions.add(Pair.create(left, right));
- }
- return positions;
- }
-
- public Iterable<DecoratedKey> getKeySamples(final Range<Token> range)
- {
- final List<Pair<Integer, Integer>> indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range));
-
- if (indexRanges.isEmpty())
- return Collections.emptyList();
-
- return new Iterable<DecoratedKey>()
- {
- public Iterator<DecoratedKey> iterator()
- {
- return new Iterator<DecoratedKey>()
- {
- private Iterator<Pair<Integer, Integer>> rangeIter = indexRanges.iterator();
- private Pair<Integer, Integer> current;
- private int idx;
-
- public boolean hasNext()
- {
- if (current == null || idx > current.right)
- {
- if (rangeIter.hasNext())
- {
- current = rangeIter.next();
- idx = current.left;
- return true;
- }
- return false;
- }
-
- return true;
- }
-
- public DecoratedKey next()
- {
- byte[] bytes = indexSummary.getKey(idx++);
- return partitioner.decorateKey(ByteBuffer.wrap(bytes));
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
- };
- }
- };
- }
-
- /**
- * Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges.
- * @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable.
- */
- public List<Pair<Long,Long>> getPositionsForRanges(Collection<Range<Token>> ranges)
- {
- // use the index to determine a minimal section for each range
- List<Pair<Long,Long>> positions = new ArrayList<>();
- for (Range<Token> range : Range.normalize(ranges))
- {
- AbstractBounds<RowPosition> keyRange = range.toRowBounds();
- RowIndexEntry idxLeft = getPosition(keyRange.left, Operator.GT);
- long left = idxLeft == null ? -1 : idxLeft.position;
- if (left == -1)
- // left is past the end of the file
- continue;
- RowIndexEntry idxRight = getPosition(keyRange.right, Operator.GT);
- long right = idxRight == null ? -1 : idxRight.position;
- if (right == -1 || Range.isWrapAround(range.left, range.right))
- // right is past the end of the file, or it wraps
- right = uncompressedLength();
- if (left == right)
- // empty range
- continue;
- positions.add(Pair.create(left, right));
- }
- return positions;
- }
-
- public void invalidateCacheKey(DecoratedKey key)
- {
- KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, key.getKey());
- keyCache.remove(cacheKey);
- }
-
- public void cacheKey(DecoratedKey key, RowIndexEntry info)
- {
- CachingOptions caching = metadata.getCaching();
-
- if (!caching.keyCache.isEnabled()
- || keyCache == null
- || keyCache.getCapacity() == 0)
- {
- return;
- }
-
- KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, key.getKey());
- logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
- keyCache.put(cacheKey, info);
- }
-
- public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
- {
- return getCachedPosition(new KeyCacheKey(metadata.cfId, descriptor, key.getKey()), updateStats);
- }
-
- private RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
- {
- if (keyCache != null && keyCache.getCapacity() > 0) {
- if (updateStats)
- {
- RowIndexEntry cachedEntry = keyCache.get(unifiedKey);
- keyCacheRequest.incrementAndGet();
- if (cachedEntry != null)
- keyCacheHit.incrementAndGet();
- return cachedEntry;
- }
- else
- {
- return keyCache.getInternal(unifiedKey);
- }
- }
- return null;
- }
-
- /**
- * Get position updating key cache and stats.
- * @see #getPosition(org.apache.cassandra.db.RowPosition, org.apache.cassandra.io.sstable.SSTableReader.Operator, boolean)
- */
- public RowIndexEntry getPosition(RowPosition key, Operator op)
- {
- return getPosition(key, op, true);
- }
-
- /**
- * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to
- * allow key selection by token bounds but only if op != * EQ
- * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins.
- * @param updateCacheAndStats true if updating stats and cache
- * @return The index entry corresponding to the key, or null if the key is not present
- */
- public RowIndexEntry getPosition(RowPosition key, Operator op, boolean updateCacheAndStats)
- {
- // first, check bloom filter
- if (op == Operator.EQ)
- {
- assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key
- if (!bf.isPresent(((DecoratedKey)key).getKey()))
- {
- Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.generation);
- return null;
- }
- }
-
- // next, the key cache (only make sense for valid row key)
- if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey))
- {
- DecoratedKey decoratedKey = (DecoratedKey)key;
- KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, decoratedKey.getKey());
- RowIndexEntry cachedPosition = getCachedPosition(cacheKey, updateCacheAndStats);
- if (cachedPosition != null)
- {
- Tracing.trace("Key cache hit for sstable {}", descriptor.generation);
- return cachedPosition;
- }
- }
-
- // check the smallest and greatest keys in the sstable to see if it can't be present
- if (first.compareTo(key) > 0 || last.compareTo(key) < 0)
- {
- if (op == Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addFalsePositive();
-
- if (op.apply(1) < 0)
- {
- Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.generation);
- return null;
- }
- }
-
- int binarySearchResult = indexSummary.binarySearch(key);
- long sampledPosition = getIndexScanPositionFromBinarySearchResult(binarySearchResult, indexSummary);
- int sampledIndex = getIndexSummaryIndexFromBinarySearchResult(binarySearchResult);
-
- // if we matched the -1th position, we'll start at the first position
- sampledPosition = sampledPosition == -1 ? 0 : sampledPosition;
-
- int effectiveInterval = indexSummary.getEffectiveIndexIntervalAfterIndex(sampledIndex);
-
- // scan the on-disk index, starting at the nearest sampled position.
- // The check against IndexInterval is to be exit the loop in the EQ case when the key looked for is not present
- // (bloom filter false positive). But note that for non-EQ cases, we might need to check the first key of the
- // next index position because the searched key can be greater the last key of the index interval checked if it
- // is lesser than the first key of next interval (and in that case we must return the position of the first key
- // of the next interval).
- int i = 0;
- Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
- while (segments.hasNext() && i <= effectiveInterval)
- {
- FileDataInput in = segments.next();
- try
- {
- while (!in.isEOF() && i <= effectiveInterval)
- {
- i++;
-
- ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
-
- boolean opSatisfied; // did we find an appropriate position for the op requested
- boolean exactMatch; // is the current position an exact match for the key, suitable for caching
-
- // Compare raw keys if possible for performance, otherwise compare decorated keys.
- if (op == Operator.EQ)
- {
- opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey());
- }
- else
- {
- DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
- int comparison = indexDecoratedKey.compareTo(key);
- int v = op.apply(comparison);
- opSatisfied = (v == 0);
- exactMatch = (comparison == 0);
- if (v < 0)
- {
- Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation);
- return null;
- }
- }
-
- if (opSatisfied)
- {
- // read data position from index entry
- RowIndexEntry indexEntry = metadata.comparator.rowIndexEntrySerializer().deserialize(in, descriptor.version);
- if (exactMatch && updateCacheAndStats)
- {
- assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key
- DecoratedKey decoratedKey = (DecoratedKey)key;
-
- if (logger.isTraceEnabled())
- {
- // expensive sanity check! see CASSANDRA-4687
- FileDataInput fdi = dfile.getSegment(indexEntry.position);
- DecoratedKey keyInDisk = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(fdi));
- if (!keyInDisk.equals(key))
- throw new AssertionError(String.format("%s != %s in %s", keyInDisk, key, fdi.getPath()));
- fdi.close();
- }
-
- // store exact match for the key
- cacheKey(decoratedKey, indexEntry);
- }
- if (op == Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addTruePositive();
- Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndex().size(), descriptor.generation);
- return indexEntry;
- }
-
- RowIndexEntry.Serializer.skip(in);
- }
- }
- catch (IOException e)
- {
- markSuspect();
- throw new CorruptSSTableException(e, in.getPath());
- }
- finally
- {
- FileUtils.closeQuietly(in);
- }
- }
-
- if (op == Operator.EQ && updateCacheAndStats)
- bloomFilterTracker.addFalsePositive();
- Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.generation);
- return null;
- }
-
- /**
- * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists.
- */
- public DecoratedKey firstKeyBeyond(RowPosition token)
- {
- long sampledPosition = getIndexScanPosition(token);
- if (sampledPosition == -1)
- sampledPosition = 0;
-
- Iterator<FileDataInput> segments = ifile.iterator(sampledPosition);
- while (segments.hasNext())
- {
- FileDataInput in = segments.next();
- try
- {
- while (!in.isEOF())
- {
- ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in);
- DecoratedKey indexDecoratedKey = partitioner.decorateKey(indexKey);
- if (indexDecoratedKey.compareTo(token) > 0)
- return indexDecoratedKey;
-
- RowIndexEntry.Serializer.skip(in);
- }
- }
- catch (IOException e)
- {
- markSuspect();
- throw new CorruptSSTableException(e, in.getPath());
- }
- finally
- {
- FileUtils.closeQuietly(in);
- }
- }
-
- return null;
- }
-
- /**
- * @return The length in bytes of the data for this SSTable. For
- * compressed files, this is not the same thing as the on disk size (see
- * onDiskLength())
- */
- public long uncompressedLength()
- {
- return dfile.length;
- }
-
- /**
- * @return The length in bytes of the on disk size for this SSTable. For
- * compressed files, this is not the same thing as the data length (see
- * length())
- */
- public long onDiskLength()
- {
- return dfile.onDiskLength;
- }
-
- public boolean acquireReference()
- {
- while (true)
- {
- int n = references.get();
- if (n <= 0)
- return false;
- if (references.compareAndSet(n, n + 1))
- return true;
- }
- }
-
- /**
- * Release reference to this SSTableReader.
- * If there is no one referring to this SSTable, and is marked as compacted,
- * all resources are cleaned up and files are deleted eventually.
- */
- public void releaseReference()
- {
- if (references.decrementAndGet() == 0)
- tidy(true);
- assert references.get() >= 0 : "Reference counter " + references.get() + " for " + dfile.path;
- }
-
- /**
- * Mark the sstable as obsolete, i.e., compacted into newer sstables.
- *
- * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere
- * except for threads holding a reference.
- *
- * @return true if the this is the first time the file was marked obsolete. Calling this
- * multiple times is usually buggy (see exceptions in DataTracker.unmarkCompacting and removeOldSSTablesSize).
- */
- public boolean markObsolete()
- {
- if (logger.isDebugEnabled())
- logger.debug("Marking {} compacted", getFilename());
-
- synchronized (replaceLock)
- {
- assert replacedBy == null;
- }
- return !isCompacted.getAndSet(true);
- }
-
- public boolean isMarkedCompacted()
- {
- return isCompacted.get();
- }
-
- public void markSuspect()
- {
- if (logger.isDebugEnabled())
- logger.debug("Marking {} as a suspect for blacklisting.", getFilename());
-
- isSuspect.getAndSet(true);
- }
-
- public boolean isMarkedSuspect()
- {
- return isSuspect.get();
- }
-
- /**
- *
- * @param dataRange filter to use when reading the columns
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public SSTableScanner getScanner(DataRange dataRange)
- {
- return new SSTableScanner(this, dataRange, null);
- }
-
- /**
- * I/O SSTableScanner
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public SSTableScanner getScanner()
- {
- return getScanner((RateLimiter) null);
- }
-
- public SSTableScanner getScanner(RateLimiter limiter)
- {
- return new SSTableScanner(this, DataRange.allData(partitioner), limiter);
- }
-
- /**
- * Direct I/O SSTableScanner over a defined range of tokens.
- *
- * @param range the range of keys to cover
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public ICompactionScanner getScanner(Range<Token> range, RateLimiter limiter)
- {
- if (range == null)
- return getScanner(limiter);
- return getScanner(Collections.singletonList(range), limiter);
- }
-
- /**
- * Direct I/O SSTableScanner over a defined collection of ranges of tokens.
- *
- * @param ranges the range of keys to cover
- * @return A Scanner for seeking over the rows of the SSTable.
- */
- public ICompactionScanner getScanner(Collection<Range<Token>> ranges, RateLimiter limiter)
- {
- // We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249)
- List<Pair<Long, Long>> positions = getPositionsForRanges(Range.normalize(ranges));
- if (positions.isEmpty())
- return new EmptyCompactionScanner(getFilename());
- else
- return new SSTableScanner(this, ranges, limiter);
- }
-
- public FileDataInput getFileDataInput(long position)
- {
- return dfile.getSegment(position);
- }
-
- /**
- * Tests if the sstable contains data newer than the given age param (in localhost currentMilli time).
- * This works in conjunction with maxDataAge which is an upper bound on the create of data in this sstable.
- * @param age The age to compare the maxDataAre of this sstable. Measured in millisec since epoc on this host
- * @return True iff this sstable contains data that's newer than the given age parameter.
- */
- public boolean newSince(long age)
- {
- return maxDataAge > age;
- }
-
- public void createLinks(String snapshotDirectoryPath)
- {
- for (Component component : components)
- {
- File sourceFile = new File(descriptor.filenameFor(component));
- File targetLink = new File(snapshotDirectoryPath, sourceFile.getName());
- FileUtils.createHardLink(sourceFile, targetLink);
- }
- }
-
- public boolean isRepaired()
- {
- return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
- }
-
- public SSTableReader getCurrentReplacement()
- {
- synchronized (replaceLock)
- {
- SSTableReader cur = this, next = replacedBy;
- while (next != null)
- {
- cur = next;
- next = next.replacedBy;
- }
- return cur;
- }
- }
-
- /**
- * TODO: Move someplace reusable
- */
- public abstract static class Operator
- {
- public static final Operator EQ = new Equals();
- public static final Operator GE = new GreaterThanOrEqualTo();
- public static final Operator GT = new GreaterThan();
-
- /**
- * @param comparison The result of a call to compare/compareTo, with the desired field on the rhs.
- * @return less than 0 if the operator cannot match forward, 0 if it matches, greater than 0 if it might match forward.
- */
- public abstract int apply(int comparison);
-
- final static class Equals extends Operator
- {
- public int apply(int comparison) { return -comparison; }
- }
-
- final static class GreaterThanOrEqualTo extends Operator
- {
- public int apply(int comparison) { return comparison >= 0 ? 0 : -comparison; }
- }
-
- final static class GreaterThan extends Operator
- {
- public int apply(int comparison) { return comparison > 0 ? 0 : 1; }
- }
- }
-
- public long getBloomFilterFalsePositiveCount()
- {
- return bloomFilterTracker.getFalsePositiveCount();
- }
-
- public long getRecentBloomFilterFalsePositiveCount()
- {
- return bloomFilterTracker.getRecentFalsePositiveCount();
- }
-
- public long getBloomFilterTruePositiveCount()
- {
- return bloomFilterTracker.getTruePositiveCount();
- }
-
- public long getRecentBloomFilterTruePositiveCount()
- {
- return bloomFilterTracker.getRecentTruePositiveCount();
- }
-
- public InstrumentingCache<KeyCacheKey, RowIndexEntry> getKeyCache()
- {
- return keyCache;
- }
-
- public EstimatedHistogram getEstimatedRowSize()
- {
- return sstableMetadata.estimatedRowSize;
- }
-
- public EstimatedHistogram getEstimatedColumnCount()
- {
- return sstableMetadata.estimatedColumnCount;
- }
-
- public double getEstimatedDroppableTombstoneRatio(int gcBefore)
- {
- return sstableMetadata.getEstimatedDroppableTombstoneRatio(gcBefore);
- }
-
- public double getDroppableTombstonesBefore(int gcBefore)
- {
- return sstableMetadata.getDroppableTombstonesBefore(gcBefore);
- }
-
- public double getCompressionRatio()
- {
- return sstableMetadata.compressionRatio;
- }
-
- public ReplayPosition getReplayPosition()
- {
- return sstableMetadata.replayPosition;
- }
-
- public long getMinTimestamp()
- {
- return sstableMetadata.minTimestamp;
- }
-
- public long getMaxTimestamp()
- {
- return sstableMetadata.maxTimestamp;
- }
-
- public Set<Integer> getAncestors()
- {
- try
- {
- CompactionMetadata compactionMetadata = (CompactionMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.COMPACTION);
- return compactionMetadata.ancestors;
- }
- catch (IOException e)
- {
- SSTableReader.logOpenException(descriptor, e);
- return Collections.emptySet();
- }
- }
-
- public int getSSTableLevel()
- {
- return sstableMetadata.sstableLevel;
- }
-
- /**
- * Reloads the sstable metadata from disk.
- *
- * Called after level is changed on sstable, for example if the sstable is dropped to L0
- *
- * Might be possible to remove in future versions
- *
- * @throws IOException
- */
- public void reloadSSTableMetadata() throws IOException
- {
- this.sstableMetadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS);
- }
-
- public StatsMetadata getSSTableMetadata()
- {
- return sstableMetadata;
- }
-
- public RandomAccessReader openDataReader(RateLimiter limiter)
- {
- assert limiter != null;
- return compression
- ? CompressedThrottledReader.open(getFilename(), getCompressionMetadata(), limiter)
- : ThrottledReader.open(new File(getFilename()), limiter);
- }
-
- public RandomAccessReader openDataReader()
- {
- return compression
- ? CompressedRandomAccessReader.open(getFilename(), getCompressionMetadata())
- : RandomAccessReader.open(new File(getFilename()));
- }
-
- public RandomAccessReader openIndexReader()
- {
- return RandomAccessReader.open(new File(getIndexFilename()));
- }
-
- /**
- * @param component component to get timestamp.
- * @return last modified time for given component. 0 if given component does not exist or IO error occurs.
- */
- public long getCreationTimeFor(Component component)
- {
- return new File(descriptor.filenameFor(component)).lastModified();
- }
-
- /**
- * @return Number of key cache hit
- */
- public long getKeyCacheHit()
- {
- return keyCacheHit.get();
- }
-
- /**
- * @return Number of key cache request
- */
- public long getKeyCacheRequest()
- {
- return keyCacheRequest.get();
- }
-
- /**
- * @param sstables
- * @return true if all desired references were acquired. Otherwise, it will unreference any partial acquisition, and return false.
- */
- public static boolean acquireReferences(Iterable<SSTableReader> sstables)
- {
- SSTableReader failed = null;
- for (SSTableReader sstable : sstables)
- {
- if (!sstable.acquireReference())
- {
- failed = sstable;
- break;
- }
- }
-
- if (failed == null)
- return true;
-
- for (SSTableReader sstable : sstables)
- {
- if (sstable == failed)
- break;
- sstable.releaseReference();
- }
- return false;
- }
-
- public static void releaseReferences(Iterable<SSTableReader> sstables)
- {
- for (SSTableReader sstable : sstables)
- {
- sstable.releaseReference();
- }
- }
-
- private void dropPageCache()
- {
- dropPageCache(dfile.path);
- dropPageCache(ifile.path);
- }
-
- private void dropPageCache(String filePath)
- {
- RandomAccessFile file = null;
-
- try
- {
- file = new RandomAccessFile(filePath, "r");
-
- int fd = CLibrary.getfd(file.getFD());
-
- if (fd > 0)
- {
- if (logger.isDebugEnabled())
- logger.debug(String.format("Dropping page cache of file %s.", filePath));
-
- CLibrary.trySkipCache(fd, 0, 0);
- }
- }
- catch (IOException e)
- {
- // we don't care if cache cleanup fails
- }
- finally
- {
- FileUtils.closeQuietly(file);
- }
- }
-
- /**
- * Increment the total row read count and read rate for this SSTable. This should not be incremented for range
- * slice queries, row cache hits, or non-query reads, like compaction.
- */
- public void incrementReadCount()
- {
- if (readMeter != null)
- readMeter.mark();
- }
-
- protected class EmptyCompactionScanner implements ICompactionScanner
- {
- private final String filename;
-
- public EmptyCompactionScanner(String filename)
- {
- this.filename = filename;
- }
-
- public long getLengthInBytes()
- {
- return 0;
- }
-
- public long getCurrentPosition()
- {
- return 0;
- }
-
- public String getBackingFiles()
- {
- return filename;
- }
-
- public boolean hasNext()
- {
- return false;
- }
-
- public OnDiskAtomIterator next()
- {
- return null;
- }
-
- public void close() throws IOException { }
-
- public void remove() { }
- }
-
- public static class SizeComparator implements Comparator<SSTableReader>
- {
- public int compare(SSTableReader o1, SSTableReader o2)
- {
- return Longs.compare(o1.onDiskLength(), o2.onDiskLength());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
index 76677ac..18825cb 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java
@@ -35,6 +35,8 @@ import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.utils.CLibrary;
import org.apache.cassandra.utils.FBUtilities;
[7/8] Extend Descriptor to include a format value and refactor
reader/writer apis
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java
index 1d2bf48..f88fe92 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactedRow.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.io.sstable.ColumnStats;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
/**
* a CompactedRow is an object that takes a bunch of rows (keys + columnfamilies)
@@ -47,7 +48,7 @@ public abstract class AbstractCompactedRow implements Closeable
*
* @return index information for the written row, or null if the compaction resulted in only expired tombstones.
*/
- public abstract RowIndexEntry write(long currentPosition, DataOutputPlus out) throws IOException;
+ public abstract RowIndexEntry write(long currentPosition, SequentialWriter out) throws IOException;
/**
* update @param digest with the data bytes of the row (not including row key or row size).
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 6a0e0df..d228cba 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +34,6 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.SSTableReader;
/**
* Pluggable compaction strategy determines how SSTables get merged.
@@ -369,7 +369,7 @@ public abstract class AbstractCompactionStrategy
long keys = sstable.estimatedKeys();
Set<Range<Token>> ranges = new HashSet<Range<Token>>(overlaps.size());
for (SSTableReader overlap : overlaps)
- ranges.add(new Range<Token>(overlap.first.getToken(), overlap.last.getToken(), overlap.partitioner));
+ ranges.add(new Range<>(overlap.first.getToken(), overlap.last.getToken(), overlap.partitioner));
long remainingKeys = keys - sstable.estimatedKeysForRanges(ranges);
// next, calculate what percentage of columns we have within those keys
long columns = sstable.getEstimatedColumnCount().mean() * remainingKeys;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
index 59338f4..7631baa 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
@@ -22,7 +22,7 @@ import java.util.Set;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DiskAwareRunnable;
public abstract class AbstractCompactionTask extends DiskAwareRunnable
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index ef27805..e57d01d 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -24,13 +24,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DataTracker;
import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.AlwaysPresentFilter;
/**
@@ -112,7 +112,7 @@ public class CompactionController implements AutoCloseable
if (compacting == null)
return Collections.<SSTableReader>emptySet();
- List<SSTableReader> candidates = new ArrayList<SSTableReader>();
+ List<SSTableReader> candidates = new ArrayList<>();
long minTimestamp = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java
index 0c9b52a..86918bc 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java
@@ -24,11 +24,14 @@ import java.util.List;
import com.google.common.collect.ImmutableList;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.MergeIterator;
public class CompactionIterable extends AbstractCompactionIterable
{
+ final SSTableFormat format;
+
private static final Comparator<OnDiskAtomIterator> comparator = new Comparator<OnDiskAtomIterator>()
{
public int compare(OnDiskAtomIterator i1, OnDiskAtomIterator i2)
@@ -37,9 +40,10 @@ public class CompactionIterable extends AbstractCompactionIterable
}
};
- public CompactionIterable(OperationType type, List<ICompactionScanner> scanners, CompactionController controller)
+ public CompactionIterable(OperationType type, List<ICompactionScanner> scanners, CompactionController controller, SSTableFormat.Type formatType)
{
super(controller, type, scanners);
+ this.format = formatType.info;
}
public CloseableIterator<AbstractCompactedRow> iterator()
@@ -71,7 +75,7 @@ public class CompactionIterable extends AbstractCompactionIterable
// create a new container for rows, since we're going to clear ours for the next one,
// and the AbstractCompactionRow code should be able to assume that the collection it receives
// won't be pulled out from under it.
- return new LazilyCompactedRow(controller, ImmutableList.copyOf(rows));
+ return format.getCompactedRowWriter(controller, ImmutableList.copyOf(rows));
}
finally
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index b08668e..18ad7ae 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -52,7 +52,9 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
-
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,10 +78,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.io.sstable.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.CompactionMetrics;
import org.apache.cassandra.repair.Validator;
@@ -327,7 +326,7 @@ public class CompactionManager implements CompactionManagerMBean
@Override
public boolean apply(SSTableReader sstable)
{
- return !(excludeCurrentVersion && sstable.descriptor.version.equals(Descriptor.Version.CURRENT));
+ return !(excludeCurrentVersion && sstable.descriptor.version.equals(sstable.descriptor.getFormat().getLatestVersion()));
}
});
}
@@ -502,7 +501,7 @@ public class CompactionManager implements CompactionManagerMBean
{
// look up the sstables now that we're on the compaction executor, so we don't try to re-compact
// something that was already being compacted earlier.
- Collection<SSTableReader> sstables = new ArrayList<SSTableReader>(dataFiles.size());
+ Collection<SSTableReader> sstables = new ArrayList<>(dataFiles.size());
for (Descriptor desc : dataFiles)
{
// inefficient but not in a performance sensitive path
@@ -845,12 +844,8 @@ public class CompactionManager implements CompactionManagerMBean
SSTableReader sstable)
{
FileUtils.createDirectory(compactionFileLocation);
- return new SSTableWriter(cfs.getTempSSTablePath(compactionFileLocation),
- expectedBloomFilterSize,
- repairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(Collections.singleton(sstable), cfs.metadata.comparator, sstable.getSSTableLevel()));
+
+ return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(compactionFileLocation)), expectedBloomFilterSize, repairedAt, sstable.getSSTableLevel());
}
public static SSTableWriter createWriterForAntiCompaction(ColumnFamilyStore cfs,
@@ -875,8 +870,8 @@ public class CompactionManager implements CompactionManagerMBean
break;
}
}
- return new SSTableWriter(cfs.getTempSSTablePath(compactionFileLocation),
- expectedBloomFilterSize,
+ return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(compactionFileLocation)),
+ (long)expectedBloomFilterSize,
repairedAt,
cfs.metadata,
cfs.partitioner,
@@ -1007,6 +1002,7 @@ public class CompactionManager implements CompactionManagerMBean
private void doAntiCompaction(ColumnFamilyStore cfs, Collection<Range<Token>> ranges,
Collection<SSTableReader> repairedSSTables, long repairedAt)
{
+
// TODO(5351): we can do better here:
logger.info("Performing anticompaction on {} sstables", repairedSSTables.size());
@@ -1044,6 +1040,7 @@ public class CompactionManager implements CompactionManagerMBean
groupMaxDataAge = sstable.maxDataAge;
}
+
if (anticompactionGroup.size() == 0)
{
logger.info("No valid anticompactions for this group, All sstables were compacted and are no longer available");
@@ -1068,7 +1065,7 @@ public class CompactionManager implements CompactionManagerMBean
repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, sstableAsSet));
unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, sstableAsSet));
- CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners.scanners, controller);
+ CompactionIterable ci = new CompactionIterable(OperationType.ANTICOMPACTION, scanners.scanners, controller, DatabaseDescriptor.getSSTableFormat());
Iterator<AbstractCompactedRow> iter = ci.iterator();
while(iter.hasNext())
{
@@ -1175,7 +1172,7 @@ public class CompactionManager implements CompactionManagerMBean
{
public ValidationCompactionIterable(ColumnFamilyStore cfs, List<ICompactionScanner> scanners, int gcBefore)
{
- super(OperationType.VALIDATION, scanners, new ValidationCompactionController(cfs, gcBefore));
+ super(OperationType.VALIDATION, scanners, new ValidationCompactionController(cfs, gcBefore), DatabaseDescriptor.getSSTableFormat());
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index bfb86c9..584ff38 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -31,6 +31,10 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,9 +43,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.UUIDGen;
@@ -150,11 +152,13 @@ public class CompactionTask extends AbstractCompactionTask
long estimatedTotalKeys = Math.max(cfs.metadata.getMinIndexInterval(), SSTableReader.getApproximateKeyCount(actuallyCompact));
long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(actuallyCompact) / strategy.getMaxSSTableBytes());
long keysPerSSTable = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
+ SSTableFormat.Type sstableFormat = getFormatType(sstables);
+
logger.debug("Expected bloom filter size : {}", keysPerSSTable);
try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact))
{
- AbstractCompactionIterable ci = new CompactionIterable(compactionType, scanners.scanners, controller);
+ AbstractCompactionIterable ci = new CompactionIterable(compactionType, scanners.scanners, controller, sstableFormat);
Iterator<AbstractCompactedRow> iter = ci.iterator();
// we can't preheat until the tracker has been set. This doesn't happen until we tell the cfs to
@@ -177,7 +181,7 @@ public class CompactionTask extends AbstractCompactionTask
return;
}
- writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt));
+ writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt, sstableFormat));
while (iter.hasNext())
{
if (ci.isStopRequested())
@@ -189,7 +193,7 @@ public class CompactionTask extends AbstractCompactionTask
totalKeysWritten++;
if (newSSTableSegmentThresholdReached(writer.currentWriter()))
{
- writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt));
+ writer.switchWriter(createCompactionWriter(sstableDirectory, keysPerSSTable, minRepairedAt, sstableFormat));
}
}
@@ -271,14 +275,14 @@ public class CompactionTask extends AbstractCompactionTask
return minRepairedAt;
}
- private SSTableWriter createCompactionWriter(File sstableDirectory, long keysPerSSTable, long repairedAt)
+ private SSTableWriter createCompactionWriter(File sstableDirectory, long keysPerSSTable, long repairedAt, SSTableFormat.Type type)
{
- return new SSTableWriter(cfs.getTempSSTablePath(sstableDirectory),
- keysPerSSTable,
- repairedAt,
- cfs.metadata,
- cfs.partitioner,
- new MetadataCollector(sstables, cfs.metadata.comparator, getLevel()));
+ return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory), type),
+ keysPerSSTable,
+ repairedAt,
+ cfs.metadata,
+ cfs.partitioner,
+ new MetadataCollector(sstables, cfs.metadata.comparator, getLevel()));
}
protected int getLevel()
@@ -312,4 +316,13 @@ public class CompactionTask extends AbstractCompactionTask
}
return max;
}
+
+ public static SSTableFormat.Type getFormatType(Collection<SSTableReader> sstables)
+ {
+ if (sstables.isEmpty() || !SSTableFormat.enableSSTableDevelopmentTestMode)
+ return DatabaseDescriptor.getSSTableFormat();
+
+ //Allows us to test compaction of non-default formats
+ return sstables.iterator().next().descriptor.formatType;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index 8c997ed..3f2aed6 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -22,13 +22,13 @@ import java.util.*;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.statements.CFPropDefs;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.Pair;
public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java b/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
index cfdbd17..1b28179 100644
--- a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
+++ b/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
@@ -30,12 +30,13 @@ import com.google.common.collect.Iterators;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.index.SecondaryIndexManager;
+import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
import org.apache.cassandra.io.sstable.ColumnNameHelper;
import org.apache.cassandra.io.sstable.ColumnStats;
import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
import org.apache.cassandra.utils.MergeIterator;
import org.apache.cassandra.utils.StreamingHistogram;
@@ -48,17 +49,17 @@ import org.apache.cassandra.utils.StreamingHistogram;
*/
public class LazilyCompactedRow extends AbstractCompactedRow
{
- private final List<? extends OnDiskAtomIterator> rows;
- private final CompactionController controller;
- private final long maxPurgeableTimestamp;
- private final ColumnFamily emptyColumnFamily;
- private ColumnStats columnStats;
- private boolean closed;
- private ColumnIndex.Builder indexBuilder;
- private final SecondaryIndexManager.Updater indexer;
- private final Reducer reducer;
- private final Iterator<OnDiskAtom> merger;
- private DeletionTime maxRowTombstone;
+ protected final List<? extends OnDiskAtomIterator> rows;
+ protected final CompactionController controller;
+ protected final long maxPurgeableTimestamp;
+ protected final ColumnFamily emptyColumnFamily;
+ protected ColumnStats columnStats;
+ protected boolean closed;
+ protected ColumnIndex.Builder indexBuilder;
+ protected final SecondaryIndexManager.Updater indexer;
+ protected final Reducer reducer;
+ protected final Iterator<OnDiskAtom> merger;
+ protected DeletionTime maxRowTombstone;
public LazilyCompactedRow(CompactionController controller, List<? extends OnDiskAtomIterator> rows)
{
@@ -99,10 +100,12 @@ public class LazilyCompactedRow extends AbstractCompactedRow
ColumnFamilyStore.removeDeletedColumnsOnly(cf, overriddenGCBefore, controller.cfs.indexManager.gcUpdaterFor(key));
}
- public RowIndexEntry write(long currentPosition, DataOutputPlus out) throws IOException
+ public RowIndexEntry write(long currentPosition, SequentialWriter dataFile) throws IOException
{
assert !closed;
+ DataOutputPlus out = dataFile.stream;
+
ColumnIndex columnsIndex;
try
{
@@ -130,7 +133,7 @@ public class LazilyCompactedRow extends AbstractCompactedRow
// in case no columns were ever written, we may still need to write an empty header with a top-level tombstone
indexBuilder.maybeWriteEmptyRowHeader();
- out.writeShort(SSTableWriter.END_OF_ROW);
+ out.writeShort(BigTableWriter.END_OF_ROW);
close();
@@ -183,7 +186,7 @@ public class LazilyCompactedRow extends AbstractCompactedRow
closed = true;
}
- private class Reducer extends MergeIterator.Reducer<OnDiskAtom, OnDiskAtom>
+ protected class Reducer extends MergeIterator.Reducer<OnDiskAtom, OnDiskAtom>
{
// all columns reduced together will have the same name, so there will only be one column
// in the container; we just want to leverage the conflict resolution code from CF.
@@ -192,9 +195,9 @@ public class LazilyCompactedRow extends AbstractCompactedRow
// tombstone reference; will be reconciled w/ column during getReduced. Note that the top-level (row) tombstone
// is held by LCR.deletionInfo.
- RangeTombstone tombstone;
+ public RangeTombstone tombstone;
- int columns = 0;
+ public int columns = 0;
// if the row tombstone is 'live' we need to set timestamp to MAX_VALUE to be able to overwrite it later
// markedForDeleteAt is MIN_VALUE for 'live' row tombstones (which we use to default maxTimestampSeen)
@@ -204,10 +207,10 @@ public class LazilyCompactedRow extends AbstractCompactedRow
// we are bound to have either a RangeTombstone or standard cells will set this properly:
ColumnStats.MaxIntTracker maxDeletionTimeTracker = new ColumnStats.MaxIntTracker(Integer.MAX_VALUE);
- StreamingHistogram tombstones = new StreamingHistogram(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE);
- List<ByteBuffer> minColumnNameSeen = Collections.emptyList();
- List<ByteBuffer> maxColumnNameSeen = Collections.emptyList();
- boolean hasLegacyCounterShards = false;
+ public StreamingHistogram tombstones = new StreamingHistogram(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE);
+ public List<ByteBuffer> minColumnNameSeen = Collections.emptyList();
+ public List<ByteBuffer> maxColumnNameSeen = Collections.emptyList();
+ public boolean hasLegacyCounterShards = false;
public Reducer()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index 994e52d..7a1f883 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -35,6 +35,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.*;
import com.google.common.primitives.Doubles;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +44,6 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
import org.apache.cassandra.notifications.SSTableAddedNotification;
@@ -160,6 +160,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem
}
}
+ @Override
public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
{
throw new UnsupportedOperationException("LevelDB compaction strategy does not allow user-specified compactions");
@@ -319,7 +320,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem
this.range = range;
// add only sstables that intersect our range, and estimate how much data that involves
- this.sstables = new ArrayList<SSTableReader>(sstables.size());
+ this.sstables = new ArrayList<>(sstables.size());
long length = 0;
for (SSTableReader sstable : sstables)
{
@@ -342,10 +343,10 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy implem
public static List<SSTableReader> intersecting(Collection<SSTableReader> sstables, Range<Token> range)
{
- ArrayList<SSTableReader> filtered = new ArrayList<SSTableReader>();
+ ArrayList<SSTableReader> filtered = new ArrayList<>();
for (SSTableReader sstable : sstables)
{
- Range<Token> sstableRange = new Range<Token>(sstable.first.getToken(), sstable.last.getToken(), sstable.partitioner);
+ Range<Token> sstableRange = new Range<>(sstable.first.getToken(), sstable.last.getToken(), sstable.partitioner);
if (range == null || sstableRange.intersects(range))
filtered.add(sstable);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
index 2731b6d..9a33b49 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
@@ -20,8 +20,8 @@ package org.apache.cassandra.db.compaction;
import java.util.Collection;
import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
public class LeveledCompactionTask extends CompactionTask
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 6d3bf69..74be143 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -574,7 +575,7 @@ public class LeveledManifest
private static Set<SSTableReader> overlapping(Token start, Token end, Iterable<SSTableReader> sstables)
{
assert start.compareTo(end) <= 0;
- Set<SSTableReader> overlapped = new HashSet<SSTableReader>();
+ Set<SSTableReader> overlapped = new HashSet<>();
Bounds<Token> promotedBounds = new Bounds<Token>(start, end);
for (SSTableReader candidate : sstables)
{
@@ -622,8 +623,8 @@ public class LeveledManifest
// Note that we ignore suspect-ness of L1 sstables here, since if an L1 sstable is suspect we're
// basically screwed, since we expect all or most L0 sstables to overlap with each L1 sstable.
// So if an L1 sstable is suspect we can't do much besides try anyway and hope for the best.
- Set<SSTableReader> candidates = new HashSet<SSTableReader>();
- Set<SSTableReader> remaining = new HashSet<SSTableReader>();
+ Set<SSTableReader> candidates = new HashSet<>();
+ Set<SSTableReader> remaining = new HashSet<>();
Iterables.addAll(remaining, Iterables.filter(getLevel(0), Predicates.not(suspectP)));
for (SSTableReader sstable : ageSortedSSTables(remaining))
{
@@ -699,7 +700,7 @@ public class LeveledManifest
private List<SSTableReader> ageSortedSSTables(Collection<SSTableReader> candidates)
{
- List<SSTableReader> ageSortedCandidates = new ArrayList<SSTableReader>(candidates);
+ List<SSTableReader> ageSortedCandidates = new ArrayList<>(candidates);
Collections.sort(ageSortedCandidates, SSTableReader.maxTimestampComparator);
return ageSortedCandidates;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
index 6b9f161..87c82e0 100644
--- a/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
+++ b/src/java/org/apache/cassandra/db/compaction/SSTableSplitter.java
@@ -20,7 +20,8 @@ package org.apache.cassandra.db.compaction;
import java.util.*;
import org.apache.cassandra.db.*;
-import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
public class SSTableSplitter {
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index b3d098d..1bea188 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -25,6 +25,8 @@ import com.google.common.base.Throwables;
import org.apache.cassandra.db.*;
import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.service.ActiveRepairService;
@@ -46,6 +48,7 @@ public class Scrubber implements Closeable
private final RandomAccessReader dataFile;
private final RandomAccessReader indexFile;
private final ScrubInfo scrubInfo;
+ private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
private final boolean isOffline;
@@ -80,6 +83,8 @@ public class Scrubber implements Closeable
this.skipCorrupted = skipCorrupted;
this.isOffline = isOffline;
+ this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata);
+
// Calculate the expected compacted filesize
this.destination = cfs.directories.getDirectoryForNewSSTables();
if (destination == null)
@@ -113,7 +118,7 @@ public class Scrubber implements Closeable
ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile);
{
// throw away variable so we don't have a side effect in the assert
- long firstRowPositionFromIndex = sstable.metadata.comparator.rowIndexEntrySerializer().deserialize(indexFile, sstable.descriptor.version).position;
+ long firstRowPositionFromIndex = rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position;
assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex;
}
@@ -147,7 +152,7 @@ public class Scrubber implements Closeable
nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile);
nextRowPositionFromIndex = indexFile.isEOF()
? dataFile.length()
- : sstable.metadata.comparator.rowIndexEntrySerializer().deserialize(indexFile, sstable.descriptor.version).position;
+ : rowIndexEntrySerializer.deserialize(indexFile, sstable.descriptor.version).position;
}
catch (Throwable th)
{
@@ -177,7 +182,7 @@ public class Scrubber implements Closeable
if (dataSize > dataFile.length())
throw new IOError(new IOException("Impossible row size " + dataSize));
- SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, dataSize, true);
+ SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, true);
if (prevKey != null && prevKey.compareTo(key) > 0)
{
saveOutOfOrderRow(prevKey, key, atoms);
@@ -206,7 +211,7 @@ public class Scrubber implements Closeable
key = sstable.partitioner.decorateKey(currentIndexKey);
try
{
- SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, dataSize, true);
+ SSTableIdentityIterator atoms = new SSTableIdentityIterator(sstable, dataFile, key, true);
if (prevKey != null && prevKey.compareTo(key) > 0)
{
saveOutOfOrderRow(prevKey, key, atoms);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
index 2fa188c..0abb68d 100644
--- a/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
@@ -24,13 +24,14 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cql3.statements.CFPropDefs;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.Pair;
public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
@@ -311,7 +312,7 @@ public class SizeTieredCompactionStrategy extends AbstractCompactionStrategy
public static List<Pair<SSTableReader, Long>> createSSTableAndLengthPairs(Iterable<SSTableReader> sstables)
{
- List<Pair<SSTableReader, Long>> sstableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>(Iterables.size(sstables));
+ List<Pair<SSTableReader, Long>> sstableLengthPairs = new ArrayList<>(Iterables.size(sstables));
for(SSTableReader sstable : sstables)
sstableLengthPairs.add(Pair.create(sstable, sstable.onDiskLength()));
return sstableLengthPairs;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/compaction/Upgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 2e33ac9..c9e7034 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -22,9 +22,12 @@ import java.util.*;
import com.google.common.base.Throwables;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.OutputHandler;
@@ -78,7 +81,7 @@ public class Upgrader
sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
}
- return new SSTableWriter(cfs.getTempSSTablePath(directory), estimatedRows, repairedAt, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
+ return SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(directory)), estimatedRows, repairedAt, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
}
public void upgrade()
@@ -88,7 +91,7 @@ public class Upgrader
SSTableRewriter writer = new SSTableRewriter(cfs, toUpgrade, CompactionTask.getMaxDataAge(this.toUpgrade), OperationType.UPGRADE_SSTABLES, true);
try (AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(this.toUpgrade))
{
- Iterator<AbstractCompactedRow> iter = new CompactionIterable(compactionType, scanners.scanners, controller).iterator();
+ Iterator<AbstractCompactedRow> iter = new CompactionIterable(compactionType, scanners.scanners, controller, DatabaseDescriptor.getSSTableFormat()).iterator();
writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata().repairedAt));
while (iter.hasNext())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/composites/AbstractCType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCType.java b/src/java/org/apache/cassandra/db/composites/AbstractCType.java
index 5af7458..1df73fe 100644
--- a/src/java/org/apache/cassandra/db/composites/AbstractCType.java
+++ b/src/java/org/apache/cassandra/db/composites/AbstractCType.java
@@ -80,12 +80,10 @@ public abstract class AbstractCType implements CType
private final Serializer serializer;
- private final ISerializer<IndexInfo> indexSerializer;
private final IVersionedSerializer<ColumnSlice> sliceSerializer;
private final IVersionedSerializer<SliceQueryFilter> sliceQueryFilterSerializer;
private final DeletionInfo.Serializer deletionInfoSerializer;
private final RangeTombstone.Serializer rangeTombstoneSerializer;
- private final RowIndexEntry.Serializer rowIndexEntrySerializer;
protected final boolean isByteOrderComparable;
@@ -115,12 +113,10 @@ public abstract class AbstractCType implements CType
serializer = new Serializer(this);
- indexSerializer = new IndexInfo.Serializer(this);
sliceSerializer = new ColumnSlice.Serializer(this);
sliceQueryFilterSerializer = new SliceQueryFilter.Serializer(this);
deletionInfoSerializer = new DeletionInfo.Serializer(this);
rangeTombstoneSerializer = new RangeTombstone.Serializer(this);
- rowIndexEntrySerializer = new RowIndexEntry.Serializer(this);
this.isByteOrderComparable = isByteOrderComparable;
}
@@ -295,11 +291,6 @@ public abstract class AbstractCType implements CType
return indexReverseComparator;
}
- public ISerializer<IndexInfo> indexSerializer()
- {
- return indexSerializer;
- }
-
public IVersionedSerializer<ColumnSlice> sliceSerializer()
{
return sliceSerializer;
@@ -320,11 +311,6 @@ public abstract class AbstractCType implements CType
return rangeTombstoneSerializer;
}
- public RowIndexEntry.Serializer rowIndexEntrySerializer()
- {
- return rowIndexEntrySerializer;
- }
-
@Override
public boolean equals(Object o)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/composites/CType.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/composites/CType.java b/src/java/org/apache/cassandra/db/composites/CType.java
index d24277e..3844342 100644
--- a/src/java/org/apache/cassandra/db/composites/CType.java
+++ b/src/java/org/apache/cassandra/db/composites/CType.java
@@ -130,12 +130,10 @@ public interface CType extends Comparator<Composite>
public Serializer serializer();
- public ISerializer<IndexInfo> indexSerializer();
public IVersionedSerializer<ColumnSlice> sliceSerializer();
public IVersionedSerializer<SliceQueryFilter> sliceQueryFilterSerializer();
public DeletionInfo.Serializer deletionInfoSerializer();
public RangeTombstone.Serializer rangeTombstoneSerializer();
- public RowIndexEntry.Serializer rowIndexEntrySerializer();
public interface Serializer extends ISerializer<Composite>
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java b/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java
index 3750c75..16be34c 100644
--- a/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/IDiskAtomFilter.java
@@ -28,7 +28,7 @@ import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.db.composites.CType;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileDataInput;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
index 0a34dfd..b5515bc 100644
--- a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
@@ -24,20 +24,19 @@ import java.util.Iterator;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.lang3.StringUtils;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
-import org.apache.cassandra.db.columniterator.SSTableNamesIterator;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.db.composites.CType;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.utils.SearchIterator;
@@ -87,12 +86,12 @@ public class NamesQueryFilter implements IDiskAtomFilter
public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key)
{
- return new SSTableNamesIterator(sstable, key, columns);
+ return sstable.iterator(key, columns);
}
public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry)
{
- return new SSTableNamesIterator(sstable, file, key, columns, indexEntry);
+ return sstable.iterator(file, key, columns, indexEntry);
}
public void collectReducedColumns(ColumnFamily container, Iterator<Cell> reducedColumns, int gcBefore, long now)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/filter/QueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/QueryFilter.java b/src/java/org/apache/cassandra/db/filter/QueryFilter.java
index f58fa9f..1914970 100644
--- a/src/java/org/apache/cassandra/db/filter/QueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/QueryFilter.java
@@ -33,7 +33,7 @@ import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.Composite;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.MergeIterator;
public class QueryFilter
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 71d9095..0887cbe 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -24,19 +24,18 @@ import java.util.*;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
-import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
import org.apache.cassandra.db.composites.CType;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.io.IVersionedSerializer;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.tracing.Tracing;
@@ -176,12 +175,12 @@ public class SliceQueryFilter implements IDiskAtomFilter
public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key)
{
- return new SSTableSliceIterator(sstable, key, slices, reversed);
+ return sstable.iterator(key, slices, reversed);
}
public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry)
{
- return new SSTableSliceIterator(sstable, file, key, slices, reversed, indexEntry);
+ return sstable.iterator(file, key, slices, reversed, indexEntry);
}
public Comparator<Cell> getColumnComparator(CellNameType comparator)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
index 529e82c..c50019a 100644
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java
@@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +50,6 @@ import org.apache.cassandra.db.marshal.LocalByPartionerType;
import org.apache.cassandra.dht.LocalToken;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
index c2d481b..5640d23 100644
--- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.Future;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +53,6 @@ import org.apache.cassandra.db.filter.ExtendedFilter;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.concurrent.OpOrder;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/dht/BytesToken.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/dht/BytesToken.java b/src/java/org/apache/cassandra/dht/BytesToken.java
index f2f9eab..da965d2 100644
--- a/src/java/org/apache/cassandra/dht/BytesToken.java
+++ b/src/java/org/apache/cassandra/dht/BytesToken.java
@@ -49,7 +49,6 @@ public class BytesToken extends Token<byte[]>
return FBUtilities.compareUnsigned(token, o.token, 0, 0, token.length, o.token.length);
}
-
@Override
public int hashCode()
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/ISSTableSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/ISSTableSerializer.java b/src/java/org/apache/cassandra/io/ISSTableSerializer.java
index 20ee352..2834662 100644
--- a/src/java/org/apache/cassandra/io/ISSTableSerializer.java
+++ b/src/java/org/apache/cassandra/io/ISSTableSerializer.java
@@ -21,6 +21,7 @@ import java.io.DataInput;
import java.io.IOException;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
public interface ISSTableSerializer<T>
@@ -43,5 +44,5 @@ public interface ISSTableSerializer<T>
* @throws IOException
* @return the type that was deserialized
*/
- public T deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException;
+ public T deserializeFromSSTable(DataInput in, Version version) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
index 3d6cb71..75fdf1b 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
@@ -39,6 +39,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Longs;
import org.apache.cassandra.cache.RefCountedMemory;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.FSReadError;
@@ -47,10 +48,12 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.Memory;
import org.apache.cassandra.utils.Pair;
+import org.hsqldb.Database;
/**
* Holds metadata about compressed file
@@ -79,7 +82,7 @@ public class CompressionMetadata
public static CompressionMetadata create(String dataFilePath)
{
Descriptor desc = Descriptor.fromFilename(dataFilePath);
- return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length(), desc.version.hasPostCompressionAdlerChecksums);
+ return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length(), desc.version.hasPostCompressionAdlerChecksums());
}
@VisibleForTesting
@@ -262,6 +265,8 @@ public class CompressionMetadata
private int maxCount = 100;
private RefCountedMemory offsets = new RefCountedMemory(maxCount * 8);
private int count = 0;
+ private Version latestVersion = DatabaseDescriptor.getSSTableFormat().info.getLatestVersion();
+
private Writer(CompressionParameters parameters, String path)
{
@@ -311,14 +316,14 @@ public class CompressionMetadata
public CompressionMetadata openEarly(long dataLength, long compressedLength)
{
- return new CompressionMetadata(filePath, parameters, offsets, count * 8L, dataLength, compressedLength, Descriptor.Version.CURRENT.hasPostCompressionAdlerChecksums);
+ return new CompressionMetadata(filePath, parameters, offsets, count * 8L, dataLength, compressedLength, latestVersion.hasPostCompressionAdlerChecksums());
}
public CompressionMetadata openAfterClose(long dataLength, long compressedLength)
{
RefCountedMemory newOffsets = offsets.copy(count * 8L);
offsets.unreference();
- return new CompressionMetadata(filePath, parameters, newOffsets, count * 8L, dataLength, compressedLength, Descriptor.Version.CURRENT.hasPostCompressionAdlerChecksums);
+ return new CompressionMetadata(filePath, parameters, newOffsets, count * 8L, dataLength, compressedLength, latestVersion.hasPostCompressionAdlerChecksums());
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
index ad8fb3e..11d6d5e 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
@@ -30,7 +30,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.CounterId;
import org.apache.cassandra.utils.Pair;
@@ -43,6 +44,8 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
protected ColumnFamily columnFamily;
protected ByteBuffer currentSuperColumn;
protected final CounterId counterid = CounterId.generate();
+ private SSTableFormat.Type formatType = DatabaseDescriptor.getSSTableFormat();
+
public AbstractSSTableSimpleWriter(File directory, CFMetaData metadata, IPartitioner partitioner)
{
@@ -51,19 +54,18 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
DatabaseDescriptor.setPartitioner(partitioner);
}
+ protected void setSSTableFormatType(SSTableFormat.Type type)
+ {
+ this.formatType = type;
+ }
+
protected SSTableWriter getWriter()
{
- return new SSTableWriter(
- makeFilename(directory, metadata.ksName, metadata.cfName),
- 0, // We don't care about the bloom filter
- ActiveRepairService.UNREPAIRED_SSTABLE,
- metadata,
- DatabaseDescriptor.getPartitioner(),
- new MetadataCollector(metadata.comparator));
+ return SSTableWriter.create(Descriptor.fromFilename(makeFilename(directory, metadata.ksName, metadata.cfName, formatType)), 0, ActiveRepairService.UNREPAIRED_SSTABLE);
}
// find available generation and pick up filename from that
- protected static String makeFilename(File directory, final String keyspace, final String columnFamily)
+ protected static String makeFilename(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt)
{
final Set<Descriptor> existing = new HashSet<Descriptor>();
directory.list(new FilenameFilter()
@@ -84,7 +86,7 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
int maxGen = 0;
for (Descriptor desc : existing)
maxGen = Math.max(maxGen, desc.generation);
- return new Descriptor(directory, keyspace, columnFamily, maxGen + 1, Descriptor.Type.TEMP).filenameFor(Component.DATA);
+ return new Descriptor(directory, keyspace, columnFamily, maxGen + 1, Descriptor.Type.TEMP, fmt).filenameFor(Component.DATA);
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
index bf4da24..f7d467e 100644
--- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
@@ -39,6 +39,7 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestValidationException;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.utils.Pair;
@@ -272,6 +273,8 @@ public class CQLSSTableWriter implements Closeable
private File directory;
private IPartitioner partitioner = new Murmur3Partitioner();
+ protected SSTableFormat.Type formatType = null;
+
private CFMetaData schema;
private UpdateStatement insert;
private List<ColumnSpecification> boundNames;
@@ -279,7 +282,7 @@ public class CQLSSTableWriter implements Closeable
private boolean sorted = false;
private long bufferSizeInMB = 128;
- private Builder() {}
+ protected Builder() {}
/**
* The directory where to write the sstables.
@@ -484,6 +487,10 @@ public class CQLSSTableWriter implements Closeable
AbstractSSTableSimpleWriter writer = sorted
? new SSTableSimpleWriter(directory, schema, partitioner)
: new BufferedWriter(directory, schema, partitioner, bufferSizeInMB);
+
+ if (formatType != null)
+ writer.setSSTableFormatType(formatType);
+
return new CQLSSTableWriter(writer, insert, boundNames);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/Descriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
index 6911b48..03522c3 100644
--- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java
+++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java
@@ -22,13 +22,18 @@ import java.util.ArrayDeque;
import java.util.Deque;
import java.util.StringTokenizer;
+import com.google.common.base.CharMatcher;
import com.google.common.base.Objects;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
import org.apache.cassandra.utils.Pair;
+import org.apache.commons.lang.StringUtils;
import static org.apache.cassandra.io.sstable.Component.separator;
@@ -41,92 +46,6 @@ import static org.apache.cassandra.io.sstable.Component.separator;
*/
public class Descriptor
{
- // versions are denoted as [major][minor]. Minor versions must be forward-compatible:
- // new fields are allowed in e.g. the metadata component, but fields can't be removed
- // or have their size changed.
- //
- // Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that,
- // we always incremented the major version.
- public static class Version
- {
- // This needs to be at the begining for initialization sake
- public static final String current_version = "la";
-
- // ja (2.0.0): super columns are serialized as composites (note that there is no real format change,
- // this is mostly a marker to know if we should expect super columns or not. We do need
- // a major version bump however, because we should not allow streaming of super columns
- // into this new format)
- // tracks max local deletiontime in sstable metadata
- // records bloom_filter_fp_chance in metadata component
- // remove data size and column count from data file (CASSANDRA-4180)
- // tracks max/min column values (according to comparator)
- // jb (2.0.1): switch from crc32 to adler32 for compression checksums
- // checksum the compressed data
- // ka (2.1.0): new Statistics.db file format
- // index summaries can be downsampled and the sampling level is persisted
- // switch uncompressed checksums to adler32
- // tracks presense of legacy (local and remote) counter shards
- // la (3.0.0): new file name format
-
- public static final Version CURRENT = new Version(current_version);
-
- private final String version;
-
- public final boolean isLatestVersion;
- public final boolean hasPostCompressionAdlerChecksums;
- public final boolean hasSamplingLevel;
- public final boolean newStatsFile;
- public final boolean hasAllAdlerChecksums;
- public final boolean hasRepairedAt;
- public final boolean tracksLegacyCounterShards;
- public final boolean newFileName;
-
- public Version(String version)
- {
- this.version = version;
- isLatestVersion = version.compareTo(current_version) == 0;
- hasPostCompressionAdlerChecksums = version.compareTo("jb") >= 0;
- hasSamplingLevel = version.compareTo("ka") >= 0;
- newStatsFile = version.compareTo("ka") >= 0;
- hasAllAdlerChecksums = version.compareTo("ka") >= 0;
- hasRepairedAt = version.compareTo("ka") >= 0;
- tracksLegacyCounterShards = version.compareTo("ka") >= 0;
- newFileName = version.compareTo("la") >= 0;
- }
-
- /**
- * @param ver SSTable version
- * @return True if the given version string matches the format.
- * @see #version
- */
- static boolean validate(String ver)
- {
- return ver != null && ver.matches("[a-z]+");
- }
-
- public boolean isCompatible()
- {
- return version.compareTo("ja") >= 0 && version.charAt(0) <= CURRENT.version.charAt(0);
- }
-
- @Override
- public String toString()
- {
- return version;
- }
-
- @Override
- public boolean equals(Object o)
- {
- return o == this || o instanceof Version && version.equals(((Version) o).version);
- }
-
- @Override
- public int hashCode()
- {
- return version.hashCode();
- }
- }
public static enum Type
{
@@ -140,6 +59,7 @@ public class Descriptor
}
}
+
public final File directory;
/** version has the following format: <code>[a-z]+</code> */
public final Version version;
@@ -147,6 +67,7 @@ public class Descriptor
public final String cfname;
public final int generation;
public final Type type;
+ public final SSTableFormat.Type formatType;
private final int hashCode;
/**
@@ -154,29 +75,41 @@ public class Descriptor
*/
public Descriptor(File directory, String ksname, String cfname, int generation, Type temp)
{
- this(Version.CURRENT, directory, ksname, cfname, generation, temp);
+ this(DatabaseDescriptor.getSSTableFormat().info.getLatestVersion(), directory, ksname, cfname, generation, temp, DatabaseDescriptor.getSSTableFormat());
}
- public Descriptor(String version, File directory, String ksname, String cfname, int generation, Type temp)
+ public Descriptor(File directory, String ksname, String cfname, int generation, Type temp, SSTableFormat.Type formatType)
{
- this(new Version(version), directory, ksname, cfname, generation, temp);
+ this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, temp, formatType);
}
- public Descriptor(Version version, File directory, String ksname, String cfname, int generation, Type temp)
+ public Descriptor(String version, File directory, String ksname, String cfname, int generation, Type temp, SSTableFormat.Type formatType)
{
- assert version != null && directory != null && ksname != null && cfname != null;
+ this(formatType.info.getVersion(version), directory, ksname, cfname, generation, temp, formatType);
+ }
+
+ public Descriptor(Version version, File directory, String ksname, String cfname, int generation, Type temp, SSTableFormat.Type formatType)
+ {
+ assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
this.version = version;
this.directory = directory;
this.ksname = ksname;
this.cfname = cfname;
this.generation = generation;
- type = temp;
- hashCode = Objects.hashCode(directory, generation, ksname, cfname, temp);
+ this.type = temp;
+ this.formatType = formatType;
+
+ hashCode = Objects.hashCode(version, directory, generation, ksname, cfname, temp, formatType);
}
public Descriptor withGeneration(int newGeneration)
{
- return new Descriptor(version, directory, ksname, cfname, newGeneration, type);
+ return new Descriptor(version, directory, ksname, cfname, newGeneration, type, formatType);
+ }
+
+ public Descriptor withFormatType(SSTableFormat.Type newType)
+ {
+ return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, type, newType);
}
public String filenameFor(Component component)
@@ -194,7 +127,7 @@ public class Descriptor
private void appendFileName(StringBuilder buff)
{
- if (!version.newFileName)
+ if (!version.hasNewFileName())
{
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
@@ -203,6 +136,8 @@ public class Descriptor
buff.append(type.marker).append(separator);
buff.append(version).append(separator);
buff.append(generation);
+ if (formatType != SSTableFormat.Type.LEGACY)
+ buff.append(separator).append(formatType.name);
}
public String relativeFilenameFor(Component component)
@@ -213,6 +148,11 @@ public class Descriptor
return buff.toString();
}
+ public SSTableFormat getFormat()
+ {
+ return formatType.info;
+ }
+
/**
* @param suffix A component suffix, such as 'Data.db'/'Index.db'/etc
* @return A filename for this descriptor with the given suffix.
@@ -233,6 +173,11 @@ public class Descriptor
return fromFilename(file.getParentFile(), file.getName(), false).left;
}
+ public static Descriptor fromFilename(String filename, SSTableFormat.Type formatType)
+ {
+ return fromFilename(filename).withFormatType(formatType);
+ }
+
public static Descriptor fromFilename(String filename, boolean skipComponent)
{
File file = new File(filename);
@@ -249,7 +194,7 @@ public class Descriptor
*
* <ul>
* <li><ksname>-<cfname>-(tmp-)?<version>-<gen>-<component> for cassandra 2.0 and before</li>
- * <li>(<tmp marker>-)?<version>-<gen>-<component> for cassandra 2.1 and later</li>
+ * <li>(<tmp marker>-)?<version>-<gen>-<component> for cassandra 3.0 and later</li>
* </ul>
*
* If this is for SSTable of secondary index, directory should ends with index name for 2.1+.
@@ -278,32 +223,42 @@ public class Descriptor
// component suffix
String component = skipComponent ? null : tokenStack.pop();
+ nexttok = tokenStack.pop();
+ // generation OR Type
+ SSTableFormat.Type fmt = SSTableFormat.Type.LEGACY;
+ if (!CharMatcher.DIGIT.matchesAllOf(nexttok))
+ {
+ fmt = SSTableFormat.Type.validate(nexttok);
+ nexttok = tokenStack.pop();
+ }
+
// generation
- int generation = Integer.parseInt(tokenStack.pop());
+ int generation = Integer.parseInt(nexttok);
// version
nexttok = tokenStack.pop();
- if (!Version.validate(nexttok))
+ Version version = fmt.info.getVersion(nexttok);
+
+ if (!version.validate(nexttok))
throw new UnsupportedOperationException("SSTable " + name + " is too old to open. Upgrade to 2.0 first, and run upgradesstables");
- Version version = new Version(nexttok);
// optional temporary marker
- Type type = Type.FINAL;
+ Type type = Descriptor.Type.FINAL;
nexttok = tokenStack.peek();
- if (Type.TEMP.marker.equals(nexttok))
+ if (Descriptor.Type.TEMP.marker.equals(nexttok))
{
- type = Type.TEMP;
+ type = Descriptor.Type.TEMP;
tokenStack.pop();
}
- else if (Type.TEMPLINK.marker.equals(nexttok))
+ else if (Descriptor.Type.TEMPLINK.marker.equals(nexttok))
{
- type = Type.TEMPLINK;
+ type = Descriptor.Type.TEMPLINK;
tokenStack.pop();
}
// ks/cf names
String ksname, cfname;
- if (version.newFileName)
+ if (version.hasNewFileName())
{
// for 2.1+ read ks and cf names from directory
File cfDirectory = parentDirectory;
@@ -332,7 +287,7 @@ public class Descriptor
}
assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory;
- return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, type), component);
+ return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, type, fmt), component);
}
/**
@@ -341,12 +296,12 @@ public class Descriptor
*/
public Descriptor asType(Type type)
{
- return new Descriptor(version, directory, ksname, cfname, generation, type);
+ return new Descriptor(version, directory, ksname, cfname, generation, type, formatType);
}
public IMetadataSerializer getMetadataSerializer()
{
- if (version.newStatsFile)
+ if (version.hasNewStatsFile())
return new MetadataSerializer();
else
return new LegacyMetadataSerializer();
@@ -378,6 +333,7 @@ public class Descriptor
&& that.generation == this.generation
&& that.ksname.equals(this.ksname)
&& that.cfname.equals(this.cfname)
+ && that.formatType == this.formatType
&& that.type == this.type;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/IndexHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java
index b0bbfc4..4518e23 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java
@@ -67,32 +67,6 @@ public class IndexHelper
}
/**
- * Deserialize the index into a structure and return it
- *
- * @param in input source
- * @param type the comparator type for the column family
- *
- * @return ArrayList<IndexInfo> - list of de-serialized indexes
- * @throws IOException if an I/O error occurs.
- */
- public static List<IndexInfo> deserializeIndex(FileDataInput in, CType type) throws IOException
- {
- int columnIndexSize = in.readInt();
- if (columnIndexSize == 0)
- return Collections.<IndexInfo>emptyList();
- ArrayList<IndexInfo> indexList = new ArrayList<IndexInfo>();
- FileMark mark = in.mark();
- ISerializer<IndexInfo> serializer = type.indexSerializer();
- while (in.bytesPastMark(mark) < columnIndexSize)
- {
- indexList.add(serializer.deserialize(in));
- }
- assert in.bytesPastMark(mark) == columnIndexSize;
-
- return indexList;
- }
-
- /**
* The index of the IndexInfo in which a scan starting with @name should begin.
*
* @param name
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
index e39d75d..468b54c 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
@@ -37,6 +37,7 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
index 6cb8653..a1fda57 100644
--- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Iterator;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.IMergeIterator;
import org.apache.cassandra.utils.MergeIterator;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java
index dee024a..50b9545 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTable.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java
@@ -208,7 +208,7 @@ public abstract class SSTable
}
/** @return An estimate of the number of keys contained in the given index file. */
- long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException
+ protected long estimateRowsFromIndex(RandomAccessReader ifile) throws IOException
{
// collect sizes for the first 10000 keys, or first 10 megabytes of data
final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length());
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java b/src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java
index 785e23b..bb84e4c 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java
@@ -24,6 +24,7 @@ import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
index b784a7e..6300749 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
@@ -21,16 +21,19 @@ import java.io.*;
import java.util.Iterator;
import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.FileDataInput;
import org.apache.cassandra.io.util.RandomAccessReader;
import org.apache.cassandra.serializers.MarshalException;
-public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterator>, OnDiskAtomIterator
+ public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterator>, OnDiskAtomIterator
{
private final DecoratedKey key;
private final DataInput in;
- public final long dataSize; // we [still] require this so compaction can tell if it's safe to read the row into memory
public final ColumnSerializer.Flag flag;
private final ColumnFamily columnFamily;
@@ -43,11 +46,10 @@ public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterat
* @param sstable SSTable we are reading ffrom.
* @param file Reading using this file.
* @param key Key of this row.
- * @param dataSize length of row data
*/
- public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key, long dataSize)
+ public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key)
{
- this(sstable, file, key, dataSize, false);
+ this(sstable, file, key, false);
}
/**
@@ -55,21 +57,19 @@ public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterat
* @param sstable SSTable we are reading ffrom.
* @param file Reading using this file.
* @param key Key of this row.
- * @param dataSize length of row data
* @param checkData if true, do its best to deserialize and check the coherence of row data
*/
- public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key, long dataSize, boolean checkData)
+ public SSTableIdentityIterator(SSTableReader sstable, RandomAccessReader file, DecoratedKey key, boolean checkData)
{
- this(sstable.metadata, file, file.getPath(), key, dataSize, checkData, sstable, ColumnSerializer.Flag.LOCAL);
+ this(sstable.metadata, file, file.getPath(), key, checkData, sstable, ColumnSerializer.Flag.LOCAL);
}
// sstable may be null *if* checkData is false
// If it is null, we assume the data is in the current file format
private SSTableIdentityIterator(CFMetaData metadata,
- DataInput in,
+ FileDataInput in,
String filename,
DecoratedKey key,
- long dataSize,
boolean checkData,
SSTableReader sstable,
ColumnSerializer.Flag flag)
@@ -78,11 +78,10 @@ public class SSTableIdentityIterator implements Comparable<SSTableIdentityIterat
this.in = in;
this.filename = filename;
this.key = key;
- this.dataSize = dataSize;
this.flag = flag;
this.validateColumns = checkData;
- Descriptor.Version dataVersion = sstable == null ? Descriptor.Version.CURRENT : sstable.descriptor.version;
+ Version dataVersion = sstable == null ? DatabaseDescriptor.getSSTableFormat().info.getLatestVersion() : sstable.descriptor.version;
int expireBefore = (int) (System.currentTimeMillis() / 1000);
columnFamily = ArrayBackedSortedColumns.factory.create(metadata);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index 6cb7f03..2a1b66f 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@ -33,6 +33,7 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.*;
import org.apache.cassandra.utils.FBUtilities;
[8/8] git commit: Extend Descriptor to include a format value and
refactor reader/writer apis
Posted by ja...@apache.org.
Extend Descriptor to include a format value and refactor reader/writer apis
patch by tjake; reviewed by Marcus Eriksson for CASSANDRA-7443
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0368e97e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0368e97e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0368e97e
Branch: refs/heads/trunk
Commit: 0368e97ee4a807cb832a90c590ae5c65a98730c1
Parents: 7e53db0
Author: Jake Luciani <ja...@apache.org>
Authored: Tue Sep 2 12:49:01 2014 -0400
Committer: T Jake Luciani <ja...@apache.org>
Committed: Thu Oct 23 11:10:55 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 12 +-
.../org/apache/cassandra/config/CFMetaData.java | 10 +-
.../cassandra/config/DatabaseDescriptor.java | 9 +
.../org/apache/cassandra/db/AbstractCell.java | 3 +-
.../apache/cassandra/db/AtomDeserializer.java | 5 +-
.../apache/cassandra/db/BatchlogManager.java | 2 +-
.../cassandra/db/CollationController.java | 2 +-
.../cassandra/db/ColumnFamilySerializer.java | 3 +-
.../apache/cassandra/db/ColumnFamilyStore.java | 31 +-
.../org/apache/cassandra/db/DataTracker.java | 2 +-
.../org/apache/cassandra/db/DeletionInfo.java | 2 +-
src/java/org/apache/cassandra/db/Keyspace.java | 4 +-
src/java/org/apache/cassandra/db/Memtable.java | 13 +-
.../org/apache/cassandra/db/OnDiskAtom.java | 5 +-
.../org/apache/cassandra/db/RangeTombstone.java | 7 +-
.../org/apache/cassandra/db/RowIndexEntry.java | 60 +-
.../apache/cassandra/db/RowIteratorFactory.java | 2 +-
.../org/apache/cassandra/db/SystemKeyspace.java | 2 +-
.../db/columniterator/IndexedSliceReader.java | 499 -----
.../db/columniterator/SSTableNamesIterator.java | 249 ---
.../db/columniterator/SSTableSliceIterator.java | 101 -
.../db/columniterator/SimpleSliceReader.java | 107 -
.../db/commitlog/CommitLogDescriptor.java | 7 +-
.../cassandra/db/commitlog/ReplayPosition.java | 2 +-
.../db/compaction/AbstractCompactedRow.java | 3 +-
.../compaction/AbstractCompactionStrategy.java | 4 +-
.../db/compaction/AbstractCompactionTask.java | 2 +-
.../db/compaction/CompactionController.java | 4 +-
.../db/compaction/CompactionIterable.java | 8 +-
.../db/compaction/CompactionManager.java | 29 +-
.../cassandra/db/compaction/CompactionTask.java | 37 +-
.../DateTieredCompactionStrategy.java | 2 +-
.../db/compaction/LazilyCompactedRow.java | 45 +-
.../compaction/LeveledCompactionStrategy.java | 9 +-
.../db/compaction/LeveledCompactionTask.java | 4 +-
.../db/compaction/LeveledManifest.java | 9 +-
.../db/compaction/SSTableSplitter.java | 3 +-
.../cassandra/db/compaction/Scrubber.java | 13 +-
.../SizeTieredCompactionStrategy.java | 5 +-
.../cassandra/db/compaction/Upgrader.java | 7 +-
.../cassandra/db/composites/AbstractCType.java | 14 -
.../apache/cassandra/db/composites/CType.java | 2 -
.../cassandra/db/filter/IDiskAtomFilter.java | 2 +-
.../cassandra/db/filter/NamesQueryFilter.java | 7 +-
.../apache/cassandra/db/filter/QueryFilter.java | 2 +-
.../cassandra/db/filter/SliceQueryFilter.java | 7 +-
.../cassandra/db/index/SecondaryIndex.java | 2 +-
.../db/index/SecondaryIndexManager.java | 2 +-
.../org/apache/cassandra/dht/BytesToken.java | 1 -
.../apache/cassandra/io/ISSTableSerializer.java | 3 +-
.../io/compress/CompressionMetadata.java | 11 +-
.../io/sstable/AbstractSSTableSimpleWriter.java | 22 +-
.../cassandra/io/sstable/CQLSSTableWriter.java | 9 +-
.../apache/cassandra/io/sstable/Descriptor.java | 172 +-
.../cassandra/io/sstable/IndexHelper.java | 26 -
.../io/sstable/IndexSummaryManager.java | 1 +
.../io/sstable/ReducingKeyIterator.java | 1 +
.../apache/cassandra/io/sstable/SSTable.java | 2 +-
.../io/sstable/SSTableDeletingTask.java | 1 +
.../io/sstable/SSTableIdentityIterator.java | 23 +-
.../cassandra/io/sstable/SSTableLoader.java | 1 +
.../cassandra/io/sstable/SSTableReader.java | 2055 ------------------
.../cassandra/io/sstable/SSTableRewriter.java | 2 +
.../cassandra/io/sstable/SSTableScanner.java | 294 ---
.../io/sstable/SSTableSimpleUnsortedWriter.java | 7 +-
.../io/sstable/SSTableSimpleWriter.java | 1 +
.../cassandra/io/sstable/SSTableWriter.java | 641 ------
.../io/sstable/format/SSTableFormat.java | 90 +
.../io/sstable/format/SSTableReader.java | 1881 ++++++++++++++++
.../io/sstable/format/SSTableWriter.java | 202 ++
.../cassandra/io/sstable/format/Version.java | 104 +
.../io/sstable/format/big/BigFormat.java | 224 ++
.../io/sstable/format/big/BigTableReader.java | 256 +++
.../io/sstable/format/big/BigTableScanner.java | 299 +++
.../io/sstable/format/big/BigTableWriter.java | 541 +++++
.../sstable/format/big/IndexedSliceReader.java | 500 +++++
.../format/big/SSTableNamesIterator.java | 250 +++
.../format/big/SSTableSliceIterator.java | 102 +
.../sstable/format/big/SimpleSliceReader.java | 108 +
.../io/sstable/metadata/CompactionMetadata.java | 3 +-
.../metadata/IMetadataComponentSerializer.java | 3 +-
.../io/sstable/metadata/MetadataCollector.java | 2 +-
.../io/sstable/metadata/StatsMetadata.java | 7 +-
.../io/sstable/metadata/ValidationMetadata.java | 3 +-
.../cassandra/io/util/AbstractDataInput.java | 6 +-
.../io/util/DataIntegrityMetadata.java | 2 +-
.../apache/cassandra/io/util/FileDataInput.java | 1 +
.../org/apache/cassandra/io/util/FileUtils.java | 29 +-
.../cassandra/io/util/MappedFileDataInput.java | 4 +-
.../cassandra/io/util/MemoryInputStream.java | 6 +-
.../cassandra/metrics/ColumnFamilyMetrics.java | 2 +-
.../apache/cassandra/net/MessagingService.java | 3 +-
.../notifications/SSTableAddedNotification.java | 2 +-
.../SSTableDeletingNotification.java | 2 +-
.../SSTableListChangedNotification.java | 3 +-
.../SSTableRepairStatusChanged.java | 2 +-
.../repair/RepairMessageVerbHandler.java | 2 +-
.../cassandra/service/ActiveRepairService.java | 3 +-
.../apache/cassandra/service/CacheService.java | 7 +-
.../cassandra/streaming/StreamLockfile.java | 2 +-
.../cassandra/streaming/StreamReader.java | 33 +-
.../cassandra/streaming/StreamReceiveTask.java | 6 +-
.../cassandra/streaming/StreamSession.java | 2 +-
.../cassandra/streaming/StreamTransferTask.java | 2 +-
.../cassandra/streaming/StreamWriter.java | 2 +-
.../compress/CompressedStreamReader.java | 16 +-
.../compress/CompressedStreamWriter.java | 2 +-
.../streaming/messages/FileMessageHeader.java | 27 +-
.../streaming/messages/IncomingFileMessage.java | 2 +-
.../streaming/messages/OutgoingFileMessage.java | 3 +-
.../streaming/messages/StreamMessage.java | 4 +-
.../apache/cassandra/tools/SSTableExport.java | 8 +-
.../apache/cassandra/tools/SSTableImport.java | 7 +-
.../tools/SSTableRepairedAtSetter.java | 2 +-
.../cassandra/tools/StandaloneScrubber.java | 3 +-
.../cassandra/tools/StandaloneSplitter.java | 3 +-
.../cassandra/tools/StandaloneUpgrader.java | 5 +-
.../utils/vint/EncodedDataInputStream.java | 6 +-
.../db/compaction/LongCompactionsTest.java | 4 +-
.../LongLeveledCompactionStrategyTest.java | 2 +-
test/unit/org/apache/cassandra/Util.java | 2 +-
.../cassandra/cache/AutoSavingCacheTest.java | 2 +-
.../org/apache/cassandra/db/CleanupTest.java | 2 +-
.../cassandra/db/ColumnFamilyStoreTest.java | 33 +-
.../org/apache/cassandra/db/KeyCacheTest.java | 5 +-
.../org/apache/cassandra/db/KeyspaceTest.java | 2 +-
.../apache/cassandra/db/RangeTombstoneTest.java | 3 +-
.../apache/cassandra/db/RowIndexEntryTest.java | 7 +-
.../unit/org/apache/cassandra/db/ScrubTest.java | 5 +-
.../db/compaction/AntiCompactionTest.java | 15 +-
.../compaction/BlacklistingCompactionsTest.java | 2 +-
.../db/compaction/CompactionsPurgeTest.java | 2 +-
.../db/compaction/CompactionsTest.java | 26 +-
.../DateTieredCompactionStrategyTest.java | 3 +-
.../LeveledCompactionStrategyTest.java | 2 +-
.../SizeTieredCompactionStrategyTest.java | 2 +-
.../cassandra/db/compaction/TTLExpiryTest.java | 5 +-
.../cassandra/io/sstable/DescriptorTest.java | 52 +-
.../io/sstable/IndexSummaryManagerTest.java | 1 +
.../cassandra/io/sstable/LegacySSTableTest.java | 15 +-
.../io/sstable/SSTableMetadataTest.java | 1 +
.../cassandra/io/sstable/SSTableReaderTest.java | 1 +
.../io/sstable/SSTableScannerTest.java | 11 +-
.../cassandra/io/sstable/SSTableUtils.java | 9 +-
.../metadata/MetadataSerializerTest.java | 2 +-
.../apache/cassandra/repair/ValidatorTest.java | 3 +-
.../streaming/StreamTransferTaskTest.java | 2 +-
.../streaming/StreamingTransferTest.java | 2 +-
.../cassandra/tools/SSTableExportTest.java | 20 +-
.../cassandra/tools/SSTableImportTest.java | 2 +-
tools/cqlstress-example.yaml | 9 +-
tools/cqlstress-insanity-example.yaml | 2 -
153 files changed, 5202 insertions(+), 4473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0dae098..524e776 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0
+ * Extend Descriptor to include a format value and refactor reader/writer apis (CASSANDRA-7443)
* Integrate JMH for microbenchmarks (CASSANDRA-8151)
* Keep sstable levels when bootstrapping (CASSANDRA-7460)
* Add Sigar library and perform basic OS settings check on startup (CASSANDRA-7838)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 9cd80c2..7a6476c 100644
--- a/build.xml
+++ b/build.xml
@@ -334,7 +334,7 @@
<dependency groupId="com.googlecode.json-simple" artifactId="json-simple" version="1.1"/>
<dependency groupId="com.boundary" artifactId="high-scale-lib" version="1.0.6"/>
<dependency groupId="com.github.jbellis" artifactId="jamm" version="0.2.6"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.7">
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.5">
<exclusion groupId="org.slf4j" artifactId="slf4j-log4j12"/>
</dependency>
<dependency groupId="org.yaml" artifactId="snakeyaml" version="1.11"/>
@@ -418,10 +418,11 @@
<dependency groupId="com.google.code.findbugs" artifactId="jsr305"/>
<dependency groupId="org.antlr" artifactId="antlr"/>
<dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core"/>
- <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>
<dependency groupId="org.javassist" artifactId="javassist"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-core"/>
<dependency groupId="org.openjdk.jmh" artifactId="jmh-generator-annprocess"/>
+ <dependency groupId="net.ju-n.compile-command-annotations" artifactId="compile-command-annotations"/>
+ <dependency groupId="org.javassist" artifactId="javassist" />
</artifact:pom>
<artifact:pom id="coverage-deps-pom"
@@ -475,10 +476,9 @@
<dependency groupId="org.mindrot" artifactId="jbcrypt"/>
<dependency groupId="com.yammer.metrics" artifactId="metrics-core"/>
<dependency groupId="com.addthis.metrics" artifactId="reporter-config"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.7"/>
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.5"/>
<dependency groupId="com.clearspring.analytics" artifactId="stream" version="2.5.2" />
<dependency groupId="net.sf.supercsv" artifactId="super-csv" version="2.1.0" />
- <dependency groupId="org.javassist" artifactId="javassist"/>
<dependency groupId="ch.qos.logback" artifactId="logback-core"/>
<dependency groupId="ch.qos.logback" artifactId="logback-classic"/>
@@ -492,6 +492,7 @@
<dependency groupId="org.apache.pig" artifactId="pig" optional="true"/>
<dependency groupId="com.datastax.cassandra" artifactId="cassandra-driver-core" optional="true"/>
+
<!-- don't need jna to run, but nice to have -->
<dependency groupId="net.java.dev.jna" artifactId="jna" version="4.0.0"/>
@@ -1110,7 +1111,8 @@
<jvmarg value="-Xss256k"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
- <jvmarg value="-Dcassandra.test.offsetseed=@{poffset}"/>
+ <jvmarg value="-Dcassandra.test.offsetseed=@{poffset}"/>
+ <jvmarg value="-Dcassandra.test.sstableformatdevelopment=true"/>
<optjvmargs/>
<classpath>
<path refid="cassandra.classpath" />
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/config/CFMetaData.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java
index 1e2a2e1..cf4d761 100644
--- a/src/java/org/apache/cassandra/config/CFMetaData.java
+++ b/src/java/org/apache/cassandra/config/CFMetaData.java
@@ -44,6 +44,8 @@ import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterables;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.FileDataInput;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -1460,17 +1462,17 @@ public final class CFMetaData
return (cfName + "_" + columnName + "_idx").replaceAll("\\W", "");
}
- public Iterator<OnDiskAtom> getOnDiskIterator(DataInput in, Descriptor.Version version)
+ public Iterator<OnDiskAtom> getOnDiskIterator(FileDataInput in, Version version)
{
return getOnDiskIterator(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
- public Iterator<OnDiskAtom> getOnDiskIterator(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version)
+ public Iterator<OnDiskAtom> getOnDiskIterator(FileDataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version)
{
- return AbstractCell.onDiskIterator(in, flag, expireBefore, version, comparator);
+ return version.getSSTableFormat().getOnDiskIterator(in, flag, expireBefore, this, version);
}
- public AtomDeserializer getOnDiskDeserializer(DataInput in, Descriptor.Version version)
+ public AtomDeserializer getOnDiskDeserializer(DataInput in, Version version)
{
return new AtomDeserializer(comparator, in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 8659c94..00e875b 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -38,6 +38,7 @@ import java.util.UUID;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Longs;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.AllowAllAuthenticator;
@@ -98,6 +99,8 @@ public class DatabaseDescriptor
private static Config conf;
+ private static SSTableFormat.Type sstable_format = SSTableFormat.Type.BIG;
+
private static IAuthenticator authenticator = new AllowAllAuthenticator();
private static IAuthorizer authorizer = new AllowAllAuthorizer();
@@ -1544,6 +1547,12 @@ public class DatabaseDescriptor
return conf.inter_dc_tcp_nodelay;
}
+
+ public static SSTableFormat.Type getSSTableFormat()
+ {
+ return sstable_format;
+ }
+
public static MemtablePool getMemtableAllocatorPool()
{
long heapLimit = ((long) conf.memtable_heap_space_in_mb) << 20;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/AbstractCell.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/AbstractCell.java b/src/java/org/apache/cassandra/db/AbstractCell.java
index f27871f..de86126 100644
--- a/src/java/org/apache/cassandra/db/AbstractCell.java
+++ b/src/java/org/apache/cassandra/db/AbstractCell.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.FBUtilities;
@@ -39,7 +40,7 @@ public abstract class AbstractCell implements Cell
public static Iterator<OnDiskAtom> onDiskIterator(final DataInput in,
final ColumnSerializer.Flag flag,
final int expireBefore,
- final Descriptor.Version version,
+ final Version version,
final CellNameType type)
{
return new AbstractIterator<OnDiskAtom>()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/AtomDeserializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/AtomDeserializer.java b/src/java/org/apache/cassandra/db/AtomDeserializer.java
index 799ed0e..0c43422 100644
--- a/src/java/org/apache/cassandra/db/AtomDeserializer.java
+++ b/src/java/org/apache/cassandra/db/AtomDeserializer.java
@@ -24,6 +24,7 @@ import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
/**
* Helper class to deserialize OnDiskAtom efficiently.
@@ -40,9 +41,9 @@ public class AtomDeserializer
private final DataInput in;
private final ColumnSerializer.Flag flag;
private final int expireBefore;
- private final Descriptor.Version version;
+ private final Version version;
- public AtomDeserializer(CellNameType type, DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version)
+ public AtomDeserializer(CellNameType type, DataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version)
{
this.type = type;
this.nameDeserializer = type.newDeserializer(in);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/BatchlogManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/BatchlogManager.java b/src/java/org/apache/cassandra/db/BatchlogManager.java
index 18d9a17..7f04cba 100644
--- a/src/java/org/apache/cassandra/db/BatchlogManager.java
+++ b/src/java/org/apache/cassandra/db/BatchlogManager.java
@@ -31,6 +31,7 @@ import javax.management.ObjectName;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
import com.google.common.util.concurrent.RateLimiter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +45,6 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessageIn;
import org.apache.cassandra.net.MessageOut;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/CollationController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/CollationController.java b/src/java/org/apache/cassandra/db/CollationController.java
index ff5fe88..922cbfe 100644
--- a/src/java/org/apache/cassandra/db/CollationController.java
+++ b/src/java/org/apache/cassandra/db/CollationController.java
@@ -35,7 +35,7 @@ import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.filter.NamesQueryFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.marshal.CounterColumnType;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.SearchIterator;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java b/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java
index e2aeb6c..29866d6 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilySerializer.java
@@ -25,6 +25,7 @@ import org.apache.cassandra.config.Schema;
import org.apache.cassandra.io.ISSTableSerializer;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.UUIDSerializer;
@@ -146,7 +147,7 @@ public class ColumnFamilySerializer implements IVersionedSerializer<ColumnFamily
throw new UnsupportedOperationException();
}
- public ColumnFamily deserializeFromSSTable(DataInput in, Descriptor.Version version)
+ public ColumnFamily deserializeFromSSTable(DataInput in, Version version)
{
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index aa266be..a23b2ad 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -34,6 +34,10 @@ import com.google.common.util.concurrent.*;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.format.Version;
import org.json.simple.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -441,7 +445,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
generations.add(desc.generation);
if (!desc.isCompatible())
throw new RuntimeException(String.format("Incompatible SSTable found. Current version %s is unable to read file: %s. Please run upgradesstables.",
- Descriptor.Version.CURRENT, desc));
+ desc.getFormat().getLatestVersion(), desc));
}
Collections.sort(generations);
int value = (generations.size() > 0) ? (generations.get(generations.size() - 1)) : 0;
@@ -672,7 +676,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
Set<Descriptor> currentDescriptors = new HashSet<Descriptor>();
for (SSTableReader sstable : data.getView().sstables)
currentDescriptors.add(sstable.descriptor);
- Set<SSTableReader> newSSTables = new HashSet<SSTableReader>();
+ Set<SSTableReader> newSSTables = new HashSet<>();
Directories.SSTableLister lister = directories.sstableLister().skipTemporary(true);
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
@@ -686,8 +690,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
if (!descriptor.isCompatible())
throw new RuntimeException(String.format("Can't open incompatible SSTable! Current version %s, found file: %s",
- Descriptor.Version.CURRENT,
- descriptor));
+ descriptor.getFormat().getLatestVersion(),
+ descriptor));
// force foreign sstables to level 0
try
@@ -711,7 +715,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
descriptor.ksname,
descriptor.cfname,
fileIndexGenerator.incrementAndGet(),
- Descriptor.Type.FINAL);
+ Descriptor.Type.FINAL,
+ descriptor.formatType);
}
while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
@@ -780,17 +785,23 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public String getTempSSTablePath(File directory)
{
- return getTempSSTablePath(directory, Descriptor.Version.CURRENT);
+ return getTempSSTablePath(directory, DatabaseDescriptor.getSSTableFormat().info.getLatestVersion(), DatabaseDescriptor.getSSTableFormat());
}
- private String getTempSSTablePath(File directory, Descriptor.Version version)
+ public String getTempSSTablePath(File directory, SSTableFormat.Type format)
+ {
+ return getTempSSTablePath(directory, format.info.getLatestVersion(), format);
+ }
+
+ private String getTempSSTablePath(File directory, Version version, SSTableFormat.Type format)
{
Descriptor desc = new Descriptor(version,
directory,
keyspace.getName(),
name,
fileIndexGenerator.incrementAndGet(),
- Descriptor.Type.TEMP);
+ Descriptor.Type.TEMP,
+ format);
return desc.filenameFor(Component.DATA);
}
@@ -2193,7 +2204,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public List<SSTableReader> getSnapshotSSTableReader(String tag) throws IOException
{
Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
- List<SSTableReader> readers = new ArrayList<SSTableReader>(snapshots.size());
+ List<SSTableReader> readers = new ArrayList<>(snapshots.size());
for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
readers.add(SSTableReader.open(entries.getKey(), entries.getValue(), metadata, partitioner));
return readers;
@@ -2779,7 +2790,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
assert data.getCompacting().isEmpty() : data.getCompacting();
- List<SSTableReader> truncatedSSTables = new ArrayList<SSTableReader>();
+ List<SSTableReader> truncatedSSTables = new ArrayList<>();
for (SSTableReader sstable : getSSTables())
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/DataTracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DataTracker.java b/src/java/org/apache/cassandra/db/DataTracker.java
index 7393323..d106190 100644
--- a/src/java/org/apache/cassandra/db/DataTracker.java
+++ b/src/java/org/apache/cassandra/db/DataTracker.java
@@ -24,13 +24,13 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.base.Predicate;
import com.google.common.collect.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.dht.AbstractBounds;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.notifications.*;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/DeletionInfo.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DeletionInfo.java b/src/java/org/apache/cassandra/db/DeletionInfo.java
index 3e5e845..048324a 100644
--- a/src/java/org/apache/cassandra/db/DeletionInfo.java
+++ b/src/java/org/apache/cassandra/db/DeletionInfo.java
@@ -147,7 +147,7 @@ public class DeletionInfo implements IMeasurableMemory
/**
* Returns a new {@link InOrderTester} in forward order.
*/
- InOrderTester inOrderTester()
+ public InOrderTester inOrderTester()
{
return inOrderTester(false);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index ca43df6..8986154 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -32,6 +32,7 @@ import java.util.concurrent.Future;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +45,6 @@ import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.pager.QueryPagers;
@@ -260,7 +260,7 @@ public class Keyspace
*/
public List<SSTableReader> getAllSSTables()
{
- List<SSTableReader> list = new ArrayList<SSTableReader>(columnFamilyStores.size());
+ List<SSTableReader> list = new ArrayList<>(columnFamilyStores.size());
for (ColumnFamilyStore cfStore : columnFamilyStores.values())
list.addAll(cfStore.getSSTables());
return list;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/Memtable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java
index a711833..80376f7 100644
--- a/src/java/org/apache/cassandra/db/Memtable.java
+++ b/src/java/org/apache/cassandra/db/Memtable.java
@@ -28,6 +28,9 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import com.google.common.base.Throwables;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,8 +40,6 @@ import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.dht.LongToken;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.DiskAwareRunnable;
import org.apache.cassandra.service.ActiveRepairService;
@@ -384,12 +385,8 @@ public class Memtable
public SSTableWriter createFlushWriter(String filename)
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator).replayPosition(context);
- return new SSTableWriter(filename,
- rows.size(),
- ActiveRepairService.UNREPAIRED_SSTABLE,
- cfs.metadata,
- cfs.partitioner,
- sstableMetadataCollector);
+
+ return SSTableWriter.create(Descriptor.fromFilename(filename), (long) rows.size(), ActiveRepairService.UNREPAIRED_SSTABLE, cfs.metadata, cfs.partitioner, sstableMetadataCollector);
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/OnDiskAtom.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/OnDiskAtom.java b/src/java/org/apache/cassandra/db/OnDiskAtom.java
index b53e43b..f97ca42 100644
--- a/src/java/org/apache/cassandra/db/OnDiskAtom.java
+++ b/src/java/org/apache/cassandra/db/OnDiskAtom.java
@@ -26,6 +26,7 @@ import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.io.ISSTableSerializer;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.serializers.MarshalException;
@@ -65,12 +66,12 @@ public interface OnDiskAtom
}
}
- public OnDiskAtom deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException
+ public OnDiskAtom deserializeFromSSTable(DataInput in, Version version) throws IOException
{
return deserializeFromSSTable(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
- public OnDiskAtom deserializeFromSSTable(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version) throws IOException
+ public OnDiskAtom deserializeFromSSTable(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Version version) throws IOException
{
Composite name = type.serializer().deserialize(in);
if (name.isEmpty())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/RangeTombstone.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/RangeTombstone.java b/src/java/org/apache/cassandra/db/RangeTombstone.java
index 6a246f9..4a0037b 100644
--- a/src/java/org/apache/cassandra/db/RangeTombstone.java
+++ b/src/java/org/apache/cassandra/db/RangeTombstone.java
@@ -27,6 +27,7 @@ import org.apache.cassandra.db.composites.CType;
import org.apache.cassandra.db.composites.Composite;
import org.apache.cassandra.io.ISSTableSerializer;
import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.serializers.MarshalException;
@@ -259,7 +260,7 @@ public class RangeTombstone extends Interval<Composite, DeletionTime> implements
DeletionTime.serializer.serialize(t.data, out);
}
- public RangeTombstone deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException
+ public RangeTombstone deserializeFromSSTable(DataInput in, Version version) throws IOException
{
Composite min = type.serializer().deserialize(in);
@@ -268,14 +269,14 @@ public class RangeTombstone extends Interval<Composite, DeletionTime> implements
return deserializeBody(in, min, version);
}
- public RangeTombstone deserializeBody(DataInput in, Composite min, Descriptor.Version version) throws IOException
+ public RangeTombstone deserializeBody(DataInput in, Composite min, Version version) throws IOException
{
Composite max = type.serializer().deserialize(in);
DeletionTime dt = DeletionTime.serializer.deserialize(in);
return new RangeTombstone(min, max, dt);
}
- public void skipBody(DataInput in, Descriptor.Version version) throws IOException
+ public void skipBody(DataInput in, Version version) throws IOException
{
type.serializer().skip(in);
DeletionTime.serializer.skip(in);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/RowIndexEntry.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java
index 01035c4..4ff61ce 100644
--- a/src/java/org/apache/cassandra/db/RowIndexEntry.java
+++ b/src/java/org/apache/cassandra/db/RowIndexEntry.java
@@ -27,15 +27,14 @@ import java.util.List;
import com.google.common.primitives.Ints;
import org.apache.cassandra.cache.IMeasurableMemory;
-import org.apache.cassandra.db.composites.CType;
import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.IndexHelper;
+import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.ObjectSizes;
-public class RowIndexEntry implements IMeasurableMemory
+public class RowIndexEntry<T> implements IMeasurableMemory
{
private static final long EMPTY_SIZE = ObjectSizes.measure(new RowIndexEntry(0));
@@ -46,12 +45,12 @@ public class RowIndexEntry implements IMeasurableMemory
this.position = position;
}
- protected int promotedSize(CType type)
+ public int promotedSize(ISerializer<T> idxSerializer)
{
return 0;
}
- public static RowIndexEntry create(long position, DeletionTime deletionTime, ColumnIndex index)
+ public static RowIndexEntry<IndexHelper.IndexInfo> create(long position, DeletionTime deletionTime, ColumnIndex index)
{
assert index != null;
assert deletionTime != null;
@@ -62,7 +61,7 @@ public class RowIndexEntry implements IMeasurableMemory
if (index.columnsIndex.size() > 1)
return new IndexedEntry(position, deletionTime, index.columnsIndex);
else
- return new RowIndexEntry(position);
+ return new RowIndexEntry<>(position);
}
/**
@@ -79,7 +78,16 @@ public class RowIndexEntry implements IMeasurableMemory
throw new UnsupportedOperationException();
}
- public List<IndexHelper.IndexInfo> columnsIndex()
+ /**
+ * @return the offset to the start of the header information for this row.
+ * For some formats this may not be the start of the row.
+ */
+ public long headerOffset()
+ {
+ return 0;
+ }
+
+ public List<T> columnsIndex()
{
return Collections.emptyList();
}
@@ -89,31 +97,37 @@ public class RowIndexEntry implements IMeasurableMemory
return EMPTY_SIZE;
}
- public static class Serializer
+ public static interface IndexSerializer<T>
{
- private final CType type;
+ void serialize(RowIndexEntry<T> rie, DataOutputPlus out) throws IOException;
+ RowIndexEntry<T> deserialize(DataInput in, Version version) throws IOException;
+ public int serializedSize(RowIndexEntry<T> rie);
+ }
- public Serializer(CType type)
+ public static class Serializer implements IndexSerializer<IndexHelper.IndexInfo>
+ {
+ private final ISerializer<IndexHelper.IndexInfo> idxSerializer;
+
+ public Serializer(ISerializer<IndexHelper.IndexInfo> idxSerializer)
{
- this.type = type;
+ this.idxSerializer = idxSerializer;
}
- public void serialize(RowIndexEntry rie, DataOutputPlus out) throws IOException
+ public void serialize(RowIndexEntry<IndexHelper.IndexInfo> rie, DataOutputPlus out) throws IOException
{
out.writeLong(rie.position);
- out.writeInt(rie.promotedSize(type));
+ out.writeInt(rie.promotedSize(idxSerializer));
if (rie.isIndexed())
{
DeletionTime.serializer.serialize(rie.deletionTime(), out);
out.writeInt(rie.columnsIndex().size());
- ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
for (IndexHelper.IndexInfo info : rie.columnsIndex())
idxSerializer.serialize(info, out);
}
}
- public RowIndexEntry deserialize(DataInput in, Descriptor.Version version) throws IOException
+ public RowIndexEntry<IndexHelper.IndexInfo> deserialize(DataInput in, Version version) throws IOException
{
long position = in.readLong();
@@ -123,8 +137,7 @@ public class RowIndexEntry implements IMeasurableMemory
DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
int entries = in.readInt();
- ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
- List<IndexHelper.IndexInfo> columnsIndex = new ArrayList<IndexHelper.IndexInfo>(entries);
+ List<IndexHelper.IndexInfo> columnsIndex = new ArrayList<>(entries);
for (int i = 0; i < entries; i++)
columnsIndex.add(idxSerializer.deserialize(in));
@@ -132,7 +145,7 @@ public class RowIndexEntry implements IMeasurableMemory
}
else
{
- return new RowIndexEntry(position);
+ return new RowIndexEntry<>(position);
}
}
@@ -151,9 +164,9 @@ public class RowIndexEntry implements IMeasurableMemory
FileUtils.skipBytesFully(in, size);
}
- public int serializedSize(RowIndexEntry rie)
+ public int serializedSize(RowIndexEntry<IndexHelper.IndexInfo> rie)
{
- int size = TypeSizes.NATIVE.sizeof(rie.position) + TypeSizes.NATIVE.sizeof(rie.promotedSize(type));
+ int size = TypeSizes.NATIVE.sizeof(rie.position) + TypeSizes.NATIVE.sizeof(rie.promotedSize(idxSerializer));
if (rie.isIndexed())
{
@@ -162,11 +175,11 @@ public class RowIndexEntry implements IMeasurableMemory
size += DeletionTime.serializer.serializedSize(rie.deletionTime(), TypeSizes.NATIVE);
size += TypeSizes.NATIVE.sizeof(index.size());
- ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
for (IndexHelper.IndexInfo info : index)
size += idxSerializer.serializedSize(info, TypeSizes.NATIVE);
}
+
return size;
}
}
@@ -174,7 +187,7 @@ public class RowIndexEntry implements IMeasurableMemory
/**
* An entry in the row index for a row whose columns are indexed.
*/
- private static class IndexedEntry extends RowIndexEntry
+ private static class IndexedEntry extends RowIndexEntry<IndexHelper.IndexInfo>
{
private final DeletionTime deletionTime;
private final List<IndexHelper.IndexInfo> columnsIndex;
@@ -204,12 +217,11 @@ public class RowIndexEntry implements IMeasurableMemory
}
@Override
- public int promotedSize(CType type)
+ public int promotedSize(ISerializer<IndexHelper.IndexInfo> idxSerializer)
{
TypeSizes typeSizes = TypeSizes.NATIVE;
long size = DeletionTime.serializer.serializedSize(deletionTime, typeSizes);
size += typeSizes.sizeof(columnsIndex.size()); // number of entries
- ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
for (IndexHelper.IndexInfo info : columnsIndex)
size += idxSerializer.serializedSize(info, typeSizes);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/RowIteratorFactory.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/RowIteratorFactory.java b/src/java/org/apache/cassandra/db/RowIteratorFactory.java
index 5bd2d9b..6ac74ae 100644
--- a/src/java/org/apache/cassandra/db/RowIteratorFactory.java
+++ b/src/java/org/apache/cassandra/db/RowIteratorFactory.java
@@ -26,7 +26,7 @@ import org.apache.cassandra.db.columniterator.LazyColumnIterator;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.MergeIterator;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/SystemKeyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index bfd92e9..5c0d935 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -29,6 +29,7 @@ import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,6 @@ import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.metrics.RestorableMeter;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
deleted file mode 100644
index 7012321..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
+++ /dev/null
@@ -1,499 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.List;
-
-import com.google.common.collect.AbstractIterator;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.composites.Composite;
-import org.apache.cassandra.db.filter.ColumnSlice;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.IndexHelper;
-import org.apache.cassandra.io.sstable.IndexHelper.IndexInfo;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileMark;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-/**
- * This is a reader that finds the block for a starting column and returns blocks before/after it for each next call.
- * This function assumes that the CF is sorted by name and exploits the name index.
- */
-class IndexedSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
-{
- private final ColumnFamily emptyColumnFamily;
-
- private final SSTableReader sstable;
- private final List<IndexHelper.IndexInfo> indexes;
- private final FileDataInput originalInput;
- private FileDataInput file;
- private final boolean reversed;
- private final ColumnSlice[] slices;
- private final BlockFetcher fetcher;
- private final Deque<OnDiskAtom> blockColumns = new ArrayDeque<OnDiskAtom>();
- private final CellNameType comparator;
-
- // Holds range tombstone in reverse queries. See addColumn()
- private final Deque<OnDiskAtom> rangeTombstonesReversed;
-
- /**
- * This slice reader assumes that slices are sorted correctly, e.g. that for forward lookup slices are in
- * lexicographic order of start elements and that for reverse lookup they are in reverse lexicographic order of
- * finish (reverse start) elements. i.e. forward: [a,b],[d,e],[g,h] reverse: [h,g],[e,d],[b,a]. This reader also
- * assumes that validation has been performed in terms of intervals (no overlapping intervals).
- */
- public IndexedSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, ColumnSlice[] slices, boolean reversed)
- {
- Tracing.trace("Seeking to partition indexed section in data file");
- this.sstable = sstable;
- this.originalInput = input;
- this.reversed = reversed;
- this.slices = slices;
- this.comparator = sstable.metadata.comparator;
- this.rangeTombstonesReversed = reversed ? new ArrayDeque<OnDiskAtom>() : null;
-
- try
- {
- this.indexes = indexEntry.columnsIndex();
- emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata);
- if (indexes.isEmpty())
- {
- setToRowStart(indexEntry, input);
- emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file));
- fetcher = new SimpleBlockFetcher();
- }
- else
- {
- emptyColumnFamily.delete(indexEntry.deletionTime());
- fetcher = new IndexedBlockFetcher(indexEntry.position);
- }
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, file.getPath());
- }
- }
-
- /**
- * Sets the seek position to the start of the row for column scanning.
- */
- private void setToRowStart(RowIndexEntry rowEntry, FileDataInput in) throws IOException
- {
- if (in == null)
- {
- this.file = sstable.getFileDataInput(rowEntry.position);
- }
- else
- {
- this.file = in;
- in.seek(rowEntry.position);
- }
- sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file));
- }
-
- public ColumnFamily getColumnFamily()
- {
- return emptyColumnFamily;
- }
-
- public DecoratedKey getKey()
- {
- throw new UnsupportedOperationException();
- }
-
- protected OnDiskAtom computeNext()
- {
- while (true)
- {
- if (reversed)
- {
- // Return all tombstone for the block first (see addColumn() below)
- OnDiskAtom column = rangeTombstonesReversed.poll();
- if (column != null)
- return column;
- }
-
- OnDiskAtom column = blockColumns.poll();
- if (column == null)
- {
- if (!fetcher.fetchMoreData())
- return endOfData();
- }
- else
- {
- return column;
- }
- }
- }
-
- public void close() throws IOException
- {
- if (originalInput == null && file != null)
- file.close();
- }
-
- protected void addColumn(OnDiskAtom col)
- {
- if (reversed)
- {
- /*
- * We put range tomstone markers at the beginning of the range they delete. But for reversed queries,
- * the caller still need to know about a RangeTombstone before it sees any column that it covers.
- * To make that simple, we keep said tombstones separate and return them all before any column for
- * a given block.
- */
- if (col instanceof RangeTombstone)
- rangeTombstonesReversed.addFirst(col);
- else
- blockColumns.addFirst(col);
- }
- else
- {
- blockColumns.addLast(col);
- }
- }
-
- private abstract class BlockFetcher
- {
- protected int currentSliceIdx;
-
- protected BlockFetcher(int sliceIdx)
- {
- this.currentSliceIdx = sliceIdx;
- }
-
- /*
- * Return the smallest key selected by the current ColumnSlice.
- */
- protected Composite currentStart()
- {
- return reversed ? slices[currentSliceIdx].finish : slices[currentSliceIdx].start;
- }
-
- /*
- * Return the biggest key selected by the current ColumnSlice.
- */
- protected Composite currentFinish()
- {
- return reversed ? slices[currentSliceIdx].start : slices[currentSliceIdx].finish;
- }
-
- protected abstract boolean setNextSlice();
-
- protected abstract boolean fetchMoreData();
-
- protected boolean isColumnBeforeSliceStart(OnDiskAtom column)
- {
- return isBeforeSliceStart(column.name());
- }
-
- protected boolean isBeforeSliceStart(Composite name)
- {
- Composite start = currentStart();
- return !start.isEmpty() && comparator.compare(name, start) < 0;
- }
-
- protected boolean isColumnBeforeSliceFinish(OnDiskAtom column)
- {
- Composite finish = currentFinish();
- return finish.isEmpty() || comparator.compare(column.name(), finish) <= 0;
- }
-
- protected boolean isAfterSliceFinish(Composite name)
- {
- Composite finish = currentFinish();
- return !finish.isEmpty() && comparator.compare(name, finish) > 0;
- }
- }
-
- private class IndexedBlockFetcher extends BlockFetcher
- {
- // where this row starts
- private final long columnsStart;
-
- // the index entry for the next block to deserialize
- private int nextIndexIdx = -1;
-
- // index of the last block we've read from disk;
- private int lastDeserializedBlock = -1;
-
- // For reversed, keep columns at the beginning of the last deserialized block that
- // may still match a slice
- private final Deque<OnDiskAtom> prefetched;
-
- public IndexedBlockFetcher(long columnsStart)
- {
- super(-1);
- this.columnsStart = columnsStart;
- this.prefetched = reversed ? new ArrayDeque<OnDiskAtom>() : null;
- setNextSlice();
- }
-
- protected boolean setNextSlice()
- {
- while (++currentSliceIdx < slices.length)
- {
- nextIndexIdx = IndexHelper.indexFor(slices[currentSliceIdx].start, indexes, comparator, reversed, nextIndexIdx);
- if (nextIndexIdx < 0 || nextIndexIdx >= indexes.size())
- // no index block for that slice
- continue;
-
- // Check if we can exclude this slice entirely from the index
- IndexInfo info = indexes.get(nextIndexIdx);
- if (reversed)
- {
- if (!isBeforeSliceStart(info.lastName))
- return true;
- }
- else
- {
- if (!isAfterSliceFinish(info.firstName))
- return true;
- }
- }
- nextIndexIdx = -1;
- return false;
- }
-
- protected boolean hasMoreSlice()
- {
- return currentSliceIdx < slices.length;
- }
-
- protected boolean fetchMoreData()
- {
- if (!hasMoreSlice())
- return false;
-
- // If we read blocks in reversed disk order, we may have columns from the previous block to handle.
- // Note that prefetched keeps columns in reversed disk order.
- if (reversed && !prefetched.isEmpty())
- {
- boolean gotSome = false;
- // Avoids some comparison when we know it's not useful
- boolean inSlice = false;
-
- OnDiskAtom prefetchedCol;
- while ((prefetchedCol = prefetched.peek() ) != null)
- {
- // col is before slice, we update the slice
- if (isColumnBeforeSliceStart(prefetchedCol))
- {
- inSlice = false;
- if (!setNextSlice())
- return false;
- }
- // col is within slice, all columns
- // (we go in reverse, so as soon as we are in a slice, no need to check
- // we're after the slice until we change slice)
- else if (inSlice || isColumnBeforeSliceFinish(prefetchedCol))
- {
- blockColumns.addLast(prefetched.poll());
- gotSome = true;
- inSlice = true;
- }
- // if col is after slice, ignore
- else
- {
- prefetched.poll();
- }
- }
- if (gotSome)
- return true;
- }
- try
- {
- return getNextBlock();
- }
- catch (IOException e)
- {
- throw new CorruptSSTableException(e, file.getPath());
- }
- }
-
- private boolean getNextBlock() throws IOException
- {
- if (lastDeserializedBlock == nextIndexIdx)
- {
- if (reversed)
- nextIndexIdx--;
- else
- nextIndexIdx++;
- }
- lastDeserializedBlock = nextIndexIdx;
-
- // Are we done?
- if (lastDeserializedBlock < 0 || lastDeserializedBlock >= indexes.size())
- return false;
-
- IndexInfo currentIndex = indexes.get(lastDeserializedBlock);
-
- /* seek to the correct offset to the data, and calculate the data size */
- long positionToSeek = columnsStart + currentIndex.offset;
-
- // With new promoted indexes, our first seek in the data file will happen at that point.
- if (file == null)
- file = originalInput == null ? sstable.getFileDataInput(positionToSeek) : originalInput;
-
- AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
-
- file.seek(positionToSeek);
- FileMark mark = file.mark();
-
- // We remenber when we are whithin a slice to avoid some comparison
- boolean inSlice = false;
-
- // scan from index start
- while (file.bytesPastMark(mark) < currentIndex.width || deserializer.hasUnprocessed())
- {
- // col is before slice
- // (If in slice, don't bother checking that until we change slice)
- Composite start = currentStart();
- if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0)
- {
- if (reversed)
- {
- // the next slice select columns that are before the current one, so it may
- // match this column, so keep it around.
- prefetched.addFirst(deserializer.readNext());
- }
- else
- {
- deserializer.skipNext();
- }
- }
- // col is within slice
- else
- {
- Composite finish = currentFinish();
- if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0)
- {
- inSlice = true;
- addColumn(deserializer.readNext());
- }
- // col is after slice.
- else
- {
- // When reading forward, if we hit a column that sorts after the current slice, it means we're done with this slice.
- // For reversed, this may either mean that we're done with the current slice, or that we need to read the previous
- // index block. However, we can be sure that we are in the first case though (the current slice is done) if the first
- // columns of the block were not part of the current slice, i.e. if we have columns in prefetched.
- if (reversed && prefetched.isEmpty())
- break;
-
- if (!setNextSlice())
- break;
-
- inSlice = false;
-
- // The next index block now corresponds to the first block that may have columns for the newly set slice.
- // So if it's different from the current block, we're done with this block. And in that case, we know
- // that our prefetched columns won't match.
- if (nextIndexIdx != lastDeserializedBlock)
- {
- if (reversed)
- prefetched.clear();
- break;
- }
-
- // Even if the next slice may have column in this blocks, if we're reversed, those columns have been
- // prefetched and we're done with that block
- if (reversed)
- break;
-
- // otherwise, we will deal with that column at the next iteration
- }
- }
- }
- return true;
- }
- }
-
- private class SimpleBlockFetcher extends BlockFetcher
- {
- public SimpleBlockFetcher() throws IOException
- {
- // Since we have to deserialize in order and will read all slices might as well reverse the slices and
- // behave as if it was not reversed
- super(reversed ? slices.length - 1 : 0);
-
- // We remenber when we are whithin a slice to avoid some comparison
- boolean inSlice = false;
-
- AtomDeserializer deserializer = emptyColumnFamily.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
- while (deserializer.hasNext())
- {
- // col is before slice
- // (If in slice, don't bother checking that until we change slice)
- Composite start = currentStart();
- if (!inSlice && !start.isEmpty() && deserializer.compareNextTo(start) < 0)
- {
- deserializer.skipNext();
- continue;
- }
-
- // col is within slice
- Composite finish = currentFinish();
- if (finish.isEmpty() || deserializer.compareNextTo(finish) <= 0)
- {
- inSlice = true;
- addColumn(deserializer.readNext());
- }
- // col is after slice. more slices?
- else
- {
- inSlice = false;
- if (!setNextSlice())
- break;
- }
- }
- }
-
- protected boolean setNextSlice()
- {
- if (reversed)
- {
- if (currentSliceIdx <= 0)
- return false;
-
- currentSliceIdx--;
- }
- else
- {
- if (currentSliceIdx >= slices.length - 1)
- return false;
-
- currentSliceIdx++;
- }
- return true;
- }
-
- protected boolean fetchMoreData()
- {
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java
deleted file mode 100644
index 224b63f..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.*;
-
-import com.google.common.collect.AbstractIterator;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.CellName;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.IndexHelper;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.io.util.FileMark;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-public class SSTableNamesIterator extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
-{
- private ColumnFamily cf;
- private final SSTableReader sstable;
- private FileDataInput fileToClose;
- private Iterator<OnDiskAtom> iter;
- public final SortedSet<CellName> columns;
- public final DecoratedKey key;
-
- public SSTableNamesIterator(SSTableReader sstable, DecoratedKey key, SortedSet<CellName> columns)
- {
- assert columns != null;
- this.sstable = sstable;
- this.columns = columns;
- this.key = key;
-
- RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
- if (indexEntry == null)
- return;
-
- try
- {
- read(sstable, null, indexEntry);
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- finally
- {
- if (fileToClose != null)
- FileUtils.closeQuietly(fileToClose);
- }
- }
-
- public SSTableNamesIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, SortedSet<CellName> columns, RowIndexEntry indexEntry)
- {
- assert columns != null;
- this.sstable = sstable;
- this.columns = columns;
- this.key = key;
-
- try
- {
- read(sstable, file, indexEntry);
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- }
-
- private FileDataInput createFileDataInput(long position)
- {
- fileToClose = sstable.getFileDataInput(position);
- return fileToClose;
- }
-
- private void read(SSTableReader sstable, FileDataInput file, RowIndexEntry indexEntry)
- throws IOException
- {
- List<IndexHelper.IndexInfo> indexList;
-
- // If the entry is not indexed or the index is not promoted, read from the row start
- if (!indexEntry.isIndexed())
- {
- if (file == null)
- file = createFileDataInput(indexEntry.position);
- else
- file.seek(indexEntry.position);
-
- DecoratedKey keyInDisk = sstable.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(file));
- assert keyInDisk.equals(key) : String.format("%s != %s in %s", keyInDisk, key, file.getPath());
- }
-
- indexList = indexEntry.columnsIndex();
-
- if (!indexEntry.isIndexed())
- {
- ColumnFamilySerializer serializer = ColumnFamily.serializer;
- try
- {
- cf = ArrayBackedSortedColumns.factory.create(sstable.metadata);
- cf.delete(DeletionTime.serializer.deserialize(file));
- }
- catch (Exception e)
- {
- throw new IOException(serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e);
- }
- }
- else
- {
- cf = ArrayBackedSortedColumns.factory.create(sstable.metadata);
- cf.delete(indexEntry.deletionTime());
- }
-
- List<OnDiskAtom> result = new ArrayList<OnDiskAtom>();
- if (indexList.isEmpty())
- {
- readSimpleColumns(file, columns, result);
- }
- else
- {
- readIndexedColumns(sstable.metadata, file, columns, indexList, indexEntry.position, result);
- }
-
- // create an iterator view of the columns we read
- iter = result.iterator();
- }
-
- private void readSimpleColumns(FileDataInput file, SortedSet<CellName> columnNames, List<OnDiskAtom> result)
- {
- Iterator<OnDiskAtom> atomIterator = cf.metadata().getOnDiskIterator(file, sstable.descriptor.version);
- int n = 0;
- while (atomIterator.hasNext())
- {
- OnDiskAtom column = atomIterator.next();
- if (column instanceof Cell)
- {
- if (columnNames.contains(column.name()))
- {
- result.add(column);
- if (++n >= columns.size())
- break;
- }
- }
- else
- {
- result.add(column);
- }
- }
- }
-
- private void readIndexedColumns(CFMetaData metadata,
- FileDataInput file,
- SortedSet<CellName> columnNames,
- List<IndexHelper.IndexInfo> indexList,
- long basePosition,
- List<OnDiskAtom> result)
- throws IOException
- {
- /* get the various column ranges we have to read */
- CellNameType comparator = metadata.comparator;
- List<IndexHelper.IndexInfo> ranges = new ArrayList<IndexHelper.IndexInfo>();
- int lastIndexIdx = -1;
- for (CellName name : columnNames)
- {
- int index = IndexHelper.indexFor(name, indexList, comparator, false, lastIndexIdx);
- if (index < 0 || index == indexList.size())
- continue;
- IndexHelper.IndexInfo indexInfo = indexList.get(index);
- // Check the index block does contain the column names and that we haven't inserted this block yet.
- if (comparator.compare(name, indexInfo.firstName) < 0 || index == lastIndexIdx)
- continue;
-
- ranges.add(indexInfo);
- lastIndexIdx = index;
- }
-
- if (ranges.isEmpty())
- return;
-
- Iterator<CellName> toFetch = columnNames.iterator();
- CellName nextToFetch = toFetch.next();
- for (IndexHelper.IndexInfo indexInfo : ranges)
- {
- long positionToSeek = basePosition + indexInfo.offset;
-
- // With new promoted indexes, our first seek in the data file will happen at that point.
- if (file == null)
- file = createFileDataInput(positionToSeek);
-
- AtomDeserializer deserializer = cf.metadata().getOnDiskDeserializer(file, sstable.descriptor.version);
- file.seek(positionToSeek);
- FileMark mark = file.mark();
- while (file.bytesPastMark(mark) < indexInfo.width && nextToFetch != null)
- {
- int cmp = deserializer.compareNextTo(nextToFetch);
- if (cmp == 0)
- {
- nextToFetch = toFetch.hasNext() ? toFetch.next() : null;
- result.add(deserializer.readNext());
- continue;
- }
-
- deserializer.skipNext();
- if (cmp > 0)
- nextToFetch = toFetch.hasNext() ? toFetch.next() : null;
- }
- }
- }
-
- public DecoratedKey getKey()
- {
- return key;
- }
-
- public ColumnFamily getColumnFamily()
- {
- return cf;
- }
-
- protected OnDiskAtom computeNext()
- {
- if (iter == null || !iter.hasNext())
- return endOfData();
- return iter.next();
- }
-
- public void close() throws IOException { }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java
deleted file mode 100644
index 0057d52..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-
-import org.apache.cassandra.db.ColumnFamily;
-import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.OnDiskAtom;
-import org.apache.cassandra.db.RowIndexEntry;
-import org.apache.cassandra.db.filter.ColumnSlice;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-
-/**
- * A Cell Iterator over SSTable
- */
-public class SSTableSliceIterator implements OnDiskAtomIterator
-{
- private final OnDiskAtomIterator reader;
- private final DecoratedKey key;
-
- public SSTableSliceIterator(SSTableReader sstable, DecoratedKey key, ColumnSlice[] slices, boolean reversed)
- {
- this.key = key;
- RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
- this.reader = indexEntry == null ? null : createReader(sstable, indexEntry, null, slices, reversed);
- }
-
- /**
- * An iterator for a slice within an SSTable
- * @param sstable Keyspace for the CFS we are reading from
- * @param file Optional parameter that input is read from. If null is passed, this class creates an appropriate one automatically.
- * If this class creates, it will close the underlying file when #close() is called.
- * If a caller passes a non-null argument, this class will NOT close the underlying file when the iterator is closed (i.e. the caller is responsible for closing the file)
- * In all cases the caller should explicitly #close() this iterator.
- * @param key The key the requested slice resides under
- * @param slices the column slices
- * @param reversed Results are returned in reverse order iff reversed is true.
- * @param indexEntry position of the row
- */
- public SSTableSliceIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, ColumnSlice[] slices, boolean reversed, RowIndexEntry indexEntry)
- {
- this.key = key;
- reader = createReader(sstable, indexEntry, file, slices, reversed);
- }
-
- private static OnDiskAtomIterator createReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput file, ColumnSlice[] slices, boolean reversed)
- {
- return slices.length == 1 && slices[0].start.isEmpty() && !reversed
- ? new SimpleSliceReader(sstable, indexEntry, file, slices[0].finish)
- : new IndexedSliceReader(sstable, indexEntry, file, slices, reversed);
- }
-
- public DecoratedKey getKey()
- {
- return key;
- }
-
- public ColumnFamily getColumnFamily()
- {
- return reader == null ? null : reader.getColumnFamily();
- }
-
- public boolean hasNext()
- {
- return reader != null && reader.hasNext();
- }
-
- public OnDiskAtom next()
- {
- return reader.next();
- }
-
- public void remove()
- {
- throw new UnsupportedOperationException();
- }
-
- public void close() throws IOException
- {
- if (reader != null)
- reader.close();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/columniterator/SimpleSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/SimpleSliceReader.java b/src/java/org/apache/cassandra/db/columniterator/SimpleSliceReader.java
deleted file mode 100644
index bdbf4bd..0000000
--- a/src/java/org/apache/cassandra/db/columniterator/SimpleSliceReader.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.columniterator;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import com.google.common.collect.AbstractIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.composites.Composite;
-import org.apache.cassandra.io.sstable.CorruptSSTableException;
-import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.io.util.FileDataInput;
-import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.ByteBufferUtil;
-
-class SimpleSliceReader extends AbstractIterator<OnDiskAtom> implements OnDiskAtomIterator
-{
- private static final Logger logger = LoggerFactory.getLogger(SimpleSliceReader.class);
-
- private final FileDataInput file;
- private final boolean needsClosing;
- private final Composite finishColumn;
- private final CellNameType comparator;
- private final ColumnFamily emptyColumnFamily;
- private final Iterator<OnDiskAtom> atomIterator;
-
- public SimpleSliceReader(SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, Composite finishColumn)
- {
- Tracing.trace("Seeking to partition beginning in data file");
- this.finishColumn = finishColumn;
- this.comparator = sstable.metadata.comparator;
- try
- {
- if (input == null)
- {
- this.file = sstable.getFileDataInput(indexEntry.position);
- this.needsClosing = true;
- }
- else
- {
- this.file = input;
- input.seek(indexEntry.position);
- this.needsClosing = false;
- }
-
- // Skip key and data size
- ByteBufferUtil.skipShortLength(file);
-
- emptyColumnFamily = ArrayBackedSortedColumns.factory.create(sstable.metadata);
- emptyColumnFamily.delete(DeletionTime.serializer.deserialize(file));
- atomIterator = emptyColumnFamily.metadata().getOnDiskIterator(file, sstable.descriptor.version);
- }
- catch (IOException e)
- {
- sstable.markSuspect();
- throw new CorruptSSTableException(e, sstable.getFilename());
- }
- }
-
- protected OnDiskAtom computeNext()
- {
- if (!atomIterator.hasNext())
- return endOfData();
-
- OnDiskAtom column = atomIterator.next();
- if (!finishColumn.isEmpty() && comparator.compare(column.name(), finishColumn) > 0)
- return endOfData();
-
- return column;
- }
-
- public ColumnFamily getColumnFamily()
- {
- return emptyColumnFamily;
- }
-
- public void close() throws IOException
- {
- if (needsClosing)
- file.close();
- }
-
- public DecoratedKey getKey()
- {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
index e50a585..f914c2c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
@@ -28,6 +28,7 @@ import java.nio.ByteBuffer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.PureJavaCrc32;
@@ -43,11 +44,13 @@ public class CommitLogDescriptor
public static final int VERSION_12 = 2;
public static final int VERSION_20 = 3;
public static final int VERSION_21 = 4;
+ public static final int VERSION_30 = 5;
/**
* Increment this number if there is a changes in the commit log disc layout or MessagingVersion changes.
* Note: make sure to handle {@link #getMessagingVersion()}
*/
- public static final int current_version = VERSION_21;
+ @VisibleForTesting
+ public static final int current_version = VERSION_30;
// [version, id, checksum]
static final int HEADER_SIZE = 4 + 8 + 4;
@@ -126,6 +129,8 @@ public class CommitLogDescriptor
return MessagingService.VERSION_20;
case VERSION_21:
return MessagingService.VERSION_21;
+ case VERSION_30:
+ return MessagingService.VERSION_30;
default:
throw new IllegalStateException("Unknown commitlog version " + version);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0368e97e/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
index 31fc28e..ca1969f 100644
--- a/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
+++ b/src/java/org/apache/cassandra/db/commitlog/ReplayPosition.java
@@ -27,7 +27,7 @@ import com.google.common.collect.Ordering;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.ISerializer;
-import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataOutputPlus;
public class ReplayPosition implements Comparable<ReplayPosition>