You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2018/03/12 08:35:39 UTC
[4/4] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Merge branch 'cassandra-3.11' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c8718d49
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c8718d49
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c8718d49
Branch: refs/heads/trunk
Commit: c8718d4974bab6ba96d48052b406adc5e66a8108
Parents: 8f840a3 9b022b8
Author: Mick Semb Wever <mc...@apache.org>
Authored: Mon Mar 12 19:26:52 2018 +1100
Committer: Mick Semb Wever <mc...@apache.org>
Committed: Mon Mar 12 19:33:47 2018 +1100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 2 +
.../apache/cassandra/db/ColumnFamilyStore.java | 4 +-
.../io/sstable/format/SSTableReader.java | 91 ++++++++++++++------
.../cassandra/io/sstable/SSTableReaderTest.java | 88 ++++++++++++++++++-
5 files changed, 156 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c8718d49/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index c791e47,6fd9c15..3cf9d93
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,208 -1,8 +1,209 @@@
+4.0
+ * Use Murmur3 for validation compactions (CASSANDRA-14002)
+ * Comma at the end of the seed list is interpretated as localhost (CASSANDRA-14285)
+ * Refactor read executor and response resolver, abstract read repair (CASSANDRA-14058)
+ * Add optional startup delay to wait until peers are ready (CASSANDRA-13993)
+ * Add a few options to nodetool verify (CASSANDRA-14201)
+ * CVE-2017-5929 Security vulnerability and redefine default log rotation policy (CASSANDRA-14183)
+ * Use JVM default SSL validation algorithm instead of custom default (CASSANDRA-13259)
+ * Better document in code InetAddressAndPort usage post 7544, incorporate port into UUIDGen node (CASSANDRA-14226)
+ * Fix sstablemetadata date string for minLocalDeletionTime (CASSANDRA-14132)
+ * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214)
+ * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174)
+ * Add nodetool clientlist (CASSANDRA-13665)
+ * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211)
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
+ * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
+ * Migrate dtests to use pytest and python3 (CASSANDRA-14134)
+ * Allow storage port to be configurable per node (CASSANDRA-7544)
+ * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182)
+ * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067)
+ * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152)
+ * Delete temp test files on exit (CASSANDRA-14153)
+ * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867)
+ * Fix CommitLogReplayer exception for CDC data (CASSANDRA-14066)
+ * Fix cassandra-stress startup failure (CASSANDRA-14106)
+ * Remove initialDirectories from CFS (CASSANDRA-13928)
+ * Fix trivial log format error (CASSANDRA-14015)
+ * Allow sstabledump to do a json object per partition (CASSANDRA-13848)
+ * Add option to optimise merkle tree comparison across replicas (CASSANDRA-3200)
+ * Remove unused and deprecated methods from AbstractCompactionStrategy (CASSANDRA-14081)
+ * Fix Distribution.average in cassandra-stress (CASSANDRA-14090)
+ * Support a means of logging all queries as they were invoked (CASSANDRA-13983)
+ * Presize collections (CASSANDRA-13760)
+ * Add GroupCommitLogService (CASSANDRA-13530)
+ * Parallelize initial materialized view build (CASSANDRA-12245)
+ * Fix flaky SecondaryIndexManagerTest.assert[Not]MarkedAsBuilt (CASSANDRA-13965)
+ * Make LWTs send resultset metadata on every request (CASSANDRA-13992)
+ * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963)
+ * Introduce leaf-only iterator (CASSANDRA-9988)
+ * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997)
+ * Allow only one concurrent call to StatusLogger (CASSANDRA-12182)
+ * Refactoring to specialised functional interfaces (CASSANDRA-13982)
+ * Speculative retry should allow more friendly params (CASSANDRA-13876)
+ * Throw exception if we send/receive repair messages to incompatible nodes (CASSANDRA-13944)
+ * Replace usages of MessageDigest with Guava's Hasher (CASSANDRA-13291)
+ * Add nodetool cmd to print hinted handoff window (CASSANDRA-13728)
+ * Fix some alerts raised by static analysis (CASSANDRA-13799)
+ * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593)
+ * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786)
+ * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941)
+ * Expose recent histograms in JmxHistograms (CASSANDRA-13642)
+ * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899)
+ * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906)
+ * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925)
+ * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961)
+ * Correctly close netty channels when a stream session ends (CASSANDRA-13905)
+ * Update lz4 to 1.4.0 (CASSANDRA-13741)
+ * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862)
+ * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299)
+ * Use compaction threshold for STCS in L0 (CASSANDRA-13861)
+ * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703)
+ * Add extra information to SASI timeout exception (CASSANDRA-13677)
+ * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818)
+ * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786)
+ * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846)
+ * Add keyspace and table name in schema validation exception (CASSANDRA-13845)
+ * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771)
+ * Make netty EventLoopGroups daemon threads (CASSANDRA-13837)
+ * Race condition when closing stream sessions (CASSANDRA-13852)
+ * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831)
+ * Allow changing log levels via nodetool for related classes (CASSANDRA-12696)
+ * Add stress profile yaml with LWT (CASSANDRA-7960)
+ * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789)
+ * Simplify mx4j configuration (Cassandra-13578)
+ * Fix trigger example on 4.0 (CASSANDRA-13796)
+ * Force minumum timeout value (CASSANDRA-9375)
+ * Use netty for streaming (CASSANDRA-12229)
+ * Use netty for internode messaging (CASSANDRA-8457)
+ * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774)
+ * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758)
+ * Fix pending repair manager index out of bounds check (CASSANDRA-13769)
+ * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576)
+ * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664)
+ * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594)
+ * Fix race / ref leak in anticompaction (CASSANDRA-13688)
+ * Expose tasks queue length via JMX (CASSANDRA-12758)
+ * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751)
+ * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615)
+ * Improve sstablemetadata output (CASSANDRA-11483)
+ * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371)
+ * Introduce error metrics for repair (CASSANDRA-13387)
+ * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732)
+ * Update metrics to 3.1.5 (CASSANDRA-13648)
+ * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699)
+ * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725)
+ * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727)
+ * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996)
+ * Default for start_native_transport now true if not set in config (CASSANDRA-13656)
+ * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583)
+ * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148)
+ * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271)
+ * Use common nowInSec for validation compactions (CASSANDRA-13671)
+ * Improve handling of IR prepare failures (CASSANDRA-13672)
+ * Send IR coordinator messages synchronously (CASSANDRA-13673)
+ * Flush system.repair table before IR finalize promise (CASSANDRA-13660)
+ * Fix column filter creation for wildcard queries (CASSANDRA-13650)
+ * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614)
+ * fix race condition in PendingRepairManager (CASSANDRA-13659)
+ * Allow noop incremental repair state transitions (CASSANDRA-13658)
+ * Run repair with down replicas (CASSANDRA-10446)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130)
+ * Improve calculation of available disk space for compaction (CASSANDRA-13068)
+ * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579)
+ * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570)
+ * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585)
+ * Fix Randomness of stress values (CASSANDRA-12744)
+ * Allow selecting Map values and Set elements (CASSANDRA-7396)
+ * Fast and garbage-free Streaming Histogram (CASSANDRA-13444)
+ * Update repairTime for keyspaces on completion (CASSANDRA-13539)
+ * Add configurable upper bound for validation executor threads (CASSANDRA-13521)
+ * Bring back maxHintTTL propery (CASSANDRA-12982)
+ * Add testing guidelines (CASSANDRA-13497)
+ * Add more repair metrics (CASSANDRA-13531)
+ * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650)
+ * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367)
+ * Log time elapsed for each incremental repair phase (CASSANDRA-13498)
+ * Add multiple table operation support to cassandra-stress (CASSANDRA-8780)
+ * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262)
+ * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421)
+ * Improve incremental repair logging (CASSANDRA-13468)
+ * Start compaction when incremental repair finishes (CASSANDRA-13454)
+ * Add repair streaming preview (CASSANDRA-13257)
+ * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430)
+ * Change protocol to allow sending key space independent of query string (CASSANDRA-10145)
+ * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661)
+ * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354)
+ * Skip building views during base table streams on range movements (CASSANDRA-13065)
+ * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197)
+ * Remove deprecated repair JMX APIs (CASSANDRA-11530)
+ * Fix version check to enable streaming keep-alive (CASSANDRA-12929)
+ * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289)
+ * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324)
+ * Upgrade junit from 4.6 to 4.12 (CASSANDRA-13360)
+ * Cleanup ParentRepairSession after repairs (CASSANDRA-13359)
+ * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336)
+ * Incremental repair not streaming correct sstables (CASSANDRA-13328)
+ * Upgrade the jna version to 4.3.0 (CASSANDRA-13300)
+ * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132)
+ * Remove config option index_interval (CASSANDRA-10671)
+ * Reduce lock contention for collection types and serializers (CASSANDRA-13271)
+ * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283)
+ * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292)
+ * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520)
+ * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226)
+ * Remove unused method (CASSANDRA-13227)
+ * Fix minor bugs related to #9143 (CASSANDRA-13217)
+ * Output warning if user increases RF (CASSANDRA-13079)
+ * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081)
+ * Add support for + and - operations on dates (CASSANDRA-11936)
+ * Fix consistency of incrementally repaired data (CASSANDRA-9143)
+ * Increase commitlog version (CASSANDRA-13161)
+ * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425)
+ * Refactor ColumnCondition (CASSANDRA-12981)
+ * Parallelize streaming of different keyspaces (CASSANDRA-4663)
+ * Improved compactions metrics (CASSANDRA-13015)
+ * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
+ * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
+ * Thrift removal (CASSANDRA-11115)
+ * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
+ * Add column definition kind to dropped columns in schema (CASSANDRA-12705)
+ * Add (automate) Nodetool Documentation (CASSANDRA-12672)
+ * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+ * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422)
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
+ * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
+ * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720)
+ * Trivial format error in StorageProxy (CASSANDRA-13551)
+ * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480)
+ * Anticompaction can cause noisy log messages (CASSANDRA-13684)
+ * Switch to client init for sstabledump (CASSANDRA-13683)
+ * CQLSH: Don't pause when capturing data (CASSANDRA-13743)
+ * nodetool clearsnapshot requires --all to clear all snapshots (CASSANDRA-13391)
+ * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527)
+ * cqlshrc.sample uses incorrect option for time formatting (CASSANDRA-14243)
+
+
3.11.3
- * Don't regenerate bloomfilter and summaries on startup (CASSANDRA-11163)
- * RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
+ * RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
* Fix wildcard GROUP BY queries (CASSANDRA-14209)
Merged from 3.0:
++ * Don't regenerate bloomfilter and summaries on startup (CASSANDRA-11163)
* Fix NPE when performing comparison against a null frozen in LWT (CASSANDRA-14087)
* Log when SSTables are deleted (CASSANDRA-14302)
* Fix batch commitlog sync regression (CASSANDRA-14292)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c8718d49/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index 5826cf2,9af8439..bcac4ea
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -155,6 -49,8 +155,8 @@@ Upgradin
---------
- Materialized view users upgrading from 3.0.15 (3.0.X series) or 3.11.1 (3.11.X series) and later that have performed range movements (join, decommission, move, etc),
should run repair on the base tables, and subsequently on the views to ensure data affected by CASSANDRA-14251 is correctly propagated to all replicas.
+ - Changes to bloom_filter_fp_chance will no longer take effect on existing sstables when the node is restarted. Only
- compactions/upgradesstables regenerates bloom filters and Summaries sstable components. See CASSANDRA-11163
++ compactions/upgradesstables regenerates bloom filters and Summaries sstable components. See CASSANDRA-11163
3.11.2
======
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c8718d49/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c8718d49/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 757b9a8,7d7fbf1..95856fc
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -369,21 -380,21 +369,21 @@@ public abstract class SSTableReader ext
return open(desc, componentsFor(desc), metadata);
}
- public static SSTableReader open(Descriptor descriptor, Set<Component> components, CFMetaData metadata) throws IOException
+ public static SSTableReader open(Descriptor descriptor, Set<Component> components, TableMetadataRef metadata)
{
- return open(descriptor, components, metadata, true, true);
+ return open(descriptor, components, metadata, true, false);
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs) throws IOException
+ public static SSTableReader openNoValidation(Descriptor descriptor, Set<Component> components, ColumnFamilyStore cfs)
{
- return open(descriptor, components, cfs.metadata, false, false); // do not track hotness
+ return open(descriptor, components, cfs.metadata, false, true);
}
// use only for offline or "Standalone" operations
- public static SSTableReader openNoValidation(Descriptor descriptor, CFMetaData metadata) throws IOException
+ public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata)
{
- return open(descriptor, componentsFor(descriptor), metadata, false, false); // do not track hotness
+ return open(descriptor, componentsFor(descriptor), metadata, false, true);
}
/**
@@@ -466,11 -460,22 +466,22 @@@
}
}
+ /**
+ * Open an SSTable for reading
+ * @param descriptor SSTable to open
+ * @param components Components included with this SSTable
+ * @param metadata for this SSTables CF
+ * @param validate Check SSTable for corruption (limited)
+ * @param isOffline Whether we are opening this SSTable "offline", for example from an external tool or not for inclusion in queries (validations)
+ * This stops regenerating BF + Summaries and also disables tracking of hotness for the SSTable.
+ * @return {@link SSTableReader}
+ * @throws IOException
+ */
public static SSTableReader open(Descriptor descriptor,
Set<Component> components,
- CFMetaData metadata,
+ TableMetadataRef metadata,
boolean validate,
- boolean trackHotness)
- boolean isOffline) throws IOException
++ boolean isOffline)
{
// Minimum components without which we can't do anything
assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor;
@@@ -701,15 -718,21 +712,21 @@@
return keyCache != null;
}
- private void load(ValidationMetadata validation) throws IOException
+ /**
+ * See {@link #load(boolean, boolean)}
+ * @param validation Metadata for SSTable being loaded
+ * @param isOffline Whether the SSTable is being loaded by an offline tool (sstabledump, scrub, etc)
+ * @throws IOException
+ */
+ private void load(ValidationMetadata validation, boolean isOffline) throws IOException
{
- if (metadata.params.bloomFilterFpChance == 1.0)
+ if (metadata().params.bloomFilterFpChance == 1.0)
{
// bf is disabled.
- load(false, true);
+ load(false, !isOffline);
bf = FilterFactory.AlwaysPresent;
}
- else if (!components.contains(Component.PRIMARY_INDEX))
+ else if (!components.contains(Component.PRIMARY_INDEX)) // What happens if filter component and primary index is missing?
{
// avoid any reading of the missing primary index component.
// this should only happen during StandaloneScrubber
@@@ -728,8 -748,8 +742,8 @@@
else
{
// bf is enabled and fp chance matches the currently configured value.
- load(false, true);
+ load(false, !isOffline);
- loadBloomFilter(descriptor.version.hasOldBfHashOrder());
+ loadBloomFilter(descriptor.version.hasOldBfFormat());
}
}
@@@ -947,6 -968,30 +964,30 @@@
}
}
+ public void saveBloomFilter()
+ {
+ saveBloomFilter(this.descriptor, bf);
+ }
+
+ public static void saveBloomFilter(Descriptor descriptor, IFilter filter)
+ {
+ File filterFile = new File(descriptor.filenameFor(Component.FILTER));
+ try (DataOutputStreamPlus stream = new BufferedDataOutputStreamPlus(new FileOutputStream(filterFile)))
+ {
- FilterFactory.serialize(filter, stream);
++ BloomFilterSerializer.serialize((BloomFilter) filter, stream);
+ stream.flush();
+ }
+ catch (IOException e)
+ {
+ logger.trace("Cannot save SSTable bloomfilter: ", e);
+
+ // corrupted hence delete it and let it load it now.
+ if (filterFile.exists())
+ FileUtils.deleteWithConfirm(filterFile);
+ }
+
+ }
+
public void setReplaced()
{
synchronized (tidy.global)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c8718d49/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 4b2a2ba,7fba926..0ebf0a2
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@@ -48,11 -51,13 +51,15 @@@ import org.apache.cassandra.io.util.Fil
import org.apache.cassandra.io.util.MmappedRegions;
import org.apache.cassandra.schema.CachingParams;
import org.apache.cassandra.schema.KeyspaceParams;
++import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FilterFactory;
import org.apache.cassandra.utils.Pair;
+ import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.junit.Assert.assertEquals;
++import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@@ -350,12 -355,12 +357,11 @@@ public class SSTableReaderTes
lastKey = key;
- new RowUpdateBuilder(store.metadata, timestamp, key.getKey())
- .clustering("col")
- .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
- .build()
- .applyUnsafe();
-
+ new RowUpdateBuilder(store.metadata(), timestamp, key.getKey())
+ .clustering("col")
+ .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
+ .build()
+ .applyUnsafe();
-
}
store.forceBlockingFlush();
@@@ -364,10 -369,88 +370,86 @@@
// test to see if sstable can be opened as expected
SSTableReader target = SSTableReader.open(desc);
-- Assert.assertEquals(target.getIndexSummarySize(), 1);
-- Assert.assertArrayEquals(ByteBufferUtil.getArray(firstKey.getKey()), target.getIndexSummaryKey(0));
assert target.first.equals(firstKey);
assert target.last.equals(lastKey);
+
+ executeInternal(String.format("ALTER TABLE \"%s\".\"%s\" WITH bloom_filter_fp_chance = 0.3", ks, cf));
+
+ File summaryFile = new File(desc.filenameFor(Component.SUMMARY));
+ Path bloomPath = new File(desc.filenameFor(Component.FILTER)).toPath();
+ Path summaryPath = summaryFile.toPath();
+
+ long bloomModified = Files.getLastModifiedTime(bloomPath).toMillis();
+ long summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
+
+ Thread.sleep(TimeUnit.MILLISECONDS.toMillis(10)); // sleep to ensure modified time will be different
+
+ // Offline tests
+ // check that bloomfilter/summary ARE NOT regenerated
+ target = SSTableReader.openNoValidation(desc, store.metadata);
+
+ assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
+ assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
+
+ target.selfRef().release();
+
+ // check that bloomfilter/summary ARE NOT regenerated and BF=AlwaysPresent when filter component is missing
+ Set<Component> components = SSTable.discoverComponentsFor(desc);
+ components.remove(Component.FILTER);
+ target = SSTableReader.openNoValidation(desc, components, store);
+
+ assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
+ assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
+ assertEquals(FilterFactory.AlwaysPresent, target.getBloomFilter());
+
+ target.selfRef().release();
+
+ // #### online tests ####
+ // check that summary & bloomfilter are not regenerated when SSTable is opened and BFFP has been changed
+ target = SSTableReader.open(desc, store.metadata);
+
+ assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
+ assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
+
+ target.selfRef().release();
+
+ // check that bloomfilter is recreated when it doesn't exist and this causes the summary to be recreated
+ components = SSTable.discoverComponentsFor(desc);
+ components.remove(Component.FILTER);
+
+ target = SSTableReader.open(desc, components, store.metadata);
+
+ assertTrue("Bloomfilter was not recreated", bloomModified < Files.getLastModifiedTime(bloomPath).toMillis());
+ assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis());
+
+ target.selfRef().release();
+
+ // check that only the summary is regenerated when it is deleted
+ components.add(Component.FILTER);
+ summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
+ summaryFile.delete();
+
+ Thread.sleep(TimeUnit.MILLISECONDS.toMillis(10)); // sleep to ensure modified time will be different
+ bloomModified = Files.getLastModifiedTime(bloomPath).toMillis();
+
+ target = SSTableReader.open(desc, components, store.metadata);
+
+ assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
+ assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis());
+
+ target.selfRef().release();
+
+ // check that summary and bloomfilter is not recreated when the INDEX is missing
+ components.add(Component.SUMMARY);
+ components.remove(Component.PRIMARY_INDEX);
+
+ summaryModified = Files.getLastModifiedTime(summaryPath).toMillis();
+ target = SSTableReader.open(desc, components, store.metadata, false, false);
+
+ Thread.sleep(TimeUnit.MILLISECONDS.toMillis(10)); // sleep to ensure modified time will be different
+ assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis());
+ assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis());
+
target.selfRef().release();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org