You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by pa...@apache.org on 2018/02/11 13:26:07 UTC
[22/29] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c231ed5b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c231ed5b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c231ed5b
Branch: refs/heads/cassandra-3.11
Commit: c231ed5be0f47b030181185f544132523a2cf908
Parents: 21978bf 1602e60
Author: Paulo Motta <pa...@apache.org>
Authored: Sat Feb 10 14:55:27 2018 -0200
Committer: Paulo Motta <pa...@apache.org>
Committed: Sat Feb 10 14:57:53 2018 -0200
----------------------------------------------------------------------
CASSANDRA-14092.txt | 81 +++++
CHANGES.txt | 1 +
NEWS.txt | 21 ++
conf/jvm.options | 11 +
debian/rules | 2 +-
redhat/cassandra.spec | 2 +-
.../org/apache/cassandra/cql3/Attributes.java | 14 +-
.../cql3/statements/ModificationStatement.java | 2 +-
.../apache/cassandra/db/ColumnFamilyStore.java | 8 +-
.../db/ExpirationDateOverflowHandling.java | 121 +++++++
.../org/apache/cassandra/db/LegacyLayout.java | 7 +-
.../org/apache/cassandra/db/LivenessInfo.java | 12 +-
.../db/compaction/CompactionManager.java | 16 +-
.../cassandra/db/compaction/Scrubber.java | 198 ++++++++++-
.../apache/cassandra/db/rows/BufferCell.java | 7 +-
src/java/org/apache/cassandra/db/rows/Cell.java | 9 +
.../cassandra/service/StorageService.java | 7 +-
.../cassandra/service/StorageServiceMBean.java | 3 +
.../cassandra/thrift/ThriftValidation.java | 8 +-
.../org/apache/cassandra/tools/NodeProbe.java | 8 +-
.../cassandra/tools/SSTableMetadataViewer.java | 1 +
.../cassandra/tools/StandaloneScrubber.java | 12 +-
.../apache/cassandra/tools/nodetool/Scrub.java | 8 +-
.../table1/mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../table1/mc-1-big-Data.db | Bin 0 -> 58 bytes
.../table1/mc-1-big-Digest.crc32 | 1 +
.../table1/mc-1-big-Filter.db | Bin 0 -> 16 bytes
.../table1/mc-1-big-Index.db | Bin 0 -> 16 bytes
.../table1/mc-1-big-Statistics.db | Bin 0 -> 4676 bytes
.../table1/mc-1-big-Summary.db | Bin 0 -> 56 bytes
.../table1/mc-1-big-TOC.txt | 8 +
.../table2/mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../table2/mc-1-big-Data.db | Bin 0 -> 60 bytes
.../table2/mc-1-big-Digest.crc32 | 1 +
.../table2/mc-1-big-Filter.db | Bin 0 -> 16 bytes
.../table2/mc-1-big-Index.db | Bin 0 -> 16 bytes
.../table2/mc-1-big-Statistics.db | Bin 0 -> 4686 bytes
.../table2/mc-1-big-Summary.db | Bin 0 -> 56 bytes
.../table2/mc-1-big-TOC.txt | 8 +
.../table3/mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../table3/mc-1-big-Data.db | Bin 0 -> 111 bytes
.../table3/mc-1-big-Digest.crc32 | 1 +
.../table3/mc-1-big-Filter.db | Bin 0 -> 16 bytes
.../table3/mc-1-big-Index.db | Bin 0 -> 16 bytes
.../table3/mc-1-big-Statistics.db | Bin 0 -> 4732 bytes
.../table3/mc-1-big-Summary.db | Bin 0 -> 56 bytes
.../table3/mc-1-big-TOC.txt | 8 +
.../table4/mc-1-big-CompressionInfo.db | Bin 0 -> 43 bytes
.../table4/mc-1-big-Data.db | Bin 0 -> 108 bytes
.../table4/mc-1-big-Digest.crc32 | 1 +
.../table4/mc-1-big-Filter.db | Bin 0 -> 16 bytes
.../table4/mc-1-big-Index.db | Bin 0 -> 16 bytes
.../table4/mc-1-big-Statistics.db | Bin 0 -> 4742 bytes
.../table4/mc-1-big-Summary.db | Bin 0 -> 56 bytes
.../table4/mc-1-big-TOC.txt | 8 +
.../cql3/validation/operations/TTLTest.java | 327 ++++++++++++++++++-
.../unit/org/apache/cassandra/db/ScrubTest.java | 6 +-
57 files changed, 867 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/CASSANDRA-14092.txt
----------------------------------------------------------------------
diff --cc CASSANDRA-14092.txt
index 0000000,5ac872c..f95380b
mode 000000,100644..100644
--- a/CASSANDRA-14092.txt
+++ b/CASSANDRA-14092.txt
@@@ -1,0 -1,81 +1,81 @@@
+ CASSANDRA-14092: MAXIMUM TTL EXPIRATION DATE
+ ---------------------------------------------
+
+ The maximum expiration timestamp that can be represented by the storage engine is
+ 2038-01-19T03:14:06+00:00, which means that INSERTS using TTL that would expire
+ after this date are not currently supported.
+
+ # Expiration Date Overflow Policy
+
+ We plan to lift this limitation in newer versions, but while the fix is not available,
+ operators can decide which policy to apply when dealing with inserts with TTL exceeding
+ the maximum supported expiration date:
+ - REJECT: this is the default policy and will reject any requests with expiration
+ date timestamp after 2038-01-19T03:14:06+00:00.
+ - CAP: any insert with TTL expiring after 2038-01-19T03:14:06+00:00 will expire on
+ 2038-01-19T03:14:06+00:00 and the client will receive a warning.
+ - CAP_NOWARN: same as previous, except that the client warning will not be emitted.
+
+ These policies may be specified via the -Dcassandra.expiration_date_overflow_policy=POLICY
-startup option.
++startup option in the jvm.options configuration file.
+
+ # Potential data loss on earlier versions
+
+ Prior to 3.0.16 (3.0.X) and 3.11.2 (3.11.x), there was no protection against
+ INSERTS with TTL expiring after the maximum supported date, causing the expiration
+ time field to overflow and the records to expire immediately. Expired records due
+ to overflow will not be queryable and will be permanently removed after a compaction.
+
+ 2.1.X, 2.2.X and earlier series are not subject to this bug when assertions are enabled
+ since an AssertionError is thrown during INSERT when the expiration time field overflows
+ on these versions. When assertions are disabled then it is possible to INSERT entries
+ with overflowed local expiration time and even the earlier versions are subject to data
+ loss due to this bug.
+
+ This issue only affected INSERTs with very large TTLs, close to the maximum allowed value
+ of 630720000 seconds (20 years), starting from 2018-01-19T03:14:06+00:00. As time progresses,
+ the maximum supported TTL will be gradually reduced as the maximum expiration date approaches.
+ For instance, a user on an affected version on 2028-01-19T03:14:06 with a TTL of 10 years
+ will be affected by this bug, so we urge users of very large TTLs to upgrade to a version
+ where this issue is addressed as soon as possible.
+
+ # Data Recovery
+
+ SSTables from Cassandra versions prior to 2.1.20/2.2.12/3.0.16/3.11.2 containing entries
+ with overflowed expiration time that were backed up or did not go through compaction can
+ be recovered by reinserting overflowed entries with a valid expiration time and a higher
+ timestamp, since tombstones may have been generated with the original timestamp.
+
+ To find out if an SSTable has an entry with overflowed expiration, inspect it with the
-sstable2json tool and look for a negative "local deletion time" field. SSTables in this
-condition should be backed up immediately, as they are subject to data loss during
++'sstablemetadata' tool and look for a negative "Minimum local deletion time" field. SSTables
++in this condition should be backed up immediately, as they are subject to data loss during
+ compaction.
+
+ A "--reinsert-overflowed-ttl" option was added to scrub to rewrite SSTables containing
+ rows with overflowed expiration time with the maximum expiration date of
+ 2038-01-19T03:14:06+00:00 and the original timestamp + 1 (ms). Two methods are offered
+ for recovery of SSTables via scrub:
+
+ - Offline scrub:
+ - Clone the data directory tree to another location, keeping only the folders and the
+ contents of the system tables.
+ - Clone the configuration directory to another location, setting the data_file_directories
+ property to the cloned data directory in the cloned cassandra.yaml.
+ - Copy the affected SSTables to the cloned data location of the affected table.
+ - Set the environment variable CASSANDRA_CONF=<cloned configuration directory>.
+ - Execute "sstablescrub --reinsert-overflowed-ttl <keyspace> <table>".
+ WARNING: not specifying --reinsert-overflowed-ttl is equivalent to a single-sstable
+ compaction, so the data with overflowed will be removed - make sure to back up
+ your SSTables before running scrub.
+ - Once the scrub is completed, copy the resulting SSTables to the original data directory.
+ - Execute "nodetool refresh" in a live node to load the recovered SSTables.
+
+ - Online scrub:
+ - Disable compaction on the node with "nodetool disableautocompaction" - this step is crucial
+ as otherwise, the data may be removed permanently during compaction.
+ - Copy the SSTables containing entries with overflowed expiration time to the data directory.
+ - run "nodetool refresh" to load the SSTables.
+ - run "nodetool scrub --reinsert-overflowed-ttl <keyspace> <table>".
+ - Re-enable compactions after verifying that scrub recovered the missing entries.
+
-See https://issues.apache.org/jira/browse/CASSANDRA-14092 for more details about this issue.
++See https://issues.apache.org/jira/browse/CASSANDRA-14092 for more details about this issue.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index f42f3f4,82da6ad..a492c42
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,29 -1,4 +1,30 @@@
-2.2.12
+3.0.16
++ * Protect against overflow of local expiration time (CASSANDRA-14092)
+ * Use the correct digest file and reload sstable metadata in nodetool verify (CASSANDRA-14217)
+ * Handle failure when mutating repaired status in Verifier (CASSANDRA-13933)
+ * Close socket on error during connect on OutboundTcpConnection (CASSANDRA-9630)
+ * Set encoding for javadoc generation (CASSANDRA-14154)
+ * Fix index target computation for dense composite tables with dropped compact storage (CASSANDRA-14104)
+ * Improve commit log chain marker updating (CASSANDRA-14108)
+ * Extra range tombstone bound creates double rows (CASSANDRA-14008)
+ * Fix SStable ordering by max timestamp in SinglePartitionReadCommand (CASSANDRA-14010)
+ * Accept role names containing forward-slash (CASSANDRA-14088)
+ * Optimize CRC check chance probability calculations (CASSANDRA-14094)
+ * Fix cleanup on keyspace with no replicas (CASSANDRA-13526)
+ * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071)
+ * Reduce garbage created by DynamicSnitch (CASSANDRA-14091)
+ * More frequent commitlog chained markers (CASSANDRA-13987)
+ * Fix serialized size of DataLimits (CASSANDRA-14057)
+ * Add flag to allow dropping oversized read repair mutations (CASSANDRA-13975)
+ * Fix SSTableLoader logger message (CASSANDRA-14003)
+ * Fix repair race that caused gossip to block (CASSANDRA-13849)
+ * Tracing interferes with digest requests when using RandomPartitioner (CASSANDRA-13964)
+ * Add flag to disable materialized views, and warnings on creation (CASSANDRA-13959)
+ * Don't let user drop or generally break tables in system_distributed (CASSANDRA-13813)
+ * Provide a JMX call to sync schema with local storage (CASSANDRA-13954)
+ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939)
+ * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
+Merged from 2.2:
* Fix the inspectJvmOptions startup check (CASSANDRA-14112)
* Fix race that prevents submitting compaction for a table when executor is full (CASSANDRA-13801)
* Rely on the JVM to handle OutOfMemoryErrors (CASSANDRA-13006)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index 1bbe1aa,4fe3508..f574c33
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,71 -33,7 +33,72 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-2.2.12
+3.0.16
+=====
+
+Upgrading
+---------
++ - See MAXIMUM TTL EXPIRATION DATE NOTICE above.
+ - Cassandra is now relying on the JVM options to properly shutdown on OutOfMemoryError. By default it will
+ rely on the OnOutOfMemoryError option as the ExitOnOutOfMemoryError and CrashOnOutOfMemoryError options
+ are not supported by the older 1.7 and 1.8 JVMs. A warning will be logged at startup if none of those JVM
+ options are used. See CASSANDRA-13006 for more details.
+ - Cassandra is not logging anymore by default an Heap histogram on OutOfMemoryError. To enable that behavior
+ set the 'cassandra.printHeapHistogramOnOutOfMemoryError' System property to 'true'. See CASSANDRA-13006
+ for more details.
+
+Materialized Views
+-------------------
+ - Following a discussion regarding concerns about the design and safety of Materialized Views, the C* development
+ community no longer recommends them for production use, and considers them experimental. Warnings messages will
+ now be logged when they are created. (See https://www.mail-archive.com/dev@cassandra.apache.org/msg11511.html)
+ - An 'enable_materialized_views' flag has been added to cassandra.yaml to allow operators to prevent creation of
+ views
+
+3.0.15
+=====
+
+Upgrading
+---------
+ - Nothing specific to this release, but please see previous upgrading sections,
+ especially if you are upgrading from 2.2.
+
+Compact Storage
+---------------
+ - Starting version 4.0, Thrift and COMPACT STORAGE is no longer supported.
+ 'ALTER ... DROP COMPACT STORAGE' statement makes Compact Tables CQL-compatible,
+ exposing internal structure of Thrift/Compact Tables. You can find more details
+ on exposed internal structure under:
+ http://cassandra.apache.org/doc/latest/cql/appendices.html#appendix-c-dropping-compact-storage
+
+ For uninterrupted cluster upgrades, drivers now support 'NO_COMPACT' startup option.
+ Supplying this flag will have same effect as 'DROP COMPACT STORAGE', but only for the
+ current connection.
+
+ In order to upgrade, clients supporting a non-compact schema view can be rolled out
+ gradually. When all the clients are updated 'ALTER ... DROP COMPACT STORAGE' can be
+ executed. After dropping compact storage, ’NO_COMPACT' option will have no effect
+ after that.
+
+Materialized Views
+-------------------
+ - Cassandra will no longer allow dropping columns on tables with Materialized Views.
+ - A change was made in the way the Materialized View timestamp is computed, which
+ may cause an old deletion to a base column which is view primary key (PK) column
+ to not be reflected in the view when repairing the base table post-upgrade. This
+ condition is only possible when a column deletion to an MV primary key (PK) column
+ not present in the base table PK (via UPDATE base SET view_pk_col = null or DELETE
+ view_pk_col FROM base) is missed before the upgrade and received by repair after the upgrade.
+ If such column deletions are done on a view PK column which is not a base PK, it's advisable
+ to run repair on the base table of all nodes prior to the upgrade. Alternatively it's possible
+ to fix potential inconsistencies by running repair on the views after upgrade or drop and
+ re-create the views. See CASSANDRA-11500 for more details.
+ - Removal of columns not selected in the Materialized View (via UPDATE base SET unselected_column
+ = null or DELETE unselected_column FROM base) may not be properly reflected in the view in some
+ situations so we advise against doing deletions on base columns not selected in views
+ until this is fixed on CASSANDRA-13826.
+
+3.0.14
======
Upgrading
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/conf/jvm.options
----------------------------------------------------------------------
diff --cc conf/jvm.options
index a7b3bd8,0000000..eb2ad19
mode 100644,000000..100644
--- a/conf/jvm.options
+++ b/conf/jvm.options
@@@ -1,108 -1,0 +1,119 @@@
+###########################################################################
+# jvm.options #
+# #
+# - all flags defined here will be used by cassandra to startup the JVM #
+# - one flag should be specified per line #
+# - lines that do not start with '-' will be ignored #
+# - only static flags are accepted (no variables or parameters) #
+# - dynamic flags will be appended to these on cassandra-env #
+###########################################################################
+
+#################
+# HEAP SETTINGS #
+#################
+
+# Heap size is automatically calculated by cassandra-env based on this
+# formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
+# That is:
+# - calculate 1/2 ram and cap to 1024MB
+# - calculate 1/4 ram and cap to 8192MB
+# - pick the max
+#
+# For production use you may wish to adjust this for your environment.
+# If that's the case, uncomment the -Xmx and Xms options below to override the
+# automatic calculation of JVM heap memory.
+#
+# It is recommended to set min (-Xms) and max (-Xmx) heap sizes to
+# the same value to avoid stop-the-world GC pauses during resize, and
+# so that we can lock the heap in memory on startup to prevent any
+# of it from being swapped out.
+#-Xms4G
+#-Xmx4G
+
+# Young generation size is automatically calculated by cassandra-env
+# based on this formula: min(100 * num_cores, 1/4 * heap size)
+#
+# The main trade-off for the young generation is that the larger it
+# is, the longer GC pause times will be. The shorter it is, the more
+# expensive GC will be (usually).
+#
+# It is not recommended to set the young generation size if using the
+# G1 GC, since that will override the target pause-time goal.
+# More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html
+#
+# The example below assumes a modern 8-core+ machine for decent
+# times. If in doubt, and if you do not particularly want to tweak, go
+# 100 MB per physical CPU core.
+#-Xmn800M
+
++###################################
++# EXPIRATION DATE OVERFLOW POLICY #
++###################################
++
++# Defines how to handle INSERT requests with TTL exceeding the maximum supported expiration date:
++# * REJECT: this is the default policy and will reject any requests with expiration date timestamp after 2038-01-19T03:14:06+00:00.
++# * CAP: any insert with TTL expiring after 2038-01-19T03:14:06+00:00 will expire on 2038-01-19T03:14:06+00:00 and the client will receive a warning.
++# * CAP_NOWARN: same as previous, except that the client warning will not be emitted.
++#
++#-Dcassandra.expiration_date_overflow_policy=REJECT
++
+#################
+# GC SETTINGS #
+#################
+
+### CMS Settings
+
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:SurvivorRatio=8
+-XX:MaxTenuringThreshold=1
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+-XX:CMSWaitDuration=10000
+-XX:+CMSParallelInitialMarkEnabled
+-XX:+CMSEdenChunksRecordAlways
+# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
+-XX:+CMSClassUnloadingEnabled
+
+### G1 Settings (experimental, comment previous section and uncomment section below to enable)
+
+## Use the Hotspot garbage-first collector.
+#-XX:+UseG1GC
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+#-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+#-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+### GC logging options -- uncomment to enable
+
+-XX:+PrintGCDetails
+-XX:+PrintGCDateStamps
+-XX:+PrintHeapAtGC
+-XX:+PrintTenuringDistribution
+-XX:+PrintGCApplicationStoppedTime
+-XX:+PrintPromotionFailure
+#-XX:PrintFLSStatistics=1
+#-Xloggc:/var/log/cassandra/gc.log
+-XX:+UseGCLogFileRotation
+-XX:NumberOfGCLogFiles=10
+-XX:GCLogFileSize=10M
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/redhat/cassandra.spec
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/src/java/org/apache/cassandra/cql3/Attributes.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/Attributes.java
index 4ed0f83,84f423a..832d0a7
--- a/src/java/org/apache/cassandra/cql3/Attributes.java
+++ b/src/java/org/apache/cassandra/cql3/Attributes.java
@@@ -18,12 -18,17 +18,11 @@@
package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
--import java.util.Collections;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.collect.Iterables;
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.List;
- import com.google.common.collect.Iterables;
-
+ import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.db.ExpiringCell;
++import org.apache.cassandra.db.ExpirationDateOverflowHandling;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.exceptions.InvalidRequestException;
@@@ -100,10 -137,13 +99,13 @@@ public class Attribute
return LongType.instance.compose(tval);
}
- public int getTimeToLive(QueryOptions options) throws InvalidRequestException
+ public int getTimeToLive(QueryOptions options, CFMetaData metadata) throws InvalidRequestException
{
if (timeToLive == null)
- return 0;
+ {
- maybeApplyExpirationDateOverflowPolicy(metadata, metadata.getDefaultTimeToLive(), true);
- return metadata.getDefaultTimeToLive();
++ ExpirationDateOverflowHandling.maybeApplyExpirationDateOverflowPolicy(metadata, metadata.params.defaultTimeToLive, true);
++ return metadata.params.defaultTimeToLive;
+ }
ByteBuffer tval = timeToLive.bindAndGet(options);
if (tval == null)
@@@ -125,9 -165,11 +127,11 @@@
if (ttl < 0)
throw new InvalidRequestException("A TTL must be greater or equal to 0, but was " + ttl);
- if (ttl > ExpiringCell.MAX_TTL)
- throw new InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", ttl, ExpiringCell.MAX_TTL));
+ if (ttl > MAX_TTL)
+ throw new InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", ttl, MAX_TTL));
- maybeApplyExpirationDateOverflowPolicy(metadata, ttl, false);
++ ExpirationDateOverflowHandling.maybeApplyExpirationDateOverflowPolicy(metadata, ttl, false);
+
return ttl;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 536681f,45908de..b5946bb
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1379,13 -1610,12 +1379,13 @@@ public class ColumnFamilyStore implemen
return CompactionManager.instance.performCleanup(ColumnFamilyStore.this, jobs);
}
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs) throws ExecutionException, InterruptedException
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTLRows, int jobs) throws ExecutionException, InterruptedException
++ public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs) throws ExecutionException, InterruptedException
{
- return scrub(disableSnapshot, skipCorrupted, false, checkData, jobs);
- return scrub(disableSnapshot, skipCorrupted, false, checkData, reinsertOverflowedTTLRows, jobs);
++ return scrub(disableSnapshot, skipCorrupted, reinsertOverflowedTTL, false, checkData, jobs);
}
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean alwaysFail, boolean checkData, boolean reinsertOverflowedTTLRows, int jobs) throws ExecutionException, InterruptedException
+ @VisibleForTesting
- public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean alwaysFail, boolean checkData, int jobs) throws ExecutionException, InterruptedException
++ public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean reinsertOverflowedTTL, boolean alwaysFail, boolean checkData, int jobs) throws ExecutionException, InterruptedException
{
// skip snapshot creation during scrub, SEE JIRA 5891
if(!disableSnapshot)
@@@ -1393,7 -1623,7 +1393,7 @@@
try
{
- return CompactionManager.instance.performScrub(ColumnFamilyStore.this, skipCorrupted, checkData, jobs);
- return CompactionManager.instance.performScrub(ColumnFamilyStore.this, skipCorrupted, checkData, reinsertOverflowedTTLRows, jobs);
++ return CompactionManager.instance.performScrub(ColumnFamilyStore.this, skipCorrupted, checkData, reinsertOverflowedTTL, jobs);
}
catch(Throwable t)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/c231ed5b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
index 0000000,0000000..852dcb1
new file mode 100644
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/ExpirationDateOverflowHandling.java
@@@ -1,0 -1,0 +1,121 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.cassandra.db;
++
++import java.util.concurrent.TimeUnit;
++
++import com.google.common.annotations.VisibleForTesting;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++import org.slf4j.helpers.MessageFormatter;
++
++import org.apache.cassandra.config.CFMetaData;
++import org.apache.cassandra.cql3.Attributes;
++import org.apache.cassandra.db.rows.BufferCell;
++import org.apache.cassandra.db.rows.Cell;
++import org.apache.cassandra.exceptions.InvalidRequestException;
++import org.apache.cassandra.service.ClientWarn;
++import org.apache.cassandra.utils.NoSpamLogger;
++
++public class ExpirationDateOverflowHandling
++{
++ private static final Logger logger = LoggerFactory.getLogger(Attributes.class);
++
++ private static final int EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES = Integer.getInteger("cassandra.expiration_overflow_warning_interval_minutes", 5);
++
++ public enum ExpirationDateOverflowPolicy
++ {
++ REJECT, CAP_NOWARN, CAP
++ }
++
++ @VisibleForTesting
++ public static ExpirationDateOverflowPolicy policy;
++
++ static {
++ String policyAsString = System.getProperty("cassandra.expiration_date_overflow_policy", ExpirationDateOverflowPolicy.REJECT.name());
++ try
++ {
++ policy = ExpirationDateOverflowPolicy.valueOf(policyAsString.toUpperCase());
++ }
++ catch (RuntimeException e)
++ {
++ logger.warn("Invalid expiration date overflow policy: {}. Using default: {}", policyAsString, ExpirationDateOverflowPolicy.REJECT.name());
++ policy = ExpirationDateOverflowPolicy.REJECT;
++ }
++ }
++
++ public static final String MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING = "Request on table {}.{} with {}ttl of {} seconds exceeds maximum supported expiration " +
++ "date of 2038-01-19T03:14:06+00:00 and will have its expiration capped to that date. " +
++ "In order to avoid this use a lower TTL or upgrade to a version where this limitation " +
++ "is fixed. See CASSANDRA-14092 for more details.";
++
++ public static final String MAXIMUM_EXPIRATION_DATE_EXCEEDED_REJECT_MESSAGE = "Request on table %s.%s with %sttl of %d seconds exceeds maximum supported expiration " +
++ "date of 2038-01-19T03:14:06+00:00. In order to avoid this use a lower TTL, change " +
++ "the expiration date overflow policy or upgrade to a version where this limitation " +
++ "is fixed. See CASSANDRA-14092 for more details.";
++
++ public static void maybeApplyExpirationDateOverflowPolicy(CFMetaData metadata, int ttl, boolean isDefaultTTL) throws InvalidRequestException
++ {
++ if (ttl == BufferCell.NO_TTL)
++ return;
++
++ // Check for localExpirationTime overflow (CASSANDRA-14092)
++ int nowInSecs = (int)(System.currentTimeMillis() / 1000);
++ if (ttl + nowInSecs < 0)
++ {
++ switch (policy)
++ {
++ case CAP:
++ ClientWarn.instance.warn(MessageFormatter.arrayFormat(MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING, new Object[] { metadata.ksName,
++ metadata.cfName,
++ isDefaultTTL? "default " : "", ttl })
++ .getMessage());
++ case CAP_NOWARN:
++ /**
++ * Capping at this stage is basically not rejecting the request. The actual capping is done
++ * by {@link #computeLocalExpirationTime(int, int)}, which converts the negative TTL
++ * to {@link org.apache.cassandra.db.BufferExpiringCell#MAX_DELETION_TIME}
++ */
++ NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, EXPIRATION_OVERFLOW_WARNING_INTERVAL_MINUTES, TimeUnit.MINUTES, MAXIMUM_EXPIRATION_DATE_EXCEEDED_WARNING,
++ metadata.ksName, metadata.cfName, isDefaultTTL? "default " : "", ttl);
++ return;
++
++ default:
++ throw new InvalidRequestException(String.format(MAXIMUM_EXPIRATION_DATE_EXCEEDED_REJECT_MESSAGE, metadata.ksName, metadata.cfName,
++ isDefaultTTL? "default " : "", ttl));
++ }
++ }
++ }
++
++ /**
++ * This method computes the {@link Cell#localDeletionTime()}, maybe capping to the maximum representable value
++ * which is {@link Cell#MAX_DELETION_TIME}.
++ *
++ * Please note that the {@link ExpirationDateOverflowHandling.ExpirationDateOverflowPolicy} is applied
++ * during {@link ExpirationDateOverflowHandling#maybeApplyExpirationDateOverflowPolicy(CFMetaData, int, boolean)},
++ * so if the request was not denied it means its expiration date should be capped.
++ *
++ * See CASSANDRA-14092
++ */
++ public static int computeLocalExpirationTime(int nowInSec, int timeToLive)
++ {
++ int localExpirationTime = nowInSec + timeToLive;
++ return localExpirationTime >= 0? localExpirationTime : Cell.MAX_DELETION_TIME;
++ }
++}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org