You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2017/09/13 13:37:03 UTC

[1/6] cassandra git commit: Wait for all management tasks to complete before shutting down CLSM

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-3.0 9882cd825 -> d826c8187
  refs/heads/cassandra-3.11 60a2d97a6 -> cbc26d81d
  refs/heads/trunk 2b57fb67d -> e7e936735


Wait for all management tasks to complete before shutting down CLSM

patch by  Jan Urbański; reviewed by jasobrown for CASSANDRA-13123


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d826c818
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d826c818
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d826c818

Branch: refs/heads/cassandra-3.0
Commit: d826c81874e5d13a30a418f8b982531cb7e5d158
Parents: 9882cd8
Author: Jan Urbański <ja...@newrelic.com>
Authored: Sat Jan 14 21:26:52 2017 +0100
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:28:47 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../db/commitlog/CommitLogSegmentManager.java   |  8 ++++--
 .../commitlog/CommitLogSegmentManagerTest.java  | 30 ++++++++++++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3d3903e..4d2d5e7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.15
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
  * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
  * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
  * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 79dd316..7651d1c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -108,14 +108,18 @@ public class CommitLogSegmentManager
         {
             public void runMayThrow() throws Exception
             {
-                while (run)
+                while (true)
                 {
                     try
                     {
                         Runnable task = segmentManagementTasks.poll();
                         if (task == null)
                         {
-                            // if we have no more work to do, check if we should create a new segment
+                            // if we have no more work to do, check if we were requested to exit before starting background tasks
+                            if (!run)
+                                return;
+
+                            // check if we should create a new segment
                             if (!atSegmentLimit() && availableSegments.isEmpty() && (activeSegments.isEmpty() || createReserveSegments))
                             {
                                 logger.trace("No segments in reserve; creating a fresh one");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index a9b0669..41f5ed5 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@ -20,8 +20,10 @@
  */
 package org.apache.cassandra.db.commitlog;
 
+import java.io.File;
 import java.nio.ByteBuffer;
 import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.Semaphore;
 
 import com.google.common.collect.ImmutableMap;
@@ -134,4 +136,32 @@ public class CommitLogSegmentManagerTest
             Thread.currentThread().interrupt();
         }
     }
+
+    @Test
+    @BMRule(name = "Make removing commitlog segments slow",
+            targetClass = "CommitLogSegment",
+            targetMethod = "discard",
+            action = "Thread.sleep(50)")
+    public void testShutdownWithPendingTasks() throws Throwable {
+        CommitLog.instance.resetUnsafe(true);
+        ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+        final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+                           .clustering("bytes")
+                           .add("val", ByteBuffer.wrap(entropy))
+                           .build();
+
+        // force creating several commitlog files
+        for (int i = 0; i < 10; i++) {
+            CommitLog.instance.add(m);
+        }
+
+        // schedule discarding completed segments and immediately issue a shutdown
+        UUID cfid = m.getColumnFamilyIds().iterator().next();
+        CommitLog.instance.discardCompletedSegments(cfid, ReplayPosition.NONE, CommitLog.instance.getContext());
+        CommitLog.instance.shutdownBlocking();
+
+        // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
+        Assert.assertTrue(new File(CommitLog.instance.location).listFiles().length <= 2);
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[2/6] cassandra git commit: Wait for all management tasks to complete before shutting down CLSM

Posted by ja...@apache.org.
Wait for all management tasks to complete before shutting down CLSM

patch by  Jan Urbański; reviewed by jasobrown for CASSANDRA-13123


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d826c818
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d826c818
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d826c818

Branch: refs/heads/cassandra-3.11
Commit: d826c81874e5d13a30a418f8b982531cb7e5d158
Parents: 9882cd8
Author: Jan Urbański <ja...@newrelic.com>
Authored: Sat Jan 14 21:26:52 2017 +0100
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:28:47 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../db/commitlog/CommitLogSegmentManager.java   |  8 ++++--
 .../commitlog/CommitLogSegmentManagerTest.java  | 30 ++++++++++++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3d3903e..4d2d5e7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.15
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
  * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
  * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
  * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 79dd316..7651d1c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -108,14 +108,18 @@ public class CommitLogSegmentManager
         {
             public void runMayThrow() throws Exception
             {
-                while (run)
+                while (true)
                 {
                     try
                     {
                         Runnable task = segmentManagementTasks.poll();
                         if (task == null)
                         {
-                            // if we have no more work to do, check if we should create a new segment
+                            // if we have no more work to do, check if we were requested to exit before starting background tasks
+                            if (!run)
+                                return;
+
+                            // check if we should create a new segment
                             if (!atSegmentLimit() && availableSegments.isEmpty() && (activeSegments.isEmpty() || createReserveSegments))
                             {
                                 logger.trace("No segments in reserve; creating a fresh one");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index a9b0669..41f5ed5 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@ -20,8 +20,10 @@
  */
 package org.apache.cassandra.db.commitlog;
 
+import java.io.File;
 import java.nio.ByteBuffer;
 import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.Semaphore;
 
 import com.google.common.collect.ImmutableMap;
@@ -134,4 +136,32 @@ public class CommitLogSegmentManagerTest
             Thread.currentThread().interrupt();
         }
     }
+
+    @Test
+    @BMRule(name = "Make removing commitlog segments slow",
+            targetClass = "CommitLogSegment",
+            targetMethod = "discard",
+            action = "Thread.sleep(50)")
+    public void testShutdownWithPendingTasks() throws Throwable {
+        CommitLog.instance.resetUnsafe(true);
+        ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+        final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+                           .clustering("bytes")
+                           .add("val", ByteBuffer.wrap(entropy))
+                           .build();
+
+        // force creating several commitlog files
+        for (int i = 0; i < 10; i++) {
+            CommitLog.instance.add(m);
+        }
+
+        // schedule discarding completed segments and immediately issue a shutdown
+        UUID cfid = m.getColumnFamilyIds().iterator().next();
+        CommitLog.instance.discardCompletedSegments(cfid, ReplayPosition.NONE, CommitLog.instance.getContext());
+        CommitLog.instance.shutdownBlocking();
+
+        // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
+        Assert.assertTrue(new File(CommitLog.instance.location).listFiles().length <= 2);
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk

Posted by ja...@apache.org.
Merge branch 'cassandra-3.11' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e7e93673
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e7e93673
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e7e93673

Branch: refs/heads/trunk
Commit: e7e936735df8e0fdd7e7d0df92d7bd60c4602648
Parents: 2b57fb6 cbc26d8
Author: Jason Brown <ja...@gmail.com>
Authored: Wed Sep 13 06:30:50 2017 -0700
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:31:37 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../commitlog/CommitLogSegmentManagerTest.java  | 31 ++++++++++++++++++++
 2 files changed, 32 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/e7e93673/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e7e93673/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index 06513a5,dfbf5ad..5db7149
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@@ -44,6 -46,6 +46,7 @@@ import org.apache.cassandra.db.compacti
  import org.apache.cassandra.db.marshal.AsciiType;
  import org.apache.cassandra.db.marshal.BytesType;
  import org.apache.cassandra.schema.KeyspaceParams;
++import org.apache.cassandra.schema.TableId;
  import org.jboss.byteman.contrib.bmunit.BMRule;
  import org.jboss.byteman.contrib.bmunit.BMRules;
  import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
@@@ -139,4 -141,32 +142,32 @@@ public class CommitLogSegmentManagerTes
              Thread.currentThread().interrupt();
          }
      }
+ 
+     @Test
+     @BMRule(name = "Make removing commitlog segments slow",
+             targetClass = "CommitLogSegment",
+             targetMethod = "discard",
+             action = "Thread.sleep(50)")
+     public void testShutdownWithPendingTasks() throws Throwable {
+         CommitLog.instance.resetUnsafe(true);
+         ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ 
 -        final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
++        final Mutation m = new RowUpdateBuilder(cfs1.metadata.get(), 0, "k")
+                            .clustering("bytes")
+                            .add("val", ByteBuffer.wrap(entropy))
+                            .build();
+ 
+         // force creating several commitlog files
+         for (int i = 0; i < 10; i++) {
+             CommitLog.instance.add(m);
+         }
+ 
+         // schedule discarding completed segments and immediately issue a shutdown
 -        UUID cfid = m.getColumnFamilyIds().iterator().next();
 -        CommitLog.instance.discardCompletedSegments(cfid, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
++        TableId tableId = m.getTableIds().iterator().next();
++        CommitLog.instance.discardCompletedSegments(tableId, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+         CommitLog.instance.shutdownBlocking();
+ 
+         // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
+         Assert.assertTrue(new File(DatabaseDescriptor.getCommitLogLocation()).listFiles().length <= 2);
+     }
  }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[4/6] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11

Posted by ja...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cbc26d81
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cbc26d81
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cbc26d81

Branch: refs/heads/trunk
Commit: cbc26d81d53822d16eb83052a69426e4f61f77ef
Parents: 60a2d97 d826c81
Author: Jason Brown <ja...@gmail.com>
Authored: Wed Sep 13 06:29:42 2017 -0700
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:30:33 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../commitlog/CommitLogSegmentManagerTest.java  | 30 ++++++++++++++++++++
 2 files changed, 31 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cbc26d81/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ec9d126,4d2d5e7..b3fb5a6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,14 -1,5 +1,15 @@@
 -3.0.15
 +3.11.1
 + * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
 + * BTree.Builder memory leak (CASSANDRA-13754)
 + * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
 + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
 + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
 + * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
 + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
 + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
 + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
 +Merged from 3.0:
+  * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
   * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
   * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
   * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cbc26d81/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index e22e86f,41f5ed5..dfbf5ad
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@@ -20,9 -20,10 +20,11 @@@
   */
  package org.apache.cassandra.db.commitlog;
  
+ import java.io.File;
  import java.nio.ByteBuffer;
 +import java.util.ArrayList;
  import java.util.Random;
+ import java.util.UUID;
  import java.util.concurrent.Semaphore;
  
  import com.google.common.collect.ImmutableMap;
@@@ -139,4 -136,32 +141,32 @@@ public class CommitLogSegmentManagerTes
              Thread.currentThread().interrupt();
          }
      }
+ 
+     @Test
+     @BMRule(name = "Make removing commitlog segments slow",
+             targetClass = "CommitLogSegment",
+             targetMethod = "discard",
+             action = "Thread.sleep(50)")
+     public void testShutdownWithPendingTasks() throws Throwable {
+         CommitLog.instance.resetUnsafe(true);
+         ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ 
+         final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+                            .clustering("bytes")
+                            .add("val", ByteBuffer.wrap(entropy))
+                            .build();
+ 
+         // force creating several commitlog files
+         for (int i = 0; i < 10; i++) {
+             CommitLog.instance.add(m);
+         }
+ 
+         // schedule discarding completed segments and immediately issue a shutdown
+         UUID cfid = m.getColumnFamilyIds().iterator().next();
 -        CommitLog.instance.discardCompletedSegments(cfid, ReplayPosition.NONE, CommitLog.instance.getContext());
++        CommitLog.instance.discardCompletedSegments(cfid, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+         CommitLog.instance.shutdownBlocking();
+ 
+         // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
 -        Assert.assertTrue(new File(CommitLog.instance.location).listFiles().length <= 2);
++        Assert.assertTrue(new File(DatabaseDescriptor.getCommitLogLocation()).listFiles().length <= 2);
+     }
  }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[5/6] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11

Posted by ja...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.11


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cbc26d81
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cbc26d81
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cbc26d81

Branch: refs/heads/cassandra-3.11
Commit: cbc26d81d53822d16eb83052a69426e4f61f77ef
Parents: 60a2d97 d826c81
Author: Jason Brown <ja...@gmail.com>
Authored: Wed Sep 13 06:29:42 2017 -0700
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:30:33 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../commitlog/CommitLogSegmentManagerTest.java  | 30 ++++++++++++++++++++
 2 files changed, 31 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cbc26d81/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ec9d126,4d2d5e7..b3fb5a6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,14 -1,5 +1,15 @@@
 -3.0.15
 +3.11.1
 + * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
 + * BTree.Builder memory leak (CASSANDRA-13754)
 + * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
 + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
 + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
 + * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
 + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
 + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
 + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
 +Merged from 3.0:
+  * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
   * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
   * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
   * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cbc26d81/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index e22e86f,41f5ed5..dfbf5ad
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@@ -20,9 -20,10 +20,11 @@@
   */
  package org.apache.cassandra.db.commitlog;
  
+ import java.io.File;
  import java.nio.ByteBuffer;
 +import java.util.ArrayList;
  import java.util.Random;
+ import java.util.UUID;
  import java.util.concurrent.Semaphore;
  
  import com.google.common.collect.ImmutableMap;
@@@ -139,4 -136,32 +141,32 @@@ public class CommitLogSegmentManagerTes
              Thread.currentThread().interrupt();
          }
      }
+ 
+     @Test
+     @BMRule(name = "Make removing commitlog segments slow",
+             targetClass = "CommitLogSegment",
+             targetMethod = "discard",
+             action = "Thread.sleep(50)")
+     public void testShutdownWithPendingTasks() throws Throwable {
+         CommitLog.instance.resetUnsafe(true);
+         ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+ 
+         final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+                            .clustering("bytes")
+                            .add("val", ByteBuffer.wrap(entropy))
+                            .build();
+ 
+         // force creating several commitlog files
+         for (int i = 0; i < 10; i++) {
+             CommitLog.instance.add(m);
+         }
+ 
+         // schedule discarding completed segments and immediately issue a shutdown
+         UUID cfid = m.getColumnFamilyIds().iterator().next();
 -        CommitLog.instance.discardCompletedSegments(cfid, ReplayPosition.NONE, CommitLog.instance.getContext());
++        CommitLog.instance.discardCompletedSegments(cfid, CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition());
+         CommitLog.instance.shutdownBlocking();
+ 
+         // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
 -        Assert.assertTrue(new File(CommitLog.instance.location).listFiles().length <= 2);
++        Assert.assertTrue(new File(DatabaseDescriptor.getCommitLogLocation()).listFiles().length <= 2);
+     }
  }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[3/6] cassandra git commit: Wait for all management tasks to complete before shutting down CLSM

Posted by ja...@apache.org.
Wait for all management tasks to complete before shutting down CLSM

patch by  Jan Urbański; reviewed by jasobrown for CASSANDRA-13123


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d826c818
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d826c818
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d826c818

Branch: refs/heads/trunk
Commit: d826c81874e5d13a30a418f8b982531cb7e5d158
Parents: 9882cd8
Author: Jan Urbański <ja...@newrelic.com>
Authored: Sat Jan 14 21:26:52 2017 +0100
Committer: Jason Brown <ja...@gmail.com>
Committed: Wed Sep 13 06:28:47 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../db/commitlog/CommitLogSegmentManager.java   |  8 ++++--
 .../commitlog/CommitLogSegmentManagerTest.java  | 30 ++++++++++++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3d3903e..4d2d5e7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.15
+ * Wait for all management tasks to complete before shutting down CLSM (CASSANDRA-13123)
  * INSERT statement fails when Tuple type is used as clustering column with default DESC order (CASSANDRA-13717)
  * Fix pending view mutations handling and cleanup batchlog when there are local and remote paired mutations (CASSANDRA-13069)
  * Improve config validation and documentation on overflow and NPE (CASSANDRA-13622)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
index 79dd316..7651d1c 100644
--- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
+++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegmentManager.java
@@ -108,14 +108,18 @@ public class CommitLogSegmentManager
         {
             public void runMayThrow() throws Exception
             {
-                while (run)
+                while (true)
                 {
                     try
                     {
                         Runnable task = segmentManagementTasks.poll();
                         if (task == null)
                         {
-                            // if we have no more work to do, check if we should create a new segment
+                            // if we have no more work to do, check if we were requested to exit before starting background tasks
+                            if (!run)
+                                return;
+
+                            // check if we should create a new segment
                             if (!atSegmentLimit() && availableSegments.isEmpty() && (activeSegments.isEmpty() || createReserveSegments))
                             {
                                 logger.trace("No segments in reserve; creating a fresh one");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d826c818/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
index a9b0669..41f5ed5 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerTest.java
@@ -20,8 +20,10 @@
  */
 package org.apache.cassandra.db.commitlog;
 
+import java.io.File;
 import java.nio.ByteBuffer;
 import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.Semaphore;
 
 import com.google.common.collect.ImmutableMap;
@@ -134,4 +136,32 @@ public class CommitLogSegmentManagerTest
             Thread.currentThread().interrupt();
         }
     }
+
+    @Test
+    @BMRule(name = "Make removing commitlog segments slow",
+            targetClass = "CommitLogSegment",
+            targetMethod = "discard",
+            action = "Thread.sleep(50)")
+    public void testShutdownWithPendingTasks() throws Throwable {
+        CommitLog.instance.resetUnsafe(true);
+        ColumnFamilyStore cfs1 = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+        final Mutation m = new RowUpdateBuilder(cfs1.metadata, 0, "k")
+                           .clustering("bytes")
+                           .add("val", ByteBuffer.wrap(entropy))
+                           .build();
+
+        // force creating several commitlog files
+        for (int i = 0; i < 10; i++) {
+            CommitLog.instance.add(m);
+        }
+
+        // schedule discarding completed segments and immediately issue a shutdown
+        UUID cfid = m.getColumnFamilyIds().iterator().next();
+        CommitLog.instance.discardCompletedSegments(cfid, ReplayPosition.NONE, CommitLog.instance.getContext());
+        CommitLog.instance.shutdownBlocking();
+
+        // the shutdown should block until all logs except the currently active one and perhaps a new, empty one are gone
+        Assert.assertTrue(new File(CommitLog.instance.location).listFiles().length <= 2);
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org